1. 包列表
  2. 类列表
  3. org.apache.hadoop.mapreduce.RandomTextWriter

Java 类org.apache.hadoop.mapreduce.RandomTextWriter 实例源码

项目：hadoop    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：aliyun-oss-hadoop-fs    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：big-c    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hadoop-2.6.0-cdh5.4.3    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hadoop-plus    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：FlexMap    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hops    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hadoop-TCP    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hardfs    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hadoop-on-lustre2    文件：GenericMRLoadGenerator.java    
/**
 * When no input dir is specified, generate random data.
 */
protected static void confRandom(JobConf job)
    throws IOException {
  // from RandomWriter
  job.setInputFormat(RandomInputFormat.class);
  job.setMapperClass(RandomMapOutput.class);

  final ClusterStatus cluster = new JobClient(job).getClusterStatus();
  int numMapsPerHost = job.getInt(RandomTextWriter.MAPS_PER_HOST, 10);
  long numBytesToWritePerMap =
    job.getLong(RandomTextWriter.BYTES_PER_MAP, 1*1024*1024*1024);
  if (numBytesToWritePerMap == 0) {
    throw new IOException(
        "Cannot have " + RandomTextWriter.BYTES_PER_MAP + " set to 0");
  }
  long totalBytesToWrite = job.getLong(RandomTextWriter.TOTAL_BYTES,
       numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
  int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
  if (numMaps == 0 && totalBytesToWrite > 0) {
    numMaps = 1;
    job.setLong(RandomTextWriter.BYTES_PER_MAP, totalBytesToWrite);
  }
  job.setNumMapTasks(numMaps);
}
项目：hadoop    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：aliyun-oss-hadoop-fs    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：big-c    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hadoop-2.6.0-cdh5.4.3    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hadoop-plus    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：FlexMap    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hops    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hadoop-TCP    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hardfs    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}
项目：hadoop-on-lustre2    文件：GenericMRLoadGenerator.java    
public void configure(JobConf job) {
  bytesToWrite = job.getLong(RandomTextWriter.BYTES_PER_MAP,
                                1*1024*1024*1024);
  keymin = job.getInt(RandomTextWriter.MIN_KEY, 5);
  keymax = job.getInt(RandomTextWriter.MAX_KEY, 10);
  valmin = job.getInt(RandomTextWriter.MIN_VALUE, 5);
  valmax = job.getInt(RandomTextWriter.MAX_VALUE, 10);
}