1. 包列表
  2. 类列表
  3. org.apache.hadoop.mapred.gridmix.FilePool

Java 类org.apache.hadoop.mapred.gridmix.FilePool 实例源码

项目：hadoop-2.6.0-cdh5.4.3    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSize = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = 13;
  String [] runtimeValues ={"LOADJOB",
     SubmitterUserResolver.class.getName(),
     "STRESS",
     inputSize+"m",
     "file:///dev/null"}; 

  int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
     conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // create files for given sizes.
  createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
   long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
  long expFilesSize = 3100 ;
  Assert.assertEquals("Files Size has not matched for specified pool.",
     expFilesSize, actFilesSize);
  int actFileCount = files.size();    
  Assert.assertEquals("File count has not matched.", 
     expFileCount, actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
          clusterSize, count);
}
项目：hadoop-on-lustre    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSize = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = 13;
  String [] runtimeValues ={"LOADJOB",
     SubmitterUserResolver.class.getName(),
     "STRESS",
     inputSize+"m",
     "file:///dev/null"}; 

  int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
     conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // create files for given sizes.
  createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
}
项目：hadoop-on-lustre    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
   long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
  long expFilesSize = 3100 ;
  Assert.assertEquals("Files Size has not matched for specified pool.",
     expFilesSize, actFilesSize);
  int actFileCount = files.size();    
  Assert.assertEquals("File count has not matched.", 
     expFileCount, actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
          clusterSize, count);
}
项目：hanoi-hadoop-2.0.0-cdh    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSize = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = 13;
  String [] runtimeValues ={"LOADJOB",
     SubmitterUserResolver.class.getName(),
     "STRESS",
     inputSize+"m",
     "file:///dev/null"}; 

  int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
     conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // create files for given sizes.
  createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
}
项目：hanoi-hadoop-2.0.0-cdh    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
   long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
  long expFilesSize = 3100 ;
  Assert.assertEquals("Files Size has not matched for specified pool.",
     expFilesSize, actFilesSize);
  int actFileCount = files.size();    
  Assert.assertEquals("File count has not matched.", 
     expFileCount, actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
          clusterSize, count);
}
项目：mapreduce-fork    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
    long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long filesSizeInBytes = pool.getInputFiles(minFileSize, files);
  long actFilesSizeInMB = filesSizeInBytes / (1024 * 1024);
  long expFilesSizeInMB = (clusterSize * 200) + 1300;
  Assert.assertEquals("Files Size has not matched for specified pool.", 
                      expFilesSizeInMB, actFilesSizeInMB);
  int actFileCount = files.size();
  Assert.assertEquals("File count has not matched.", expFileCount, 
                      actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
                      clusterSize, count);
}
项目：hortonworks-extension    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSize = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = 13;
  String [] runtimeValues ={"LOADJOB",
     SubmitterUserResolver.class.getName(),
     "STRESS",
     inputSize+"m",
     "file:///dev/null"}; 

  int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
     conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // create files for given sizes.
  createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
}
项目：hortonworks-extension    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
   long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
  long expFilesSize = 3100 ;
  Assert.assertEquals("Files Size has not matched for specified pool.",
     expFilesSize, actFilesSize);
  int actFileCount = files.size();    
  Assert.assertEquals("File count has not matched.", 
     expFileCount, actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
          clusterSize, count);
}
项目：hortonworks-extension    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSize = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = 13;
  String [] runtimeValues ={"LOADJOB",
     SubmitterUserResolver.class.getName(),
     "STRESS",
     inputSize+"m",
     "file:///dev/null"}; 

  int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
     conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // create files for given sizes.
  createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
}
项目：hortonworks-extension    文件：TestGridMixFilePool.java    
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
   long minFileSize, FilePool pool) throws IOException {
  final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
  long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
  long expFilesSize = 3100 ;
  Assert.assertEquals("Files Size has not matched for specified pool.",
     expFilesSize, actFilesSize);
  int actFileCount = files.size();    
  Assert.assertEquals("File count has not matched.", 
     expFileCount, actFileCount);
  int count = 0;
  for (FileStatus fstat : files) {
    String fp = fstat.getPath().toString();
    count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
  }
  Assert.assertEquals("Total folders are not matched with cluster size", 
          clusterSize, count);
}
项目：mapreduce-fork    文件：TestGridMixFilePool.java    
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
  conf = remoteClient.getDaemonConf();
  final long inputSizeInMB = clusterSize * 200;
  int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
  long targetSize = Long.MAX_VALUE;
  final int expFileCount = clusterSize + 4;
  String [] runtimeValues ={"LOADJOB",
                            SubmitterUserResolver.class.getName(),
                            "STRESS",
                            inputSizeInMB + "m",
                            "file:///dev/null"}; 

  String [] otherArgs = {
      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
  };

  // Generate the input data by using gridmix framework.
  int exitCode = 
      UtilsForGridmix.runGridmixJob(gridmixDir, conf, 
          GridMixRunMode.DATA_GENERATION.getValue(), 
          runtimeValues, otherArgs);
  Assert.assertEquals("Data generation has failed.", 0 , exitCode);
  // Create the files without using gridmix input generation with 
  // above mentioned sizes in a array.
  createFiles(new Path(gridmixDir, "input"), fileSizesInMB);
  conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
  FilePool fpool = new FilePool(conf, new Path(gridmixDir, "input"));
  fpool.refresh();
  verifyFilesSizeAndCountForSpecifiedPool(expFileCount, targetSize, fpool);
}