Java 类org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection 实例源码

项目：hadoop 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：aliyun-oss-hadoop-fs 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：big-c 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hadoop-2.6.0-cdh5.4.3 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hadoop-plus 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：FlexMap 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hops 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  String baseDir = System.getProperty("test.build.data", "build/test/data");
  Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test");
  Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hadoop-TCP 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hardfs 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：hadoop-on-lustre2 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}

项目：mapreduce-fork 文件：TestFieldSelection.java

public static void launch() throws Exception {
  JobConf conf = new JobConf(TestFieldSelection.class);
  FileSystem fs = FileSystem.get(conf);
  int numOfInputLines = 10;

  Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
  Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
  String inputFile = "input.txt";
  fs.delete(INPUT_DIR, true);
  fs.mkdirs(INPUT_DIR);
  fs.delete(OUTPUT_DIR, true);

  StringBuffer inputData = new StringBuffer();
  StringBuffer expectedOutput = new StringBuffer();

  TestMRFieldSelection.constructInputOutputData(inputData,
    expectedOutput, numOfInputLines);
  FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
  fileOut.write(inputData.toString().getBytes("utf-8"));
  fileOut.close();

  System.out.println("inputData:");
  System.out.println(inputData.toString());
  JobConf job = new JobConf(conf, TestFieldSelection.class);
  FileInputFormat.setInputPaths(job, INPUT_DIR);
  job.setInputFormat(TextInputFormat.class);
  job.setMapperClass(FieldSelectionMapReduce.class);
  job.setReducerClass(FieldSelectionMapReduce.class);

  FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormat(TextOutputFormat.class);
  job.setNumReduceTasks(1);

  job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
  job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
  job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");

  JobClient.runJob(job);

  //
  // Finally, we compare the reconstructed answer key with the
  // original one.  Remember, we need to ignore zero-count items
  // in the original key.
  //
  boolean success = true;
  Path outPath = new Path(OUTPUT_DIR, "part-00000");
  String outdata = MapReduceTestUtil.readOutput(outPath,job);

  assertEquals(expectedOutput.toString(),outdata);
  fs.delete(OUTPUT_DIR, true);
  fs.delete(INPUT_DIR, true);
}