Java 类org.apache.hadoop.mapreduce.lib.db.DBInputFormat 实例源码

项目：bigdata_pattern 文件：DBToFileMapReduce.java

public static void main(String[] args) throws IOException, Exception {  
 Configuration conf = new Configuration();
 Job job = new Job(conf);
 job.setJarByClass(DBToFileMapReduce.class);  
 DistributedCache.addFileToClassPath(new Path(  
       "/lib/mysql-connector-java-5.1.0-bin.jar"), conf);  

 job.setMapperClass(DBInputMapper.class);  
 job.setReducerClass(Reducer.class); 

 job.setMapOutputKeyClass(LongWritable.class);  
 job.setMapOutputValueClass(Text.class);  
 job.setOutputKeyClass(LongWritable.class);  
 job.setOutputValueClass(Text.class);  

 job.setInputFormatClass(DBInputFormat.class); 

 Path outputPath = new Path("/hua01");
 FileOutputFormat.setOutputPath(job, outputPath);  
 outputPath.getFileSystem(conf).delete(outputPath, true);

 DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver",  
       "jdbc:mysql://192.168.3.244:3306/hadoop", "hua", "hadoop");  
 String[] fields = { "id", "name" };  
 DBInputFormat.setInput(job, StudentinfoRecord.class, "studentinfo", null, "id", fields);   

 job.waitForCompletion(true); 
}

项目：hadoop 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：aliyun-oss-hadoop-fs 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = Job.getInstance(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "HAccess"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：big-c 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hadoop-2.6.0-cdh5.4.3 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hadoop-plus 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：FlexMap 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hops 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：glusterfs-hadoop-examples 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hadoop-TCP 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hardfs 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：hadoop-on-lustre2 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：mapreduce-fork 文件：DBCountPageView.java

@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;

  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }

  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);

  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}

项目：lembos 文件：DBInputFormatHelper.java

/**
 * Validate whether class is {@link DBInputFormat} or a subclass.
 *
 * @param clazz the class to validate
 * @param ctx the context to throw the error in if class is invalid
 * @param scope the scope to throw the error in if class is invalid
 */
public static void validateClass(final Class<?> clazz, final Context ctx, final Scriptable scope) {
    if (!ReflectionUtils.isClassOrSubclass(DBInputFormat.class, clazz)) {
        throw Utils.makeError(ctx, scope, LembosMessages.makeInvalidClassErrorMessage(DBInputFormat.class,
                                                                                      clazz));
    }
}

项目：lembos 文件：DBInputFormatWrap.java

/**
 * Java wrapper for {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String)} and
 * {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String, String, String...)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void setInput(final Context ctx, final Scriptable thisObj, final Object[] args,
                                final Function func) {
    DBInputFormatHelper.setInput(DBInputFormat.class, ctx, thisObj, args, func);
}