Java 类org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat 实例源码

private static SortedSet<byte[]> readFileToSearch(final Configuration conf,
    final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException,
    InterruptedException {
  SortedSet<byte []> result = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
  // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is
  // what is missing.
  TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
  try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr =
      new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) {
    InputSplit is =
      new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {});
    rr.initialize(is, context);
    while (rr.nextKeyValue()) {
      rr.getCurrentKey();
      BytesWritable bw = rr.getCurrentValue();
      if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) {
        byte[] key = new byte[rr.getCurrentKey().getLength()];
        System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey()
            .getLength());
        result.add(key);
      }
    }
  }
  return result;
}

private static SortedSet<byte[]> readFileToSearch(final Configuration conf,
    final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException,
    InterruptedException {
  SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is
  // what is missing.
  TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
  try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr =
      new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) {
    InputSplit is =
      new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {});
    rr.initialize(is, context);
    while (rr.nextKeyValue()) {
      rr.getCurrentKey();
      BytesWritable bw = rr.getCurrentValue();
      if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) {
        byte[] key = new byte[rr.getCurrentKey().getLength()];
        System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey()
            .getLength());
        result.add(key);
      }
    }
  }
  return result;
}

/**
 * Java wrapper for
 * {@link SequenceFileAsBinaryInputFormat#addInputPath(org.apache.hadoop.mapreduce.Job, org.apache.hadoop.fs.Path)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void addInputPath(final Context ctx, final Scriptable thisObj, final Object[] args,
                                final Function func) {
    FileInputFormatHelper.addInputPath(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for {@link SequenceFileAsBinaryInputFormat#addInputPaths(org.apache.hadoop.mapreduce.Job, String)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void addInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args,
                                 final Function func) {
    FileInputFormatHelper.addInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for
 * {@link SequenceFileAsBinaryInputFormat#getInputPathFilter(org.apache.hadoop.mapreduce.JobContext)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 *
 * @return class name for the input path filter or undefined
 */
@JSStaticFunction
public static Object getInputPathFilter(final Context ctx, final Scriptable thisObj, final Object[] args,
                                        final Function func) {
    return FileInputFormatHelper.getInputPathFilter(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getInputPaths(org.apache.hadoop.mapreduce.JobContext)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 *
 * @return array of input paths
 */
@JSStaticFunction
public static Object getInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args,
                                   final Function func) {
    return FileInputFormatHelper.getInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getMaxSplitSize(org.apache.hadoop.mapreduce.JobContext)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 *
 * @return the max split size
 */
@JSStaticFunction
public static Object getMaxSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args,
                                     final Function func) {
    return FileInputFormatHelper.getMaxSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for {@link SequenceFileAsBinaryInputFormat#getMinSplitSize(org.apache.hadoop.mapreduce.JobContext)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 *
 * @return the max split size
 */
@JSStaticFunction
public static Object getMinSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args,
                                     final Function func) {
   return FileInputFormatHelper.getMinSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Wraps {@link SequenceFileAsBinaryInputFormat#setInputPathFilter(org.apache.hadoop.mapreduce.Job, Class)}.
 *
 * @param ctx the JavaScript context (unused)
 * @param thisObj the 'this' object of the caller
 * @param args the arguments for the call
 * @param func the function called (unused)
 */
@JSStaticFunction
public static void setInputPathFilter(final Context ctx, final Scriptable thisObj, final Object[] args,
                                        final Function func) {
    FileInputFormatHelper.setInputPathFilter(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for
 * {@link SequenceFileAsBinaryInputFormat#setInputPaths(org.apache.hadoop.mapreduce.Job,
 *                                                      org.apache.hadoop.fs.Path...)} and
 * {@link SequenceFileAsBinaryInputFormat#setInputPaths(org.apache.hadoop.mapreduce.Job, String)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void setInputPaths(final Context ctx, final Scriptable thisObj, final Object[] args,
                                 final Function func) {
    FileInputFormatHelper.setInputPaths(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for
 * {@link SequenceFileAsBinaryInputFormat#setMaxInputSplitSize(org.apache.hadoop.mapreduce.Job, long)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void setMaxInputSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args,
                                        final Function func) {
    FileInputFormatHelper.setMaxInputSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}

/**
 * Java wrapper for
 * {@link SequenceFileAsBinaryInputFormat#setMinInputSplitSize(org.apache.hadoop.mapreduce.Job, long)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void setMinInputSplitSize(final Context ctx, final Scriptable thisObj, final Object[] args,
                                        final Function func) {
    FileInputFormatHelper.setMinInputSplitSize(SequenceFileAsBinaryInputFormat.class, ctx, thisObj, args);
}