Java 类java.util.zip.Checksum 实例源码

public static byte[] calCrc32(byte[] data) {
    Checksum checksum = new CRC32();
    // update the current checksum with the specified array of bytes
    checksum.update(data, 0, data.length);

    // get the current checksum value
    long checksumValue = checksum.getValue();

    String hex = Long.toHexString(checksumValue).toUpperCase();
    while (hex.length() < 8) {
        hex = "0" + hex;
    }
    byte[] crc32 = Hex.decode(hex);

    inverseBytes(crc32);
    reverseArray(crc32);
    return crc32;
}

/**
 * Compute the CRC-32 of the contents of a stream.
 * \r\n and \r are both normalized to \n for purposes of the calculation.
 */
static String computeCrc32(InputStream is) throws IOException {
    Checksum crc = new CRC32();
    int last = -1;
    int curr;
    while ((curr = is.read()) != -1) {
        if (curr != '\n' && last == '\r') {
            crc.update('\n');
        }
        if (curr != '\r') {
            crc.update(curr);
        }
        last = curr;
    }
    if (last == '\r') {
        crc.update('\n');
    }
    int val = (int)crc.getValue();
    String hex = Integer.toHexString(val);
    while (hex.length() < 8) {
        hex = "0" + hex; // NOI18N
    }
    return hex;
}

private static String computeCrc32(final FileObject fo) throws IOException {
    final Checksum crc = new CRC32();        
    try (final InputStream in = new BufferedInputStream(fo.getInputStream())) {
        int last = -1;
        int curr;
        while ((curr = in.read()) != -1) {
            if (curr != '\n' && last == '\r') { //NOI18N
                crc.update('\n');               //NOI18N
            }
            if (curr != '\r') {                 //NOI18N
                crc.update(curr);
            }
            last = curr;
        }
        if (last == '\r') {                     //NOI18N
            crc.update('\n');                   //NOI18N
        }
    }
    int val = (int)crc.getValue();
    String hex = Integer.toHexString(val);
    while (hex.length() < 8) {
        hex = "0" + hex; // NOI18N
    }
    return hex;        
}

/**
 * Generates a CRC 32 Value of String passed
 *
 * @param data
 * @return long crc32 value of input. -1 if string is null
 * @throws RuntimeException if UTF-8 is not a supported character set
 */
public static long crc32(String data) {
    if (data == null) {
        return -1;
    }

    try {
        // get bytes from string
        byte bytes[] = data.getBytes("UTF-8");
        Checksum checksum = new CRC32();
        // update the current checksum with the specified array of bytes
        checksum.update(bytes, 0, bytes.length);
        // get the current checksum value
        return checksum.getValue();
    } catch (UnsupportedEncodingException e) {
        throw new RuntimeException(e);
    }
}

private static void doBench(final List<Class<? extends Checksum>> crcs,
    final PrintStream out) throws Exception {
  final byte[] bytes = new byte[MAX_LEN];
  new Random().nextBytes(bytes);

  // Print header
  out.printf("\nPerformance Table (The unit is MB/sec; #T = #Theads)\n");

  // Warm up implementations to get jit going.
  for (Class<? extends Checksum> c : crcs) {
    doBench(c, 1, bytes, 2);
    doBench(c, 1, bytes, 2101);
  }

  // Test on a variety of sizes with different number of threads
  for (int size = 32; size <= MAX_LEN; size <<= 1) {
    doBench(crcs, bytes, size, out);
  }
}

public ZipArchiveOutputStream(OutputStream out,
                              Checksum cksum, ZipEntry e) {
  super(out, cksum);

  if (out == null) {
    throw new NullPointerException("out == null");
  }

  if (cksum == null) {
    throw new NullPointerException("cksum == null");
  }

  if (e == null) {
    throw new NullPointerException("e == null");
  }

  entry = e;
}

private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}

private static void doBench(final List<Class<? extends Checksum>> crcs,
    final PrintStream out) throws Exception {
  final byte[] bytes = new byte[MAX_LEN];
  new Random().nextBytes(bytes);

  // Print header
  out.printf("\nPerformance Table (The unit is MB/sec; #T = #Theads)\n");

  // Warm up implementations to get jit going.
  for (Class<? extends Checksum> c : crcs) {
    doBench(c, 1, bytes, 2);
    doBench(c, 1, bytes, 2101);
  }

  // Test on a variety of sizes with different number of threads
  for (int size = 32; size <= MAX_LEN; size <<= 1) {
    doBench(crcs, bytes, size, out);
  }
}

@Test
public void testUpdate() {
    final byte[] bytes = "Any String you want".getBytes();
    final int len = bytes.length;

    Checksum crc1 = Crc32C.create();
    Checksum crc2 = Crc32C.create();
    Checksum crc3 = Crc32C.create();

    crc1.update(bytes, 0, len);
    for (int i = 0; i < len; i++)
        crc2.update(bytes[i]);
    crc3.update(bytes, 0, len / 2);
    crc3.update(bytes, len / 2, len - len / 2);

    assertEquals("Crc values should be the same", crc1.getValue(), crc2.getValue());
    assertEquals("Crc values should be the same", crc1.getValue(), crc3.getValue());
}

@Test
public void testUpdate() {
    final byte[] bytes = "Any String you want".getBytes();
    final int len = bytes.length;

    Checksum crc1 = Crc32C.create();
    Checksum crc2 = Crc32C.create();
    Checksum crc3 = Crc32C.create();

    crc1.update(bytes, 0, len);
    for (int i = 0; i < len; i++)
        crc2.update(bytes[i]);
    crc3.update(bytes, 0, len / 2);
    crc3.update(bytes, len / 2, len - len / 2);

    assertEquals("Crc values should be the same", crc1.getValue(), crc2.getValue());
    assertEquals("Crc values should be the same", crc1.getValue(), crc3.getValue());
}

public static void updateChecksumInt(Checksum checksum, int v)
{
    checksum.update((v >>> 24) & 0xFF);
    checksum.update((v >>> 16) & 0xFF);
    checksum.update((v >>> 8) & 0xFF);
    checksum.update((v >>> 0) & 0xFF);
}

/**
 * Create a new instance of a Checksum object.
 * @return The newly created Checksum object
 */
static public Checksum newInstance(String className) throws IOException {
  try {
    Class<?> clazz = getClassByName(className);
    return (Checksum)newInstance(clazz);
  } catch (ClassNotFoundException e) {
    throw new IOException(e);
  }
}

private static void testLittleEndianReadonlyByteBufferOffset(Checksum checksum, long expected) {
    byte[] unaligned_bytes_123456789 = new byte[BYTES_123456789.length + 64];
    for (int i = 0; i < unaligned_bytes_123456789.length - BYTES_123456789.length; i++) {
        checksum.reset();
        System.arraycopy(BYTES_123456789, 0, unaligned_bytes_123456789, i, BYTES_123456789.length);
        ByteBuffer bb = ByteBuffer.wrap(unaligned_bytes_123456789).asReadOnlyBuffer();
        bb.order(ByteOrder.LITTLE_ENDIAN);
        bb.position(i);
        bb.limit(i + BYTES_123456789.length);
        checksum.update(bb);
        checkChecksumOffset(checksum, expected, i);
    }
}

private static void throwChecksumException(Type type, Checksum algorithm,
    String filename, long errPos, int expected, int computed)
        throws ChecksumException {
  throw new ChecksumException("Checksum " + type
      + " not matched for file " + filename + " at position "+ errPos
      + String.format(": expected=%X but computed=%X", expected, computed)
      + ", algorithm=" + algorithm.getClass().getSimpleName(), errPos);
}

@Override
public void verifyChunked(ByteBuffer data, int bytesPerCrc,
    ByteBuffer crcs, String filename, long basePos)
        throws ChecksumException {
  final Checksum algorithm = newAlgorithm();
  if (data.hasArray() && crcs.hasArray()) {
    DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
        data.array(), data.position(), data.remaining(), bytesPerCrc,
        crcs.array(), crcs.position(), filename, basePos);
  } else {
    DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
        data, bytesPerCrc, crcs, filename, basePos);
  }
}

public Encoder crc32() {
    Checksum checksum = new CRC32();
    checksum.update(data, 0, data.length);
    long checksumValue = checksum.getValue();
    data = Long.toHexString(checksumValue).getBytes();
    return unhex();
}

public SpillRecord(Path indexFileName, JobConf job, Checksum crc,
                   String expectedIndexOwner)
    throws IOException {

  final FileSystem rfs = FileSystem.getLocal(job).getRaw();
  final FSDataInputStream in =
      SecureIOUtils.openFSDataInputStream(new File(indexFileName.toUri()
          .getRawPath()), expectedIndexOwner, null);
  try {
    final long length = rfs.getFileStatus(indexFileName).getLen();
    final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
    final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;
    buf = ByteBuffer.allocate(size);
    if (crc != null) {
      crc.reset();
      CheckedInputStream chk = new CheckedInputStream(in, crc);
      IOUtils.readFully(chk, buf.array(), 0, size);

      if (chk.getChecksum().getValue() != in.readLong()) {
        throw new ChecksumException("Checksum error reading spill index: " +
                              indexFileName, -1);
      }
    } else {
      IOUtils.readFully(in, buf.array(), 0, size);
    }
    entries = buf.asLongBuffer();
  } finally {
    in.close();
  }
}

private static void assertHash32(int expected, Supplier<Checksum> supplier, String input) {
  byte[] bytes = HashTestUtils.ascii(input);
  String toString = "name";
  HashFunction func = new ChecksumHashFunction(supplier, 32, toString);
  assertEquals(expected, func.hashBytes(bytes).asInt());
  assertEquals(toString, func.toString());
}

public Encoder crc32() {
    Checksum checksum = new CRC32();
    checksum.update(data, 0, data.length);
    long checksumValue = checksum.getValue();
    data = Long.toHexString(checksumValue).getBytes();
    return unhex();
}

private static void assertChecksum(Supplier<Checksum> supplier, String input) {
  byte[] bytes = HashTestUtils.ascii(input);

  Checksum checksum = supplier.get();
  checksum.update(bytes, 0, bytes.length);
  long value = checksum.getValue();

  String toString = "name";
  HashFunction func = new ChecksumHashFunction(supplier, 32, toString);
  assertEquals(toString, func.toString());
  assertEquals(value, func.hashBytes(bytes).padToLong());
}

private static void assertHash32(int expected, Supplier<Checksum> supplier, String input) {
  byte[] bytes = HashTestUtils.ascii(input);
  String toString = "name";
  HashFunction func = new ChecksumHashFunction(supplier, 32, toString);
  assertEquals(expected, func.hashBytes(bytes).asInt());
  assertEquals(toString, func.toString());
}

private static void testWrappedByteBufferOffset(Checksum checksum, long expected) {
    byte[] unaligned_bytes_123456789 = new byte[BYTES_123456789.length + 64];
    for (int i = 0; i < unaligned_bytes_123456789.length - BYTES_123456789.length; i++) {
        checksum.reset();
        System.arraycopy(BYTES_123456789, 0, unaligned_bytes_123456789, i, BYTES_123456789.length);
        ByteBuffer bb = ByteBuffer.wrap(unaligned_bytes_123456789);
        bb.position(i);
        bb.limit(i + BYTES_123456789.length);
        checksum.update(bb);
        checkChecksumOffset(checksum, expected, i);
    }
}

ChecksumTransformer(Checksum checksum, Mode mode, int checksumLengthByte) {
    if (checksumLengthByte <= 0 || checksumLengthByte > 8)
        throw new IllegalArgumentException("checksumlength must be between 1 and 8 bytes");

    Objects.requireNonNull(checksum, "checksum instance must not be null");
    this.checksum = checksum;
    this.mode = mode;
    this.checksumLengthByte = checksumLengthByte;
}

@Test
public void checksumTest() throws Exception {
    Checksum crc32Checksum = new CRC32();
    crc32Checksum.update(example2_bytes_seven, 0, example2_bytes_seven.length);
    assertEquals(crc32Checksum.getValue(), Bytes.from(example2_bytes_seven).transform(checksumCrc32()).resize(8).toLong());
    assertEquals(Bytes.from(example2_bytes_seven, Bytes.from(crc32Checksum.getValue()).resize(4).array()), Bytes.from(example2_bytes_seven).transform(checksumAppendCrc32()));

    Checksum adlerChecksum = new Adler32();
    adlerChecksum.update(example2_bytes_seven, 0, example2_bytes_seven.length);
    assertEquals(Bytes.from(adlerChecksum.getValue()).resize(4),
            Bytes.from(example2_bytes_seven).transform(checksum(new Adler32(), ChecksumTransformer.Mode.TRANSFORM, 4)));
}

private static void checkChecksumOffset(Checksum checksum, long expected, int offset) {
    if (checksum.getValue() != expected) {
        throw new RuntimeException("Calculated CRC32C result was invalid. Array offset "
                + offset + ". Expected: " + Long.toHexString(expected) + ", Got: "
                + Long.toHexString(checksum.getValue()));
    }
}

private static void report(String s, Checksum crc1, Checksum crc2,
        Checksum crc3, Checksum crc4) {
    System.out.println(s + ", crc1 = " + crc1.getValue() +
            ", crc2 = " + crc2.getValue()+
            ", crc3 = " + crc3.getValue()+
            ", crc4 = " + crc4.getValue());
}

private static void check(Checksum crc1, Checksum crc2) throws Exception {
    if (crc1.getValue() != crc2.getValue()) {
        String s = "value 1 = " + crc1.getValue() + ", value 2 = " + crc2.getValue();
        System.err.println(s);
        throw new Exception(s);
    }
}

/**
 * Create a new {@link OutputStream} with configurable block size. Large
 * blocks require more memory at compression and decompression time but
 * should improve the compression ratio.
 *
 * @param out         the {@link OutputStream} to feed
 * @param blockSize   the maximum number of bytes to try to compress at once,
 *                    must be >= 64 and <= 32 M
 * @param compressor  the {@link LZ4Compressor} instance to use to compress
 *                    data
 * @param checksum    the {@link Checksum} instance to use to check data for
 *                    integrity.
 * @param syncFlush   true if pending data should also be flushed on {@link #flush()}
 */
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) {
  super(out);
  this.blockSize = blockSize;
  this.compressor = compressor;
  this.checksum = checksum;
  this.compressionLevel = compressionLevel(blockSize);
  this.buffer = new byte[blockSize];
  final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize);
  this.compressedBuffer = new byte[compressedBlockSize];
  this.syncFlush = syncFlush;
  o = 0;
  finished = false;
  System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH);
}

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}

/**
 * Validate a transaction's checksum
 */
private void validateChecksum(DataInputStream in,
                              Checksum checksum,
                              long txid)
    throws IOException {
  if (checksum != null) {
    int calculatedChecksum = (int)checksum.getValue();
    int readChecksum = in.readInt(); // read in checksum
    if (readChecksum != calculatedChecksum) {
      throw new ChecksumException(
          "Transaction is corrupt. Calculated checksum is " +
          calculatedChecksum + " but read checksum " + readChecksum, txid);
    }
  }
}

public static void update(Checksum checksum, ByteBuffer buffer, int offset, int length) {
    if (buffer.hasArray()) {
        checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset() + offset, length);
    } else {
        int start = buffer.position() + offset;
        for (int i = start; i < start + length; i++)
            checksum.update(buffer.get(i));
    }
}

private static void testLittleEndianWrappedByteBufferOffset(Checksum checksum, long expected) {
    byte[] unaligned_bytes_123456789 = new byte[BYTES_123456789.length + 64];
    for (int i = 0; i < unaligned_bytes_123456789.length - BYTES_123456789.length; i++) {
        checksum.reset();
        System.arraycopy(BYTES_123456789, 0, unaligned_bytes_123456789, i, BYTES_123456789.length);
        ByteBuffer bb = ByteBuffer.wrap(unaligned_bytes_123456789);
        bb.order(ByteOrder.LITTLE_ENDIAN);
        bb.position(i);
        bb.limit(i + BYTES_123456789.length);
        checksum.update(bb);
        checkChecksumOffset(checksum, expected, i);
    }
}

public static void updateLong(Checksum checksum, long input) {
    checksum.update((byte) (input >> 56));
    checksum.update((byte) (input >> 48));
    checksum.update((byte) (input >> 40));
    checksum.update((byte) (input >> 32));
    checksum.update((byte) (input >> 24));
    checksum.update((byte) (input >> 16));
    checksum.update((byte) (input >> 8));
    checksum.update((byte) input /* >> 0 */);
}

@Override
public Checksum create() {
    try {
        return (Checksum) CONSTRUCTOR.invoke();
    } catch (Throwable throwable) {
        // Should never happen
        throw new RuntimeException(throwable);
    }
}

public static long computePartialChecksum(long timestamp, int serializedKeySize, int serializedValueSize) {
    Checksum checksum = Crc32C.create();
    Checksums.updateLong(checksum, timestamp);
    Checksums.updateInt(checksum, serializedKeySize);
    Checksums.updateInt(checksum, serializedValueSize);
    return checksum.getValue();
}

private void doTestUpdateByteBuffer(byte[] bytes, ByteBuffer buffer) {
    buffer.put(bytes);
    buffer.flip();
    Checksum bufferCrc = new Crc32();
    Checksums.update(bufferCrc, buffer, buffer.remaining());
    assertEquals(Crc32.crc32(bytes), bufferCrc.getValue());
    assertEquals(0, buffer.position());
}

@Test
public void testUpdateInt() {
    final int value = 1000;
    final ByteBuffer buffer = ByteBuffer.allocate(4);
    buffer.putInt(value);

    Checksum crc1 = Crc32C.create();
    Checksum crc2 = Crc32C.create();

    Checksums.updateInt(crc1, value);
    crc2.update(buffer.array(), buffer.arrayOffset(), 4);

    assertEquals("Crc values should be the same", crc1.getValue(), crc2.getValue());
}

@Test
public void testUpdateLong() {
    final long value = Integer.MAX_VALUE + 1;
    final ByteBuffer buffer = ByteBuffer.allocate(8);
    buffer.putLong(value);

    Checksum crc1 = new Crc32();
    Checksum crc2 = new Crc32();

    Checksums.updateLong(crc1, value);
    crc2.update(buffer.array(), buffer.arrayOffset(), 8);

    assertEquals("Crc values should be the same", crc1.getValue(), crc2.getValue());
}

private void doTestUpdateByteBufferWithOffsetPosition(byte[] bytes, ByteBuffer buffer, int offset) {
    buffer.put(bytes);
    buffer.flip();
    buffer.position(offset);

    Checksum bufferCrc = Crc32C.create();
    Checksums.update(bufferCrc, buffer, buffer.remaining());
    assertEquals(Crc32C.compute(bytes, offset, buffer.remaining()), bufferCrc.getValue());
    assertEquals(offset, buffer.position());
}

/**
 * Calculates the CRC32 checksum of the specified buffer.
 * 
 * @param buffer The buffer.
 * @return The CRC32 checksum.
 */
public static int getCrcChecksum(ByteBuffer buffer) {
    Checksum crc = new CRC32();
    for (int i = 0; i < buffer.limit(); i++) {
        crc.update(buffer.get(i));
    }
    return (int) crc.getValue();
}