Java 类org.apache.hadoop.io.MD5Hash 实例源码

public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId,
    Storage dstStorage, boolean needDigest) throws IOException {
  String fileid = ImageServlet.getParamStringForImage(null,
      imageTxId, dstStorage);
  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(
      NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
      dstFiles.get(0).length() + " bytes.");
  return hash;
}

static MD5Hash handleUploadImageRequest(HttpServletRequest request,
    long imageTxId, Storage dstStorage, InputStream stream,
    long advertisedSize, DataTransferThrottler throttler) throws IOException {

  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash advertisedDigest = parseMD5Header(request);
  MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
      advertisedSize, advertisedDigest, fileName, stream, throttler);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
      + dstFiles.get(0).length() + " bytes.");
  return hash;
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery,
    boolean requireSameLayoutVersion) throws IOException {
  // BlockPoolId is required when the FsImageLoader loads the rolling upgrade
  // information. Make sure the ID is properly set.
  target.setBlockPoolId(this.getBlockPoolID());

  FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target);
  loader.load(curFile, requireSameLayoutVersion);

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Read the md5 checksum stored alongside the given data file.
 * @param dataFile the file containing data
 * @return the checksum stored in dataFile.md5
 */
public static MD5Hash readStoredMd5ForFile(File dataFile) throws IOException {
  final File md5File = getDigestFileForFile(dataFile);
  if (!md5File.exists()) {
    return null;
  }

  final Matcher matcher = readStoredMd5(md5File);
  String storedHash = matcher.group(1);
  File referencedFile = new File(matcher.group(2));

  // Sanity check: Make sure that the file referenced in the .md5 file at
  // least has the same name as the file we expect
  if (!referencedFile.getName().equals(dataFile.getName())) {
    throw new IOException(
        "MD5 file at " + md5File + " references file named " +
        referencedFile.getName() + " but we expected it to reference " +
        dataFile);
  }
  return new MD5Hash(storedHash);
}

/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}

public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId,
    Storage dstStorage, boolean needDigest, boolean isBootstrapStandby)
    throws IOException {
  String fileid = ImageServlet.getParamStringForImage(null,
      imageTxId, dstStorage, isBootstrapStandby);
  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(
      NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
      dstFiles.get(0).length() + " bytes.");
  return hash;
}

static MD5Hash handleUploadImageRequest(HttpServletRequest request,
    long imageTxId, Storage dstStorage, InputStream stream,
    long advertisedSize, DataTransferThrottler throttler) throws IOException {

  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash advertisedDigest = parseMD5Header(request);
  MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
      advertisedSize, advertisedDigest, fileName, stream, throttler);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
      + dstFiles.get(0).length() + " bytes.");
  return hash;
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery,
    boolean requireSameLayoutVersion) throws IOException {
  // BlockPoolId is required when the FsImageLoader loads the rolling upgrade
  // information. Make sure the ID is properly set.
  target.setBlockPoolId(this.getBlockPoolID());

  FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target);
  loader.load(curFile, requireSameLayoutVersion);

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Read the md5 checksum stored alongside the given data file.
 * @param dataFile the file containing data
 * @return the checksum stored in dataFile.md5
 */
public static MD5Hash readStoredMd5ForFile(File dataFile) throws IOException {
  final File md5File = getDigestFileForFile(dataFile);
  if (!md5File.exists()) {
    return null;
  }

  final Matcher matcher = readStoredMd5(md5File);
  String storedHash = matcher.group(1);
  File referencedFile = new File(matcher.group(2));

  // Sanity check: Make sure that the file referenced in the .md5 file at
  // least has the same name as the file we expect
  if (!referencedFile.getName().equals(dataFile.getName())) {
    throw new IOException(
        "MD5 file at " + md5File + " references file named " +
        referencedFile.getName() + " but we expected it to reference " +
        dataFile);
  }
  return new MD5Hash(storedHash);
}

/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}

public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId,
    Storage dstStorage, boolean needDigest) throws IOException {
  String fileid = ImageServlet.getParamStringForImage(null,
      imageTxId, dstStorage);
  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(
      NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
      dstFiles.get(0).length() + " bytes.");
  return hash;
}

static MD5Hash handleUploadImageRequest(HttpServletRequest request,
    long imageTxId, Storage dstStorage, InputStream stream,
    long advertisedSize, DataTransferThrottler throttler) throws IOException {

  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash advertisedDigest = parseMD5Header(request);
  MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
      advertisedSize, advertisedDigest, fileName, stream, throttler);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
      + dstFiles.get(0).length() + " bytes.");
  return hash;
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery,
    boolean requireSameLayoutVersion) throws IOException {
  // BlockPoolId is required when the FsImageLoader loads the rolling upgrade
  // information. Make sure the ID is properly set.
  target.setBlockPoolId(this.getBlockPoolID());

  FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target);
  loader.load(curFile, requireSameLayoutVersion);

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Read the md5 checksum stored alongside the given data file.
 * @param dataFile the file containing data
 * @return the checksum stored in dataFile.md5
 */
public static MD5Hash readStoredMd5ForFile(File dataFile) throws IOException {
  final File md5File = getDigestFileForFile(dataFile);
  if (!md5File.exists()) {
    return null;
  }

  final Matcher matcher = readStoredMd5(md5File);
  String storedHash = matcher.group(1);
  File referencedFile = new File(matcher.group(2));

  // Sanity check: Make sure that the file referenced in the .md5 file at
  // least has the same name as the file we expect
  if (!referencedFile.getName().equals(dataFile.getName())) {
    throw new IOException(
        "MD5 file at " + md5File + " references file named " +
        referencedFile.getName() + " but we expected it to reference " +
        dataFile);
  }
  return new MD5Hash(storedHash);
}

/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}

public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId,
    Storage dstStorage, boolean needDigest) throws IOException {
  String fileid = ImageServlet.getParamStringForImage(null,
      imageTxId, dstStorage);
  String fileName = NNStorage.getCheckpointImageFileName(imageTxId);

  List<File> dstFiles = dstStorage.getFiles(
      NameNodeDirType.IMAGE, fileName);
  if (dstFiles.isEmpty()) {
    throw new IOException("No targets in destination storage!");
  }

  MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
  LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
      dstFiles.get(0).length() + " bytes.");
  return hash;
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery,
    boolean requireSameLayoutVersion) throws IOException {
  // BlockPoolId is required when the FsImageLoader loads the rolling upgrade
  // information. Make sure the ID is properly set.
  target.setBlockPoolId(this.getBlockPoolID());

  FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target);
  loader.load(curFile, requireSameLayoutVersion);

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery,
    boolean requireSameLayoutVersion) throws IOException {
  // BlockPoolId is required when the FsImageLoader loads the rolling upgrade
  // information. Make sure the ID is properly set.
  target.setBlockPoolId(this.getBlockPoolID());

  FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target);
  loader.load(curFile, requireSameLayoutVersion);

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}

/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}

@Override
protected synchronized byte[] createPassword(TokenIdent identifier) {
  int sequenceNum;
  long now = Time.now();
  sequenceNum = incrementDelegationTokenSeqNum();
  identifier.setIssueDate(now);
  identifier.setMaxDate(now + tokenMaxLifetime);
  identifier.setMasterKeyId(currentKey.getKeyId());
  identifier.setSequenceNumber(sequenceNum);
  LOG.info("Creating password for identifier: [" + MD5Hash.digest(identifier.getBytes()) + ", " + currentKey.getKeyId() + "]");
  byte[] password = createPassword(identifier.getBytes(), currentKey.getKey());
  DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
      + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
  try {
    storeToken(identifier, tokenInfo);
  } catch (IOException ioe) {
    LOG.error("Could not store token !!", ioe);
  }
  return password;
}

/**
 * Test upload with one failed channel.
 */
private void testSingleFailure(InjectionEventI failOn) throws Exception {
  LOG.info("----- testSingleFailure for event : " + failOn);
  Random r = new Random();
  int numNodes = cluster.getNumNodes();
  TestImageUploadStreamInjectionHandler h = new TestImageUploadStreamInjectionHandler(
      numNodes);
  InjectionHandler.set(h);

  for (int i = 0; i < iterations; i++) {
    LOG.info("-- iteration: " + i);
    int failJournal = r.nextInt(numNodes);

    h.setFailure(failJournal, failOn);

    // the write should succeed
    MD5Hash digest = writeDataAndAssertContents(h, i);

    // clear hashes for next iteration
    h.clearHandler();

    // finalize the image
    assertManifest(i, digest, true);
  }
}

/** Return the object represented in the attributes. */
public static MD5MD5CRC32FileChecksum valueOf(Attributes attrs
    ) throws SAXException {
  final String bytesPerCRC = attrs.getValue("bytesPerCRC");
  final String crcPerBlock = attrs.getValue("crcPerBlock");
  final String md5 = attrs.getValue("md5");
  if (bytesPerCRC == null || crcPerBlock == null || md5 == null) {
    return null;
  }

  try {
    return new MD5MD5CRC32FileChecksum(Integer.valueOf(bytesPerCRC),
        Integer.valueOf(crcPerBlock), new MD5Hash(md5));
  } catch(Exception e) {
    throw new SAXException("Invalid attributes: bytesPerCRC=" + bytesPerCRC
        + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5, e);
  }
}

/**
 * Read the md5 checksum stored alongside the given data file.
 * @param dataFile the file containing data
 * @return the checksum stored in dataFile.md5
 */
public static MD5Hash readStoredMd5ForFile(File dataFile) throws IOException {
  final File md5File = getDigestFileForFile(dataFile);
  if (!md5File.exists()) {
    return null;
  }

  final Matcher matcher = readStoredMd5(md5File);
  String storedHash = matcher.group(1);
  File referencedFile = new File(matcher.group(2));

  // Sanity check: Make sure that the file referenced in the .md5 file at
  // least has the same name as the file we expect
  if (!referencedFile.getName().equals(dataFile.getName())) {
    throw new IOException(
        "MD5 file at " + md5File + " references file named " +
        referencedFile.getName() + " but we expected it to reference " +
        dataFile);
  }
  return new MD5Hash(storedHash);
}

/**
 * This is called by the 2NN after having downloaded an image, and by
 * the NN after having received a new image from the 2NN. It
 * renames the image from fsimage_N.ckpt to fsimage_N and also
 * saves the related .md5 file into place.
 */
synchronized void saveDigestAndRenameCheckpointImage(
    long txid, MD5Hash digest) throws IOException {
  if (!digest.equals(storage.getCheckpointImageDigest(txid))) {
    throw new IOException(
        "Checkpoint image is corrupt: expecting an MD5 checksum of" +
            digest + " but is " + storage.getCheckpointImageDigest(txid));
  }

  imageSet.saveDigestAndRenameCheckpointImage(txid, digest);

  // So long as this is the newest image available,
  // advertise it as such to other checkpointers
  // from now on
  storage.setMostRecentCheckpointTxId(txid);
}

/**
 * Load in the filesystem image from file. It's a big list of
 * filenames and blocks.
 */
private void loadFSImage(File curFile, MD5Hash expectedMd5,
    FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
  FSImageFormat.Loader loader = new FSImageFormat.Loader(
      conf, target);
  loader.load(curFile);
  target.setBlockPoolId(this.getBlockPoolID());

  // Check that the image digest we loaded matches up with what
  // we expected
  MD5Hash readImageMd5 = loader.getLoadedImageMd5();
  if (expectedMd5 != null &&
      !expectedMd5.equals(readImageMd5)) {
    throw new IOException("Image file " + curFile +
        " is corrupt with MD5 checksum of " + readImageMd5 +
        " but expecting " + expectedMd5);
  }

  long txId = loader.getLoadedImageTxId();
  LOG.info("Loaded image for txid " + txId + " from " + curFile);
  lastAppliedTxId = txId;
  storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

/**
 * Client-side Method to fetch file from a server
 * Copies the response from the URL to a list of local files.
 * @param dstStorage if an error occurs writing to one of the files,
 *                   this storage object will be notified. 
 * @Return a digest of the received file if getChecksum is true
 */
static MD5Hash getFileClient(URL infoServer,
    String queryString, List<File> localPaths,
    Storage dstStorage, boolean getChecksum) throws IOException {
  URL url = new URL(infoServer, ImageServlet.PATH_SPEC + "?" + queryString);
  LOG.info("Opening connection to " + url);
  return doGetUrl(url, localPaths, dstStorage, getChecksum);
}

void loadFSImageFile(FSNamesystem target, MetaRecoveryContext recovery,
    FSImageFile imageFile, StartupOption startupOption) throws IOException {
  LOG.debug("Planning to load image :\n" + imageFile);
  StorageDirectory sdForProperties = imageFile.sd;
  storage.readProperties(sdForProperties, startupOption);

  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) {
    // For txid-based layout, we should have a .md5 file
    // next to the image file
    boolean isRollingRollback = RollingUpgradeStartupOption.ROLLBACK
        .matches(startupOption);
    loadFSImage(imageFile.getFile(), target, recovery, isRollingRollback);
  } else if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.FSIMAGE_CHECKSUM, getLayoutVersion())) {
    // In 0.22, we have the checksum stored in the VERSION file.
    String md5 = storage.getDeprecatedProperty(
        NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY);
    if (md5 == null) {
      throw new InconsistentFSStateException(sdForProperties.getRoot(),
          "Message digest property " +
          NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY +
          " not set for storage directory " + sdForProperties.getRoot());
    }
    loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery,
        false);
  } else {
    // We don't have any record of the md5sum
    loadFSImage(imageFile.getFile(), null, target, recovery, false);
  }
}

/**
 * Load the image namespace from the given image file, verifying
 * it against the MD5 sum stored in its associated .md5 file.
 */
private void loadFSImage(File imageFile, FSNamesystem target,
    MetaRecoveryContext recovery, boolean requireSameLayoutVersion)
    throws IOException {
  MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
  if (expectedMD5 == null) {
    throw new IOException("No MD5 file found corresponding to image file "
        + imageFile);
  }
  loadFSImage(imageFile, expectedMD5, target, recovery,
      requireSameLayoutVersion);
}

/**
 * This is called by the 2NN after having downloaded an image, and by
 * the NN after having received a new image from the 2NN. It
 * renames the image from fsimage_N.ckpt to fsimage_N and also
 * saves the related .md5 file into place.
 */
public synchronized void saveDigestAndRenameCheckpointImage(NameNodeFile nnf,
    long txid, MD5Hash digest) throws IOException {
  // Write and rename MD5 file
  List<StorageDirectory> badSds = Lists.newArrayList();

  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
    File imageFile = NNStorage.getImageFile(sd, nnf, txid);
    try {
      MD5FileUtils.saveMD5File(imageFile, digest);
    } catch (IOException ioe) {
      badSds.add(sd);
    }
  }
  storage.reportErrorsOnDirectories(badSds);

  CheckpointFaultInjector.getInstance().afterMD5Rename();

  // Rename image from tmp file
  renameCheckpoint(txid, NameNodeFile.IMAGE_NEW, nnf, false);
  // So long as this is the newest image available,
  // advertise it as such to other checkpointers
  // from now on
  if (txid > storage.getMostRecentCheckpointTxId()) {
    storage.setMostRecentCheckpointInfo(txid, Time.now());
  }
}

/**
 * Set headers for content length, and, if available, md5.
 * @throws IOException 
 */
public static void setVerificationHeadersForGet(HttpServletResponse response,
    File file) throws IOException {
  response.setHeader(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    response.setHeader(TransferFsImage.MD5_HEADER, hash.toString());
  }
}

/**
 * Set headers for image length and if available, md5.
 * 
 * @throws IOException
 */
static void setVerificationHeadersForPut(HttpURLConnection connection,
    File file) throws IOException {
  connection.setRequestProperty(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    connection
        .setRequestProperty(TransferFsImage.MD5_HEADER, hash.toString());
  }
}

private MD5Hash calcPartialBlockChecksum(ExtendedBlock block,
    long requestLength, DataChecksum checksum, DataInputStream checksumIn)
    throws IOException {
  final int bytesPerCRC = checksum.getBytesPerChecksum();
  final int csize = checksum.getChecksumSize();
  final byte[] buffer = new byte[4*1024];
  MessageDigest digester = MD5Hash.getDigester();

  long remaining = requestLength / bytesPerCRC * csize;
  for (int toDigest = 0; remaining > 0; remaining -= toDigest) {
    toDigest = checksumIn.read(buffer, 0,
        (int) Math.min(remaining, buffer.length));
    if (toDigest < 0) {
      break;
    }
    digester.update(buffer, 0, toDigest);
  }

  int partialLength = (int) (requestLength % bytesPerCRC);
  if (partialLength > 0) {
    byte[] buf = new byte[partialLength];
    final InputStream blockIn = datanode.data.getBlockInputStream(block,
        requestLength - partialLength);
    try {
      // Get the CRC of the partialLength.
      IOUtils.readFully(blockIn, buf, 0, partialLength);
    } finally {
      IOUtils.closeStream(blockIn);
    }
    checksum.update(buf, 0, partialLength);
    byte[] partialCrc = new byte[csize];
    checksum.writeValue(partialCrc, 0, true);
    digester.update(partialCrc);
  }
  return new MD5Hash(digester.digest());
}

/**
 * Verify that the previously saved md5 for the given file matches
 * expectedMd5.
 * @throws IOException 
 */
public static void verifySavedMD5(File dataFile, MD5Hash expectedMD5)
    throws IOException {
  MD5Hash storedHash = readStoredMd5ForFile(dataFile);
  // Check the hash itself
  if (!expectedMD5.equals(storedHash)) {
    throw new IOException(
        "File " + dataFile + " did not match stored MD5 checksum " +
        " (stored: " + storedHash + ", computed: " + expectedMD5);
  }
}

/**
 * Read dataFile and compute its MD5 checksum.
 */
public static MD5Hash computeMd5ForFile(File dataFile) throws IOException {
  InputStream in = new FileInputStream(dataFile);
  try {
    MessageDigest digester = MD5Hash.getDigester();
    DigestInputStream dis = new DigestInputStream(in, digester);
    IOUtils.copyBytes(dis, new IOUtils.NullOutputStream(), 128*1024);

    return new MD5Hash(digester.digest());
  } finally {
    IOUtils.closeStream(in);
  }
}

/**
 * Run `hdfs dfsadmin -fetchImage ...' and verify that the downloaded image is
 * correct.
 */
private static void runFetchImage(DFSAdmin dfsAdmin, MiniDFSCluster cluster)
    throws Exception {
  int retVal = dfsAdmin.run(new String[]{"-fetchImage",
      FETCHED_IMAGE_FILE.getPath() });

  assertEquals(0, retVal);

  File highestImageOnNn = getHighestFsImageOnCluster(cluster);
  MD5Hash expected = MD5FileUtils.computeMd5ForFile(highestImageOnNn);
  MD5Hash actual = MD5FileUtils.computeMd5ForFile(
      new File(FETCHED_IMAGE_FILE, highestImageOnNn.getName()));

  assertEquals(expected, actual);
}

/**
 * Test when .md5 file exists but incorrect checksum
 */
@Test
public void testVerifyMD5FileBadDigest() throws Exception {
  MD5FileUtils.saveMD5File(TEST_FILE, MD5Hash.digest(new byte[0]));
  try {
    MD5FileUtils.verifySavedMD5(TEST_FILE, TEST_MD5);
    fail("Did not throw");
  } catch (IOException ioe) {
    // Expected
  }
}

protected static String getTraceSignature(String input) throws IOException {
  Path inputPath = new Path(input);
  FileSystem fs = inputPath.getFileSystem(new Configuration());
  FileStatus status = fs.getFileStatus(inputPath);
  Path qPath = fs.makeQualified(status.getPath());
  String traceID = status.getModificationTime() + qPath.toString()
                   + status.getOwner() + status.getLen();
  return MD5Hash.digest(traceID).toString();
}

/**
 * Client-side Method to fetch file from a server
 * Copies the response from the URL to a list of local files.
 * @param dstStorage if an error occurs writing to one of the files,
 *                   this storage object will be notified. 
 * @Return a digest of the received file if getChecksum is true
 */
static MD5Hash getFileClient(URL infoServer,
    String queryString, List<File> localPaths,
    Storage dstStorage, boolean getChecksum) throws IOException {
  URL url = new URL(infoServer, ImageServlet.PATH_SPEC + "?" + queryString);
  LOG.info("Opening connection to " + url);
  return doGetUrl(url, localPaths, dstStorage, getChecksum);
}

public static MD5Hash doGetUrl(URL url, List<File> localPaths,
    Storage dstStorage, boolean getChecksum) throws IOException {
  HttpURLConnection connection;
  try {
    connection = (HttpURLConnection)
      connectionFactory.openConnection(url, isSpnegoEnabled);
  } catch (AuthenticationException e) {
    throw new IOException(e);
  }

  setTimeout(connection);

  if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
    throw new HttpGetFailedException(
        "Image transfer servlet at " + url +
        " failed with status code " + connection.getResponseCode() +
        "\nResponse message:\n" + connection.getResponseMessage(),
        connection);
  }

  long advertisedSize;
  String contentLength = connection.getHeaderField(CONTENT_LENGTH);
  if (contentLength != null) {
    advertisedSize = Long.parseLong(contentLength);
  } else {
    throw new IOException(CONTENT_LENGTH + " header is not provided " +
                          "by the namenode when trying to fetch " + url);
  }
  MD5Hash advertisedDigest = parseMD5Header(connection);
  String fsImageName = connection
      .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER);
  InputStream stream = connection.getInputStream();

  return receiveFile(url.toExternalForm(), localPaths, dstStorage,
      getChecksum, advertisedSize, advertisedDigest, fsImageName, stream,
      null);
}