Java 类org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException 实例源码

private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
    throws QuotaByStorageTypeExceededException {
  if (!isQuotaByStorageTypeSet()) {
    return;
  }
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (!isQuotaByStorageTypeSet(t)) {
      continue;
    }
    if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
        typeDelta.get(t))) {
      throw new QuotaByStorageTypeExceededException(
        quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
    }
  }
}

/**
 * Creates a symbolic link.
 *
 * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean)
 */
public void createSymlink(String target, String link, boolean createParent)
    throws IOException {
  checkOpen();
  try (TraceScope ignored = newPathTraceScope("createSymlink", target)) {
    final FsPermission dirPerm = applyUMask(null);
    namenode.createSymlink(target, link, dirPerm, createParent);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileAlreadyExistsException.class,
        FileNotFoundException.class,
        ParentNotDirectoryException.class,
        NSQuotaExceededException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Set replication for an existing file.
 * @param src file name
 * @param replication replication to set the file to
 *
 * @see ClientProtocol#setReplication(String, short)
 */
public boolean setReplication(String src, short replication)
    throws IOException {
  checkOpen();
  try (TraceScope ignored = newPathTraceScope("setReplication", src)) {
    return namenode.setReplication(src, replication);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileNotFoundException.class,
        SafeModeException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  try (TraceScope ignored = newSrcDstTraceScope("rename2", src, dst)) {
    namenode.rename2(src, dst, options);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        FileAlreadyExistsException.class,
        FileNotFoundException.class,
        ParentNotDirectoryException.class,
        SafeModeException.class,
        NSQuotaExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Same {{@link #mkdirs(String, FsPermission, boolean)} except
 * that the permissions has already been masked against umask.
 */
public boolean primitiveMkdir(String src, FsPermission absPermission,
    boolean createParent) throws IOException {
  checkOpen();
  if (absPermission == null) {
    absPermission = applyUMask(null);
  }

  LOG.debug("{}: masked={}", src, absPermission);
  try (TraceScope ignored = tracer.newScope("mkdir")) {
    return namenode.mkdirs(src, absPermission, createParent);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        InvalidPathException.class,
        FileAlreadyExistsException.class,
        FileNotFoundException.class,
        ParentNotDirectoryException.class,
        SafeModeException.class,
        NSQuotaExceededException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Sets or resets quotas by storage type for a directory.
 * @see ClientProtocol#setQuota(String, long, long, StorageType)
 */
void setQuotaByStorageType(String src, StorageType type, long quota)
    throws IOException {
  checkOpen();
  if (quota <= 0 && quota != HdfsConstants.QUOTA_DONT_SET &&
      quota != HdfsConstants.QUOTA_RESET) {
    throw new IllegalArgumentException("Invalid values for quota :" +
        quota);
  }
  if (type == null) {
    throw new IllegalArgumentException("Invalid storage type(null)");
  }
  if (!type.supportTypeQuota()) {
    throw new IllegalArgumentException(
        "Don't support Quota for storage type : " + type.toString());
  }
  try (TraceScope ignored = newPathTraceScope("setQuotaByStorageType", src)) {
    namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, quota, type);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileNotFoundException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
    throws QuotaByStorageTypeExceededException {
  if (!isQuotaByStorageTypeSet()) {
    return;
  }
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (!isQuotaByStorageTypeSet(t)) {
      continue;
    }
    if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
        typeDelta.get(t))) {
      throw new QuotaByStorageTypeExceededException(
        quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
    }
  }
}

private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
    throws QuotaByStorageTypeExceededException {
  if (!isQuotaByStorageTypeSet()) {
    return;
  }
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (!isQuotaByStorageTypeSet(t)) {
      continue;
    }
    if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
        typeDelta.get(t))) {
      throw new QuotaByStorageTypeExceededException(
        quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
    }
  }
}

/**
 * Sets or resets quotas by storage type for a directory.
 * @see ClientProtocol#setQuota(String, long, long, StorageType)
 */
void setQuotaByStorageType(String src, StorageType type, long quota)
    throws IOException {
  if (quota <= 0 && quota != HdfsConstants.QUOTA_DONT_SET &&
      quota != HdfsConstants.QUOTA_RESET) {
    throw new IllegalArgumentException("Invalid values for quota :" +
      quota);
  }
  if (type == null) {
    throw new IllegalArgumentException("Invalid storage type(null)");
  }
  if (!type.supportTypeQuota()) {
    throw new IllegalArgumentException("Don't support Quota for storage type : "
      + type.toString());
  }
  TraceScope scope = getPathTraceScope("setQuotaByStorageType", src);
  try {
    namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, quota, type);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
      FileNotFoundException.class,
      QuotaByStorageTypeExceededException.class,
      UnresolvedPathException.class,
      SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}

/** Method to get stream returned by append call */
private DFSOutputStream callAppend(String src, EnumSet<CreateFlag> flag,
    Progressable progress, String[] favoredNodes) throws IOException {
  CreateFlag.validateForAppend(flag);
  try {
    LastBlockWithStatus blkWithStatus = namenode.append(src, clientName,
        new EnumSetWritable<>(flag, CreateFlag.class));
    HdfsFileStatus status = blkWithStatus.getFileStatus();
    if (status == null) {
      DFSClient.LOG.debug("NameNode is on an older version, request file " +
          "info with additional RPC call for file: " + src);
      status = getFileInfo(src);
    }
    return DFSOutputStream.newStreamForAppend(this, src, flag, progress,
        blkWithStatus.getLastBlock(), status,
        dfsClientConf.createChecksum(null), favoredNodes);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileNotFoundException.class,
        SafeModeException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnsupportedOperationException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Rename file or directory.
 * @see ClientProtocol#rename(String, String)
 * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
 */
@Deprecated
public boolean rename(String src, String dst) throws IOException {
  checkOpen();
  try (TraceScope ignored = newSrcDstTraceScope("rename", src, dst)) {
    return namenode.rename(src, dst);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        NSQuotaExceededException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Sets or resets quotas for a directory.
 * @see ClientProtocol#setQuota(String, long, long, StorageType)
 */
void setQuota(String src, long namespaceQuota, long storagespaceQuota)
    throws IOException {
  checkOpen();
  // sanity check
  if ((namespaceQuota <= 0 &&
        namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
        namespaceQuota != HdfsConstants.QUOTA_RESET) ||
      (storagespaceQuota <= 0 &&
          storagespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
          storagespaceQuota != HdfsConstants.QUOTA_RESET)) {
    throw new IllegalArgumentException("Invalid values for quota : " +
        namespaceQuota + " and " +
        storagespaceQuota);

  }
  try (TraceScope ignored = newPathTraceScope("setQuota", src)) {
    // Pass null as storage type for traditional namespace/storagespace quota.
    namenode.setQuota(src, namespaceQuota, storagespaceQuota, null);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileNotFoundException.class,
        NSQuotaExceededException.class,
        DSQuotaExceededException.class,
        QuotaByStorageTypeExceededException.class,
        UnresolvedPathException.class,
        SnapshotAccessControlException.class);
  }
}

/**
 * Test append over a specific type of storage quota does not mark file as
 * UC or create a lease
 */
@Test (timeout=60000)
public void testAppendOverTypeQuota() throws Exception {
  final Path dir = new Path("/TestAppendOverTypeQuota");
  final Path file = new Path(dir, "file");

  // create partial block file
  dfs.mkdirs(dir);
  // set the storage policy on dir
  dfs.setStoragePolicy(dir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
  DFSTestUtil.createFile(dfs, file, BLOCKSIZE/2, REPLICATION, seed);

  // set quota of SSD to 1L
  dfs.setQuotaByStorageType(dir, StorageType.SSD, 1L);
  final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString())
      .asDirectory();
  final long spaceUsed = dirNode.getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getStorageSpace();
  try {
    DFSTestUtil.appendFile(dfs, file, BLOCKSIZE);
    Assert.fail("append didn't fail");
  } catch (QuotaByStorageTypeExceededException e) {
    //ignore
  }

  // check that the file exists, isn't UC, and has no dangling lease
  LeaseManager lm = cluster.getNamesystem().getLeaseManager();
  INodeFile inode = fsdir.getINode(file.toString()).asFile();
  Assert.assertNotNull(inode);
  Assert.assertFalse("should not be UC", inode.isUnderConstruction());
  Assert.assertNull("should not have a lease", lm.getLease(inode));
  // make sure the quota usage is unchanged
  final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getStorageSpace();
  assertEquals(spaceUsed, newSpaceUsed);
  // make sure edits aren't corrupted
  dfs.recoverLease(file);
  cluster.restartNameNodes();
}

/**
 * Sets or resets quotas by storage type for a directory.
 * @see ClientProtocol#setQuota(String, long, long, StorageType)
 */
void setQuotaByStorageType(String src, StorageType type, long quota)
    throws IOException {
  if (quota <= 0 && quota != HdfsConstants.QUOTA_DONT_SET &&
      quota != HdfsConstants.QUOTA_RESET) {
    throw new IllegalArgumentException("Invalid values for quota :" +
      quota);
  }
  if (type == null) {
    throw new IllegalArgumentException("Invalid storage type(null)");
  }
  if (!type.supportTypeQuota()) {
    throw new IllegalArgumentException("Don't support Quota for storage type : "
      + type.toString());
  }
  TraceScope scope = getPathTraceScope("setQuotaByStorageType", src);
  try {
    namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, quota, type);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
      FileNotFoundException.class,
      QuotaByStorageTypeExceededException.class,
      UnresolvedPathException.class,
      SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}

static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
    FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
    short replication, long blockSize, Progressable progress,
    DataChecksum checksum, String[] favoredNodes) throws IOException {
  try (TraceScope ignored =
           dfsClient.newPathTraceScope("newStreamForCreate", src)) {
    HdfsFileStatus stat = null;

    // Retry the create if we get a RetryStartFileException up to a maximum
    // number of times
    boolean shouldRetry = true;
    int retryCount = CREATE_RETRY_COUNT;
    while (shouldRetry) {
      shouldRetry = false;
      try {
        stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
            new EnumSetWritable<>(flag), createParent, replication,
            blockSize, SUPPORTED_CRYPTO_VERSIONS);
        break;
      } catch (RemoteException re) {
        IOException e = re.unwrapRemoteException(
            AccessControlException.class,
            DSQuotaExceededException.class,
            QuotaByStorageTypeExceededException.class,
            FileAlreadyExistsException.class,
            FileNotFoundException.class,
            ParentNotDirectoryException.class,
            NSQuotaExceededException.class,
            RetryStartFileException.class,
            SafeModeException.class,
            UnresolvedPathException.class,
            SnapshotAccessControlException.class,
            UnknownCryptoProtocolVersionException.class);
        if (e instanceof RetryStartFileException) {
          if (retryCount > 0) {
            shouldRetry = true;
            retryCount--;
          } else {
            throw new IOException("Too many retries because of encryption" +
                " zone operations", e);
          }
        } else {
          throw e;
        }
      }
    }
    Preconditions.checkNotNull(stat, "HdfsFileStatus should not be null!");
    final DFSOutputStream out;
    if(stat.getErasureCodingPolicy() != null) {
      out = new DFSStripedOutputStream(dfsClient, src, stat,
          flag, progress, checksum, favoredNodes);
    } else {
      out = new DFSOutputStream(dfsClient, src, stat,
          flag, progress, checksum, favoredNodes, true);
    }
    out.start();
    return out;
  }
}

static LocatedBlock addBlock(DatanodeInfo[] excludedNodes,
    DFSClient dfsClient, String src, ExtendedBlock prevBlock, long fileId,
    String[] favoredNodes) throws IOException {
  final DfsClientConf conf = dfsClient.getConf();
  int retries = conf.getNumBlockWriteLocateFollowingRetry();
  long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
  long localstart = Time.monotonicNow();
  while (true) {
    try {
      return dfsClient.namenode.addBlock(src, dfsClient.clientName, prevBlock,
          excludedNodes, fileId, favoredNodes);
    } catch (RemoteException e) {
      IOException ue = e.unwrapRemoteException(FileNotFoundException.class,
          AccessControlException.class,
          NSQuotaExceededException.class,
          DSQuotaExceededException.class,
          QuotaByStorageTypeExceededException.class,
          UnresolvedPathException.class);
      if (ue != e) {
        throw ue; // no need to retry these exceptions
      }
      if (NotReplicatedYetException.class.getName()
          .equals(e.getClassName())) {
        if (retries == 0) {
          throw e;
        } else {
          --retries;
          LOG.info("Exception while adding a block", e);
          long elapsed = Time.monotonicNow() - localstart;
          if (elapsed > 5000) {
            LOG.info("Waiting for replication for " + (elapsed / 1000)
                + " seconds");
          }
          try {
            LOG.warn("NotReplicatedYetException sleeping " + src
                + " retries left " + retries);
            Thread.sleep(sleeptime);
            sleeptime *= 2;
          } catch (InterruptedException ie) {
            LOG.warn("Caught exception", ie);
          }
        }
      } else {
        throw e;
      }
    }
  }
}

/**
 * Test quota by storage type.
 */
@Test
public void testQuotaByStorageType() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  // set a smaller block size so that we can test with smaller
  // diskspace quotas
  conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512");
  // Make it relinquish locks. When run serially, the result should
  // be identical.
  conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
  final MiniDFSCluster cluster =
      new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  final FileSystem fs = cluster.getFileSystem();
  assertTrue("Not a HDFS: " + fs.getUri(),
      fs instanceof DistributedFileSystem);
  final DistributedFileSystem dfs = (DistributedFileSystem) fs;

  try {
    int fileLen = 1024;
    short replication = 3;
    int fileSpace = fileLen * replication;

    final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
    assertTrue(dfs.mkdirs(quotaDir20));
    dfs.setQuota(quotaDir20, HdfsConstants.QUOTA_DONT_SET, 6 * fileSpace);

    // Verify DirectoryWithQuotaFeature's storage type usage
    // is updated properly after deletion.
    // File creation followed by deletion shouldn't change storage type
    // usage regardless whether storage policy is set.
    Path file = new Path(quotaDir20, "fileDir/file1");
    DFSTestUtil.createFile(dfs, file, fileLen * 3, replication, 0);
    dfs.delete(file, false);
    dfs.setStoragePolicy(quotaDir20, HdfsConstants.HOT_STORAGE_POLICY_NAME);
    dfs.setQuotaByStorageType(quotaDir20, StorageType.DEFAULT,
        2 * fileSpace);
    boolean hasException = false;
    try {
      DFSTestUtil.createFile(dfs, file, fileLen * 3, replication, 0);
    } catch (QuotaByStorageTypeExceededException e) {
      hasException = true;
    }
    assertTrue(hasException);
    dfs.delete(file, false);
    dfs.setQuotaByStorageType(quotaDir20, StorageType.DEFAULT,
        6 * fileSpace);
  } finally {
    cluster.shutdown();
  }
}