Java 类org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal 实例源码

项目：hadoop-2.6.0-cdh5.4.3 文件：CleanupQueue.java

/**
 * Deletes the path (and its subdirectories recursively)
 * @throws IOException, InterruptedException 
 */
protected void deletePath() throws IOException, InterruptedException {
  final Path p = getPathForCleanup();
  (ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs(
      new PrivilegedExceptionAction<Object>() {
        public Object run() throws IOException {
          fs = (fs == null ?  p.getFileSystem(conf) : fs);
          try {
            fs.delete(p, true);
            return null;
          } finally {
            // So that we don't leave an entry in the FileSystem cache for
            // every UGI that a job is submitted with.
            if (ugi != null) {
              fs.close();
            }
          }
        }
      });

  // Cancel renewal of job-delegation token if necessary
  if (jobIdTokenRenewalToCancel != null && 
      conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(
        jobIdTokenRenewalToCancel);
  }
}

项目：hanoi-hadoop-2.0.0-cdh 文件：CleanupQueue.java

/**
 * Deletes the path (and its subdirectories recursively)
 * @throws IOException, InterruptedException 
 */
protected void deletePath() throws IOException, InterruptedException {
  final Path p = getPathForCleanup();
  (ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs(
      new PrivilegedExceptionAction<Object>() {
        public Object run() throws IOException {
          FileSystem fs = p.getFileSystem(conf);
          try {
            fs.delete(p, true);
            return null;
          } finally {
            // So that we don't leave an entry in the FileSystem cache for
            // every UGI that a job is submitted with.
            if (ugi != null) {
              fs.close();
            }
          }
        }
      });

  // Cancel renewal of job-delegation token if necessary
  if (jobIdTokenRenewalToCancel != null && 
      conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(
        jobIdTokenRenewalToCancel);
  }
}

项目：mapreduce-fork 文件：JobInProgress.java

/**
* The job is dead.  We're now GC'ing it, getting rid of the job
* from all tables.  Be sure to remove all of this job's tasks
* from the various tables.
*/
void garbageCollect() {
  synchronized(this) {
    // Cancel task tracker reservation
    cancelReservedSlots();


    // Let the JobTracker know that a job is complete
    jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
    jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
    jobtracker.storeCompletedJob(this);
    jobtracker.finalizeJob(this);

    try {
      // Definitely remove the local-disk copy of the job file
      if (localJobFile != null) {
        localFs.delete(localJobFile, true);
        localJobFile = null;
      }

      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      new CleanupQueue().addToQueue(new PathDeletionContext(
          jobtracker.getFileSystem(), tempDir.toUri().getPath())); 
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

    // free up the memory used by the data structures
    this.nonRunningMapCache = null;
    this.runningMapCache = null;
    this.nonRunningReduces = null;
    this.runningReduces = null;

  }
  // remove jobs delegation tokens
  if(conf.getBoolean(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId);
  } // else don't remove it.May be used by spawned tasks
}

项目：hadoop-on-lustre 文件：JobInProgress.java

/**
 * The job is dead.  We're now GC'ing it, getting rid of the job
 * from all tables.  Be sure to remove all of this job's tasks
 * from the various tables.
 */
void garbageCollect() {
  synchronized(this) {
    // Cancel task tracker reservation
    cancelReservedSlots();

    //  Waiting metrics are incremented in JobInProgress.initTasks()
    //  If a job gets an exception before that, we do not want to
    //  incorrectly decrement.
    if (tasksInited) {
      jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
      jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
      this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
      this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
    }
    // Let the JobTracker know that a job is complete
    jobtracker.storeCompletedJob(this);
    jobtracker.finalizeJob(this);

    try {
      // Definitely remove the local-disk copy of the job file
      if (localJobFile != null) {
        localFs.delete(localJobFile, true);
        localJobFile = null;
      }

      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      CleanupQueue.getInstance().addToQueue(
          new PathDeletionContext(tempDir, conf)); 
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

    cleanUpMetrics();
    // free up the memory used by the data structures
    this.failedMaps.clear();
    this.nonRunningMapCache = null;
    this.runningMapCache = null;
    this.nonRunningReduces = null;
    this.runningReduces = null;
  }

  // remove jobs delegation tokens
  if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId);
  } // else don't remove it.May be used by spawned tasks

  //close the user's FS
  try {
    fs.close();
  } catch (IOException ie) {
    LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + 
        " while closing FileSystem for " + userUGI);
  }
}

项目：mammoth 文件：JobInProgress.java

/**
 * The job is dead.  We're now GC'ing it, getting rid of the job
 * from all tables.  Be sure to remove all of this job's tasks
 * from the various tables.
 */
void garbageCollect() {
  synchronized(this) {
    // Cancel task tracker reservation
    cancelReservedSlots();

    // Let the JobTracker know that a job is complete
    jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
    jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
    this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
    this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
    jobtracker.storeCompletedJob(this);
    jobtracker.finalizeJob(this);

    try {
      // Definitely remove the local-disk copy of the job file
      if (localJobFile != null) {
        localFs.delete(localJobFile, true);
        localJobFile = null;
      }

      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      CleanupQueue.getInstance().addToQueue(
          new PathDeletionContext(tempDir, conf)); 
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

    cleanUpMetrics();
    // free up the memory used by the data structures
    this.failedMaps.clear();
    this.nonRunningMapCache = null;
    this.runningMapCache = null;
    this.nonRunningReduces = null;
    this.runningReduces = null;
  }

  // remove jobs delegation tokens
  if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId);
  } // else don't remove it.May be used by spawned tasks

  //close the user's FS
  try {
    fs.close();
  } catch (IOException ie) {
    LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + 
        " while closing FileSystem for " + userUGI);
  }
}

项目：hortonworks-extension 文件：JobInProgress.java

/**
 * The job is dead.  We're now GC'ing it, getting rid of the job
 * from all tables.  Be sure to remove all of this job's tasks
 * from the various tables.
 */
void garbageCollect() {
  synchronized(this) {
    // Cancel task tracker reservation
    cancelReservedSlots();

    //  Waiting metrics are incremented in JobInProgress.initTasks()
    //  If a job gets an exception before that, we do not want to
    //  incorrectly decrement.
    if (tasksInited) {
      jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
      jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
      this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
      this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
    }
    // Let the JobTracker know that a job is complete
    jobtracker.storeCompletedJob(this);
    jobtracker.finalizeJob(this);

    try {
      // Definitely remove the local-disk copy of the job file
      if (localJobFile != null) {
        localFs.delete(localJobFile, true);
        localJobFile = null;
      }

      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      CleanupQueue.getInstance().addToQueue(
          new PathDeletionContext(tempDir, conf)); 
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

    cleanUpMetrics();
    // free up the memory used by the data structures
    this.failedMaps.clear();
    this.nonRunningMapCache = null;
    this.runningMapCache = null;
    this.nonRunningReduces = null;
    this.runningReduces = null;
  }

  // remove jobs delegation tokens
  if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId);
  } // else don't remove it.May be used by spawned tasks

  //close the user's FS
  try {
    fs.close();
  } catch (IOException ie) {
    LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + 
        " while closing FileSystem for " + userUGI);
  }
}

项目：hortonworks-extension 文件：JobInProgress.java

/**
 * The job is dead.  We're now GC'ing it, getting rid of the job
 * from all tables.  Be sure to remove all of this job's tasks
 * from the various tables.
 */
void garbageCollect() {
  synchronized(this) {
    // Cancel task tracker reservation
    cancelReservedSlots();

    //  Waiting metrics are incremented in JobInProgress.initTasks()
    //  If a job gets an exception before that, we do not want to
    //  incorrectly decrement.
    if (tasksInited) {
      jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
      jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
      this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
      this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
    }
    // Let the JobTracker know that a job is complete
    jobtracker.storeCompletedJob(this);
    jobtracker.finalizeJob(this);

    try {
      // Definitely remove the local-disk copy of the job file
      if (localJobFile != null) {
        localFs.delete(localJobFile, true);
        localJobFile = null;
      }

      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      CleanupQueue.getInstance().addToQueue(
          new PathDeletionContext(tempDir, conf)); 
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

    cleanUpMetrics();
    // free up the memory used by the data structures
    this.failedMaps.clear();
    this.nonRunningMapCache = null;
    this.runningMapCache = null;
    this.nonRunningReduces = null;
    this.runningReduces = null;
  }

  // remove jobs delegation tokens
  if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) {
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId);
  } // else don't remove it.May be used by spawned tasks

  //close the user's FS
  try {
    fs.close();
  } catch (IOException ie) {
    LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + 
        " while closing FileSystem for " + userUGI);
  }
}