/** * Deletes the path (and its subdirectories recursively) * @throws IOException, InterruptedException */ protected void deletePath() throws IOException, InterruptedException { final Path p = getPathForCleanup(); (ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs( new PrivilegedExceptionAction<Object>() { public Object run() throws IOException { fs = (fs == null ? p.getFileSystem(conf) : fs); try { fs.delete(p, true); return null; } finally { // So that we don't leave an entry in the FileSystem cache for // every UGI that a job is submitted with. if (ugi != null) { fs.close(); } } } }); // Cancel renewal of job-delegation token if necessary if (jobIdTokenRenewalToCancel != null && conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) { DelegationTokenRenewal.removeDelegationTokenRenewalForJob( jobIdTokenRenewalToCancel); } }
/** * Deletes the path (and its subdirectories recursively) * @throws IOException, InterruptedException */ protected void deletePath() throws IOException, InterruptedException { final Path p = getPathForCleanup(); (ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs( new PrivilegedExceptionAction<Object>() { public Object run() throws IOException { FileSystem fs = p.getFileSystem(conf); try { fs.delete(p, true); return null; } finally { // So that we don't leave an entry in the FileSystem cache for // every UGI that a job is submitted with. if (ugi != null) { fs.close(); } } } }); // Cancel renewal of job-delegation token if necessary if (jobIdTokenRenewalToCancel != null && conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) { DelegationTokenRenewal.removeDelegationTokenRenewalForJob( jobIdTokenRenewalToCancel); } }
/** * The job is dead. We're now GC'ing it, getting rid of the job * from all tables. Be sure to remove all of this job's tasks * from the various tables. */ void garbageCollect() { synchronized(this) { // Cancel task tracker reservation cancelReservedSlots(); // Let the JobTracker know that a job is complete jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps()); jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces()); jobtracker.storeCompletedJob(this); jobtracker.finalizeJob(this); try { // Definitely remove the local-disk copy of the job file if (localJobFile != null) { localFs.delete(localJobFile, true); localJobFile = null; } Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID()); new CleanupQueue().addToQueue(new PathDeletionContext( jobtracker.getFileSystem(), tempDir.toUri().getPath())); } catch (IOException e) { LOG.warn("Error cleaning up "+profile.getJobID()+": "+e); } // free up the memory used by the data structures this.nonRunningMapCache = null; this.runningMapCache = null; this.nonRunningReduces = null; this.runningReduces = null; } // remove jobs delegation tokens if(conf.getBoolean(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN, true)) { DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId); } // else don't remove it.May be used by spawned tasks }
/** * The job is dead. We're now GC'ing it, getting rid of the job * from all tables. Be sure to remove all of this job's tasks * from the various tables. */ void garbageCollect() { synchronized(this) { // Cancel task tracker reservation cancelReservedSlots(); // Waiting metrics are incremented in JobInProgress.initTasks() // If a job gets an exception before that, we do not want to // incorrectly decrement. if (tasksInited) { jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps()); jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces()); this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps()); this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces()); } // Let the JobTracker know that a job is complete jobtracker.storeCompletedJob(this); jobtracker.finalizeJob(this); try { // Definitely remove the local-disk copy of the job file if (localJobFile != null) { localFs.delete(localJobFile, true); localJobFile = null; } Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID()); CleanupQueue.getInstance().addToQueue( new PathDeletionContext(tempDir, conf)); } catch (IOException e) { LOG.warn("Error cleaning up "+profile.getJobID()+": "+e); } cleanUpMetrics(); // free up the memory used by the data structures this.failedMaps.clear(); this.nonRunningMapCache = null; this.runningMapCache = null; this.nonRunningReduces = null; this.runningReduces = null; } // remove jobs delegation tokens if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) { DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId); } // else don't remove it.May be used by spawned tasks //close the user's FS try { fs.close(); } catch (IOException ie) { LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + " while closing FileSystem for " + userUGI); } }
/** * The job is dead. We're now GC'ing it, getting rid of the job * from all tables. Be sure to remove all of this job's tasks * from the various tables. */ void garbageCollect() { synchronized(this) { // Cancel task tracker reservation cancelReservedSlots(); // Let the JobTracker know that a job is complete jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps()); jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces()); this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps()); this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces()); jobtracker.storeCompletedJob(this); jobtracker.finalizeJob(this); try { // Definitely remove the local-disk copy of the job file if (localJobFile != null) { localFs.delete(localJobFile, true); localJobFile = null; } Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID()); CleanupQueue.getInstance().addToQueue( new PathDeletionContext(tempDir, conf)); } catch (IOException e) { LOG.warn("Error cleaning up "+profile.getJobID()+": "+e); } cleanUpMetrics(); // free up the memory used by the data structures this.failedMaps.clear(); this.nonRunningMapCache = null; this.runningMapCache = null; this.nonRunningReduces = null; this.runningReduces = null; } // remove jobs delegation tokens if(conf.getBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, true)) { DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jobId); } // else don't remove it.May be used by spawned tasks //close the user's FS try { fs.close(); } catch (IOException ie) { LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) + " while closing FileSystem for " + userUGI); } }
