@Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException { addHistoryToken(ts); // Construct necessary information to start the MR AM ApplicationSubmissionContext appContext = createApplicationSubmissionContext(conf, jobSubmitDir, ts); // Submit to ResourceManager try { ApplicationId applicationId = resMgrDelegate.submitApplication(appContext); ApplicationReport appMaster = resMgrDelegate.getApplicationReport(applicationId); String diagnostics = (appMaster == null ? "application report is null" : appMaster.getDiagnostics()); if (appMaster == null || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) { throw new IOException("Failed to run job : " + diagnostics); } return clientCache.getClient(jobId).getJobStatus(jobId); } catch (YarnException e) { throw new IOException(e); } }
public JobStatus getJobStatus(JobID oldJobID) throws IOException { org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobID); GetJobReportRequest request = recordFactory.newRecordInstance(GetJobReportRequest.class); request.setJobId(jobId); JobReport report = ((GetJobReportResponse) invoke("getJobReport", GetJobReportRequest.class, request)).getJobReport(); JobStatus jobStatus = null; if (report != null) { if (StringUtils.isEmpty(report.getJobFile())) { String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID); report.setJobFile(jobFile); } String historyTrackingUrl = report.getTrackingUrl(); String url = StringUtils.isNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl; jobStatus = TypeConverter.fromYarn(report, url); } return jobStatus; }
@Test public void testRetriesOnConnectionFailure() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow( new RuntimeException("1")).thenThrow(new RuntimeException("2")) .thenReturn(getJobReportResponse()); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())) .thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate( historyServerProxy, rm); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); verify(historyServerProxy, times(3)).getJobReport( any(GetJobReportRequest.class)); }
@Test public void testHistoryServerNotConfigured() throws Exception { //RM doesn't have app report and job History Server is not configured ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate( null, getRMDelegate()); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertEquals("N/A", jobStatus.getUsername()); Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState()); //RM has app report and job History Server is not configured ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); ApplicationReport applicationReport = getFinishedApplicationReport(); when(rm.getApplicationReport(jobId.getAppId())).thenReturn( applicationReport); clientServiceDelegate = getClientServiceDelegate(null, rm); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername()); Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState()); }
@Test public void testJobReportFromHistoryServer() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn( getJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())) .thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate( historyServerProxy, rm); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl()); Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f); Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f); }
@Test public void tesAllJobs() throws Exception { final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class); GetApplicationsResponse allApplicationsResponse = Records .newRecord(GetApplicationsResponse.class); List<ApplicationReport> applications = new ArrayList<ApplicationReport>(); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.FAILED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.KILLED)); applications.add(getApplicationReport(YarnApplicationState.FAILED, FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( applicationsManager.getApplications(Mockito .any(GetApplicationsRequest.class))).thenReturn( allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override protected void serviceStart() throws Exception { Assert.assertTrue(this.client instanceof YarnClientImpl); ((YarnClientImpl) this.client).setRMClient(applicationsManager); } }; JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); Assert.assertEquals(State.FAILED, allJobs[0].getState()); Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState()); Assert.assertEquals(State.KILLED, allJobs[2].getState()); Assert.assertEquals(State.FAILED, allJobs[3].getState()); }
/** * To ensure nothing broken after we removed normalization * from the MRAM side * @throws Exception */ @Test public void testJobWithNonNormalizedCapabilities() throws Exception { if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return; } JobConf jobConf = new JobConf(mrCluster.getConfig()); jobConf.setInt("mapreduce.map.memory.mb", 700); jobConf.setInt("mapred.reduce.memory.mb", 1500); SleepJob sleepJob = new SleepJob(); sleepJob.setConf(jobConf); Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1); job.setJarByClass(SleepJob.class); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.submit(); boolean completed = job.waitForCompletion(true); Assert.assertTrue("Job should be completed", completed); Assert.assertEquals("Job should be finished successfully", JobStatus.State.SUCCEEDED, job.getJobState()); }
/** * Method used to display information pertaining to a Single JobQueue * registered with the {@link QueueManager}. Display of the Jobs is determine * by the boolean * * @throws IOException, InterruptedException */ private void displayQueueInfo(String queue, boolean showJobs) throws IOException, InterruptedException { JobQueueInfo jobQueueInfo = jc.getQueueInfo(queue); if (jobQueueInfo == null) { System.out.println("Queue \"" + queue + "\" does not exist."); return; } printJobQueueInfo(jobQueueInfo, new PrintWriter(new OutputStreamWriter( System.out, Charsets.UTF_8))); if (showJobs && (jobQueueInfo.getChildren() == null || jobQueueInfo.getChildren().size() == 0)) { JobStatus[] jobs = jobQueueInfo.getJobStatuses(); if (jobs == null) jobs = new JobStatus[0]; jc.displayJobList(jobs); } }
@Private public void displayJobList(JobStatus[] jobs, PrintWriter writer) { writer.println("Total jobs:" + jobs.length); writer.printf(headerPattern, "JobId", "State", "StartTime", "UserName", "Queue", "Priority", "UsedContainers", "RsvdContainers", "UsedMem", "RsvdMem", "NeededMem", "AM info"); for (JobStatus job : jobs) { int numUsedSlots = job.getNumUsedSlots(); int numReservedSlots = job.getNumReservedSlots(); int usedMem = job.getUsedMem(); int rsvdMem = job.getReservedMem(); int neededMem = job.getNeededMem(); writer.printf(dataPattern, job.getJobID().toString(), job.getState(), job.getStartTime(), job.getUsername(), job.getQueue(), job.getPriority().name(), numUsedSlots < 0 ? UNAVAILABLE : numUsedSlots, numReservedSlots < 0 ? UNAVAILABLE : numReservedSlots, usedMem < 0 ? UNAVAILABLE : String.format(memPattern, usedMem), rsvdMem < 0 ? UNAVAILABLE : String.format(memPattern, rsvdMem), neededMem < 0 ? UNAVAILABLE : String.format(memPattern, neededMem), job.getSchedulingInfo()); } writer.flush(); }
/** * Test {@link Statistics.JobStats}. */ @Test @SuppressWarnings("deprecation") public void testJobStats() throws Exception { Job job = new Job() {}; JobStats stats = new JobStats(1, 2, job); assertEquals("Incorrect num-maps", 1, stats.getNoOfMaps()); assertEquals("Incorrect num-reds", 2, stats.getNoOfReds()); assertTrue("Incorrect job", job == stats.getJob()); assertNull("Unexpected job status", stats.getJobStatus()); // add a new status JobStatus status = new JobStatus(); stats.updateJobStatus(status); assertNotNull("Missing job status", stats.getJobStatus()); assertTrue("Incorrect job status", status == stats.getJobStatus()); }
private static void testJobStats(JobStats stats, int numMaps, int numReds, JobStatus jStatus, Job job) { assertEquals("Incorrect num map tasks", numMaps, stats.getNoOfMaps()); assertEquals("Incorrect num reduce tasks", numReds, stats.getNoOfReds()); if (job != null) { assertNotNull("Missing job", job); } // check running job assertTrue("Incorrect job", job == stats.getJob()); if (jStatus != null) { assertNotNull("Missing job status", jStatus); } // check job stats assertTrue("Incorrect job status", jStatus == stats.getJobStatus()); }
@Private public void displayJobList(JobStatus[] jobs, PrintWriter writer) { writer.println("Total jobs:" + jobs.length); writer.printf(headerPattern, "JobId", "JobName", "State", "StartTime", "UserName", "Queue", "Priority", "UsedContainers", "RsvdContainers", "UsedMem", "RsvdMem", "NeededMem", "AM info"); for (JobStatus job : jobs) { int numUsedSlots = job.getNumUsedSlots(); int numReservedSlots = job.getNumReservedSlots(); int usedMem = job.getUsedMem(); int rsvdMem = job.getReservedMem(); int neededMem = job.getNeededMem(); int jobNameLength = job.getJobName().length(); writer.printf(dataPattern, job.getJobID().toString(), job.getJobName().substring(0, jobNameLength > 20 ? 20 : jobNameLength), job.getState(), job.getStartTime(), job.getUsername(), job.getQueue(), job.getPriority().name(), numUsedSlots < 0 ? UNAVAILABLE : numUsedSlots, numReservedSlots < 0 ? UNAVAILABLE : numReservedSlots, usedMem < 0 ? UNAVAILABLE : String.format(memPattern, usedMem), rsvdMem < 0 ? UNAVAILABLE : String.format(memPattern, rsvdMem), neededMem < 0 ? UNAVAILABLE : String.format(memPattern, neededMem), job.getSchedulingInfo()); } writer.flush(); }
public static int getReturnCode(JobStatus.State state) { switch (state) { case RUNNING: return -1; case SUCCEEDED: return 0; case FAILED: return 1; case PREP: return 2; case KILLED: return 3; default: return 4; } }
/** {@inheritDoc} */ @Override public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException { try { Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1); HadoopJobStatus status; if (delay >= 0) status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay); else status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId()); if (status == null) throw new IOException("Job tracker doesn't have any information about the job: " + jobId); return processStatus(status); } catch (GridClientException e) { throw new IOException("Failed to get job status: " + jobId, e); } }
public void readFields(DataInput in) throws IOException { jobFile = StringInterner.weakIntern(Text.readString(in)); taskId = TaskAttemptID.read(in); partition = in.readInt(); numSlotsRequired = in.readInt(); taskStatus.readFields(in); skipRanges.readFields(in); currentRecIndexIterator = skipRanges.skipRangeIterator(); currentRecStartIndex = currentRecIndexIterator.next(); skipping = in.readBoolean(); jobCleanup = in.readBoolean(); if (jobCleanup) { jobRunStateForCleanup = WritableUtils.readEnum(in, JobStatus.State.class); } jobSetup = in.readBoolean(); writeSkipRecs = in.readBoolean(); taskCleanup = in.readBoolean(); if (taskCleanup) { setPhase(TaskStatus.Phase.CLEANUP); } user = StringInterner.weakIntern(Text.readString(in)); extraData.readFields(in); }
protected void runJobCleanupTask(TaskUmbilicalProtocol umbilical, TaskReporter reporter ) throws IOException, InterruptedException { // set phase for this task setPhase(TaskStatus.Phase.CLEANUP); getProgress().setStatus("cleanup"); statusUpdate(umbilical); // do the cleanup LOG.info("Cleaning up job"); if (jobRunStateForCleanup == JobStatus.State.FAILED || jobRunStateForCleanup == JobStatus.State.KILLED) { LOG.info("Aborting job with runstate : " + jobRunStateForCleanup); committer.abortJob(jobContext, jobRunStateForCleanup); } else if (jobRunStateForCleanup == JobStatus.State.SUCCEEDED){ LOG.info("Committing job"); committer.commitJob(jobContext); } else { throw new IOException("Invalid state of the job for cleanup. State found " + jobRunStateForCleanup + " expecting " + JobStatus.State.SUCCEEDED + ", " + JobStatus.State.FAILED + " or " + JobStatus.State.KILLED); } done(umbilical, reporter); }
public void readFields(DataInput in) throws IOException { jobFile = Text.readString(in); taskId = TaskAttemptID.read(in); partition = in.readInt(); numSlotsRequired = in.readInt(); taskStatus.readFields(in); this.mapOutputFile.setJobId(taskId.getJobID()); skipRanges.readFields(in); currentRecIndexIterator = skipRanges.skipRangeIterator(); currentRecStartIndex = currentRecIndexIterator.next(); skipping = in.readBoolean(); jobCleanup = in.readBoolean(); if (jobCleanup) { jobRunStateForCleanup = WritableUtils.readEnum(in, JobStatus.State.class); } jobSetup = in.readBoolean(); username = Text.readString(in); writeSkipRecs = in.readBoolean(); taskCleanup = in.readBoolean(); if (taskCleanup) { setPhase(TaskStatus.Phase.CLEANUP); } extraData.readFields(in); }
