public void testIntegerRanges() { Configuration conf = new Configuration(); conf.set("first", "-100"); conf.set("second", "4-6,9-10,27"); conf.set("third", "34-"); Configuration.IntegerRanges range = conf.getRange("first", null); System.out.println("first = " + range); assertEquals(true, range.isIncluded(0)); assertEquals(true, range.isIncluded(1)); assertEquals(true, range.isIncluded(100)); assertEquals(false, range.isIncluded(101)); range = conf.getRange("second", null); System.out.println("second = " + range); assertEquals(false, range.isIncluded(3)); assertEquals(true, range.isIncluded(4)); assertEquals(true, range.isIncluded(6)); assertEquals(false, range.isIncluded(7)); assertEquals(false, range.isIncluded(8)); assertEquals(true, range.isIncluded(9)); assertEquals(true, range.isIncluded(10)); assertEquals(false, range.isIncluded(11)); assertEquals(false, range.isIncluded(26)); assertEquals(true, range.isIncluded(27)); assertEquals(false, range.isIncluded(28)); range = conf.getRange("third", null); System.out.println("third = " + range); assertEquals(false, range.isIncluded(33)); assertEquals(true, range.isIncluded(34)); assertEquals(true, range.isIncluded(100000000)); }
private void printTaskEvents(TaskCompletionEvent[] events, Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges, IntegerRanges reduceRanges) throws IOException, InterruptedException { for (TaskCompletionEvent event : events) { switch (filter) { case NONE: break; case SUCCEEDED: if (event.getStatus() == TaskCompletionEvent.Status.SUCCEEDED) { LOG.info(event.toString()); } break; case FAILED: if (event.getStatus() == TaskCompletionEvent.Status.FAILED) { LOG.info(event.toString()); // Displaying the task diagnostic information TaskAttemptID taskId = event.getTaskAttemptId(); String[] taskDiagnostics = getTaskDiagnostics(taskId); if (taskDiagnostics != null) { for (String diagnostics : taskDiagnostics) { System.err.println(diagnostics); } } } break; case KILLED: if (event.getStatus() == TaskCompletionEvent.Status.KILLED){ LOG.info(event.toString()); } break; case ALL: LOG.info(event.toString()); break; } } }
/** * Monitor a job and print status in real-time as progress is made and tasks * fail. * @return true if the job succeeded * @throws IOException if communication to the JobTracker fails */ public boolean monitorAndPrintJob() throws IOException, InterruptedException { String lastReport = null; Job.TaskStatusFilter filter; Configuration clientConf = cluster.getConf(); filter = Job.getTaskOutputFilter(clientConf); JobID jobId = getJobID(); LOG.info("Running job: " + jobId); int eventCounter = 0; boolean profiling = getProfileEnabled(); IntegerRanges mapRanges = getProfileTaskRange(true); IntegerRanges reduceRanges = getProfileTaskRange(false); int progMonitorPollIntervalMillis = Job.getProgressPollInterval(clientConf); while (!isComplete()) { Thread.sleep(progMonitorPollIntervalMillis); String report = (" map " + StringUtils.formatPercent(mapProgress(), 0)+ " reduce " + StringUtils.formatPercent(reduceProgress(), 0)); if (!report.equals(lastReport)) { LOG.info(report); lastReport = report; } TaskCompletionEvent[] events = getTaskCompletionEvents(eventCounter, 10); eventCounter += events.length; printTaskEvents(events, filter, profiling, mapRanges, reduceRanges); } Counters counters = getCounters(); if (counters != null) { LOG.info(counters.toString()); } LOG.info("Job " + jobId + " completed with status: " + getStatus().getState()); return isSuccessful(); }
@Override public IntegerRanges getProfileTaskRange(boolean arg0) { return null; }
/** * Monitor a job and print status in real-time as progress is made and tasks * fail. * @return true if the job succeeded * @throws IOException if communication to the JobTracker fails */ public boolean monitorAndPrintJob() throws IOException, InterruptedException { String lastReport = null; Job.TaskStatusFilter filter; Configuration clientConf = getConfiguration(); filter = Job.getTaskOutputFilter(clientConf); JobID jobId = getJobID(); LOG.info("Running job: " + jobId); int eventCounter = 0; boolean profiling = getProfileEnabled(); IntegerRanges mapRanges = getProfileTaskRange(true); IntegerRanges reduceRanges = getProfileTaskRange(false); int progMonitorPollIntervalMillis = Job.getProgressPollInterval(clientConf); /* make sure to report full progress after the job is done */ boolean reportedAfterCompletion = false; boolean reportedUberMode = false; while (!isComplete() || !reportedAfterCompletion) { if (isComplete()) { reportedAfterCompletion = true; } else { Thread.sleep(progMonitorPollIntervalMillis); } if (status.getState() == JobStatus.State.PREP) { continue; } if (!reportedUberMode) { reportedUberMode = true; LOG.info("Job " + jobId + " running in uber mode : " + isUber()); } String report = (" map " + StringUtils.formatPercent(mapProgress(), 0)+ " reduce " + StringUtils.formatPercent(reduceProgress(), 0)); if (!report.equals(lastReport)) { LOG.info(report); lastReport = report; } TaskCompletionEvent[] events = getTaskCompletionEvents(eventCounter, 10); eventCounter += events.length; printTaskEvents(events, filter, profiling, mapRanges, reduceRanges); } boolean success = isSuccessful(); if (success) { LOG.info("Job " + jobId + " completed successfully"); } else { LOG.info("Job " + jobId + " failed with state " + status.getState() + " due to: " + status.getFailureInfo()); } Counters counters = getCounters(); if (counters != null) { LOG.info(counters.toString()); } return success; }
@Override public IntegerRanges getProfileTaskRange(boolean isMap) { return mapContext.getProfileTaskRange(isMap); }
@Override public IntegerRanges getProfileTaskRange(boolean isMap) { return reduceContext.getProfileTaskRange(isMap); }
@Override public IntegerRanges getProfileTaskRange(boolean isMap) { return base.getProfileTaskRange(isMap); }