/** * Test that verifies that total memory values are calculated and reported * correctly. * * @throws Exception */ public void testResourceValuesOnLinux() throws Exception { if (!System.getProperty("os.name").startsWith("Linux")) { return; } JobConf conf = new JobConf(); LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin(); // In this case, we only check these four fields because they are static conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize()); conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize()); conf.setLong("numProcessors", plugin.getNumProcessors()); try { setUpCluster(conf); runSleepJob(miniMRCluster.createJobConf()); verifyTestResults(true); } finally { tearDownCluster(); } }
/** * Test that verifies that total memory values are calculated and reported * correctly. * * @throws Exception */ @Test(timeout=60000) public void testResourceValuesOnLinux() throws Exception { if (!System.getProperty("os.name").startsWith("Linux")) { return; } JobConf conf = new JobConf(); LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin(); // In this case, we only check these three fields because they are static conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize()); conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize()); conf.setLong("numProcessors", plugin.getNumProcessors()); try { setUpCluster(conf); runSleepJob(miniMRCluster.createJobConf()); verifyTestResults(true); } finally { tearDownCluster(); } }
/** * Test that verifies that total memory values are calculated and reported * correctly. * * @throws Exception */ public void testResourceValuesOnLinux() throws Exception { if (!System.getProperty("os.name").startsWith("Linux")) { return; } JobConf conf = new JobConf(); LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin(); // In this case, we only check these three fields because they are static conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize()); conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize()); conf.setLong("numProcessors", plugin.getNumProcessors()); try { setUpCluster(conf); runSleepJob(miniMRCluster.createJobConf()); verifyTestResults(true); } finally { tearDownCluster(); } }
/** * Test for verifying that tasks causing cumulative usage of physical memory * to go beyond TT's limit get killed. * * @throws Exception */ public void testTasksCumulativelyExceedingTTPhysicalLimits() throws Exception { // Run the test only if memory management is enabled if (!isProcfsBasedTreeAvailable()) { return; } // Start cluster with proper configuration. JobConf fConf = new JobConf(); // very small value, so that no task escapes to successful completion. fConf.set(TaskMemoryManagerThread.TT_MEMORY_MANAGER_MONITORING_INTERVAL, String.valueOf(300)); // reserve all memory on TT so that the job will exceed memory limits LinuxResourceCalculatorPlugin memoryCalculatorPlugin = new LinuxResourceCalculatorPlugin(); long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize(); long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1; fConf.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024 * 1024L); fConf.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024 * 1024L); fConf.setLong(TaskMemoryManagerThread.TT_RESERVED_PHYSICAL_MEMORY_MB, reservedPhysicalMemory); long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory - reservedPhysicalMemory * 1024 * 1024L; Pattern physicalMemoryOverLimitPattern = Pattern.compile( "Killing.*" + maxRssMemoryAllowedForAllTasks); TaskMemoryManagerThread.disableUpdateReservedPhysicalMemory(); startCluster(fConf); Matcher mat = null; // Set up job. JobConf conf = new JobConf(miniMRCluster.createJobConf()); JobClient jClient = new JobClient(conf); SleepJob sleepJob = new SleepJob(); sleepJob.setConf(conf); // Start the job RunningJob job = jClient.submitJob(sleepJob.setupJobConf(1, 1, 5000, 1, 1000, 1)); boolean TTOverFlowMsgPresent = false; while (true) { List<TaskReport> allTaskReports = new ArrayList<TaskReport>(); allTaskReports.addAll(Arrays.asList(jClient .getSetupTaskReports((org.apache.hadoop.mapred.JobID) job.getID()))); allTaskReports.addAll(Arrays.asList(jClient .getMapTaskReports((org.apache.hadoop.mapred.JobID) job.getID()))); for (TaskReport tr : allTaskReports) { String[] diag = tr.getDiagnostics(); for (String str : diag) { mat = physicalMemoryOverLimitPattern.matcher(str); if (mat.find()) { TTOverFlowMsgPresent = true; } } } if (TTOverFlowMsgPresent) { break; } try { Thread.sleep(1000); } catch (InterruptedException e) { // nothing } } // If it comes here without a test-timeout, it means there was a task that // was killed because of crossing cumulative TT limit. // Test succeeded, kill the job. job.killJob(); }
