Java 类org.apache.hadoop.util.MemoryCalculatorPlugin 实例源码

/**
 * Test that verifies default values are configured and reported correctly.
 * 
 * @throws Exception
 */
public void testDefaultMemoryValues()
    throws Exception {
  JobConf conf = new JobConf();
  try {
    // Memory values are disabled by default.
    conf.setClass(
        TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
        DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
    setUpCluster(conf);
    runSleepJob(miniMRCluster.createJobConf());
    verifyTestResults();
  } finally {
    tearDownCluster();
  }
}

/**
 * Test that verifies default values are configured and reported correctly.
 * 
 * @throws Exception
 */
public void testDefaultMemoryValues()
    throws Exception {
  JobConf conf = new JobConf();
  try {
    // Memory values are disabled by default.
    conf.setClass(
        TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
        DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
    setUpCluster(conf);
    runSleepJob(miniMRCluster.createJobConf());
    verifyTestResults();
  } finally {
    tearDownCluster();
  }
}

/**
 * Test that verifies that configured values are reported correctly.
 * 
 * @throws Exception
 */
public void testConfiguredMemoryValues()
    throws Exception {
  JobConf conf = new JobConf();
  conf.setLong("totalVmemOnTT", 4 * 1024 * 1024 * 1024L);
  conf.setLong("totalPmemOnTT", 2 * 1024 * 1024 * 1024L);
  conf.setLong("mapSlotMemorySize", 1 * 512L);
  conf.setLong("reduceSlotMemorySize", 1 * 1024L);

  conf.setClass(
      TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
      DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
  conf.setLong(DummyMemoryCalculatorPlugin.MAXVMEM_TESTING_PROPERTY,
      4 * 1024 * 1024 * 1024L);
  conf.setLong(DummyMemoryCalculatorPlugin.MAXPMEM_TESTING_PROPERTY,
      2 * 1024 * 1024 * 1024L);
  conf.setLong(JobTracker.MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY,
      512L);
  conf.setLong(
      JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY, 1024L);

  try {
    setUpCluster(conf);
    JobConf jobConf = miniMRCluster.createJobConf();
    jobConf.setMemoryForMapTask(1 * 1024L);
    jobConf.setMemoryForReduceTask(2 * 1024L);
    runSleepJob(jobConf);
    verifyTestResults();
  } finally {
    tearDownCluster();
  }
}

/**
 * Test that verifies that configured values are reported correctly.
 * 
 * @throws Exception
 */
public void testConfiguredMemoryValues()
    throws Exception {
  JobConf conf = new JobConf();
  conf.setLong("totalVmemOnTT", 4 * 1024 * 1024 * 1024L);
  conf.setLong("totalPmemOnTT", 2 * 1024 * 1024 * 1024L);
  conf.setLong("mapSlotMemorySize", 1 * 512L);
  conf.setLong("reduceSlotMemorySize", 1 * 1024L);

  conf.setClass(
      TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
      DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
  conf.setLong(DummyMemoryCalculatorPlugin.MAXVMEM_TESTING_PROPERTY,
      4 * 1024 * 1024 * 1024L);
  conf.setLong(DummyMemoryCalculatorPlugin.MAXPMEM_TESTING_PROPERTY,
      2 * 1024 * 1024 * 1024L);
  conf.setLong(JobTracker.MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY,
      512L);
  conf.setLong(
      JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY, 1024L);

  try {
    setUpCluster(conf);
    JobConf jobConf = miniMRCluster.createJobConf();
    jobConf.setMemoryForMapTask(1 * 1024L);
    jobConf.setMemoryForReduceTask(2 * 1024L);
    runSleepJob(jobConf);
    verifyTestResults();
  } finally {
    tearDownCluster();
  }
}