Java 类org.apache.hadoop.mapreduce.server.tasktracker.Localizer 实例源码

项目：hadoop-2.6.0-cdh5.4.3 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());
  tracker.setLocalFileSystem(tracker.systemFS);

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR);

  // setup task controller
  taskController = getTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs()));
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs));
}

项目：hadoop-2.6.0-cdh5.4.3 文件：TestUserLogCleanup.java

private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
                    TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
    new LocalDirAllocator("mapred.local.dir");
  tt.setLocalDirAllocator(localDirAllocator);
  LocalStorage localStorage = new LocalStorage(conf.getLocalDirs());
  LocalFileSystem localFs = FileSystem.getLocal(conf);
  localStorage.checkDirs(localFs, true);
  tt.setLocalStorage(localStorage);
  localizer = new Localizer(FileSystem.get(conf), conf
      .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator, localStorage);
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}

项目：hadoop-on-lustre 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());
  tracker.setLocalFileSystem(tracker.systemFS);

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR);

  // setup task controller
  taskController = getTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs()));
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs));
}

项目：hadoop-on-lustre 文件：TestUserLogCleanup.java

private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
                    TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
                    new LocalDirAllocator("mapred.local.dir");
  localizer = new Localizer(FileSystem.get(conf), conf
      .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator,
      new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)));
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}

项目：hanoi-hadoop-2.0.0-cdh 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());
  tracker.setLocalFileSystem(tracker.systemFS);

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR);

  // setup task controller
  taskController = getTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs()));
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs));
}

项目：hanoi-hadoop-2.0.0-cdh 文件：TestUserLogCleanup.java

private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
                    TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
    new LocalDirAllocator("mapred.local.dir");
  tt.setLocalDirAllocator(localDirAllocator);
  LocalStorage localStorage = new LocalStorage(conf.getLocalDirs());
  LocalFileSystem localFs = FileSystem.getLocal(conf);
  localStorage.checkDirs(localFs, true);
  tt.setLocalStorage(localStorage);
  localizer = new Localizer(FileSystem.get(conf), conf
      .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator, localStorage);
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}

项目：mapreduce-fork 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.setLocalFileSystem(tracker.systemFS);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf));
  tracker.getAsyncDiskService().cleanupAllVolumes();

  // Set up TaskTracker instrumentation
  tracker.setTaskTrackerInstrumentation(
      TaskTracker.createInstrumentation(tracker, trackerFConf));

  // setup task controller
  taskController = createTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup();
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs,
      taskController));
}

项目：hortonworks-extension 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());
  tracker.setLocalFileSystem(tracker.systemFS);

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR);

  // setup task controller
  taskController = getTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs()));
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs));
}

项目：hortonworks-extension 文件：TestUserLogCleanup.java

private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
                    TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
                    new LocalDirAllocator("mapred.local.dir");
  localizer = new Localizer(FileSystem.get(conf), conf
      .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator,
      new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)));
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}

项目：hortonworks-extension 文件：TestTaskTrackerLocalization.java

private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());
  tracker.setLocalFileSystem(tracker.systemFS);

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  trackerFConf.deleteLocalFiles(TaskTracker.SUBDIR);

  // setup task controller
  taskController = getTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup(lDirAlloc, new LocalStorage(trackerFConf.getLocalDirs()));
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(),localDirs));
}

项目：hortonworks-extension 文件：TestUserLogCleanup.java

private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
                    TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
                    new LocalDirAllocator("mapred.local.dir");
  localizer = new Localizer(FileSystem.get(conf), conf
      .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator,
      new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)));
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}

项目：mapreduce-fork 文件：TestUserLogCleanup.java

public TestUserLogCleanup() throws IOException {
  Configuration conf = new Configuration();
  localizer = new Localizer(FileSystem.get(conf), conf
      .getStrings(MRConfig.LOCAL_DIR), new DefaultTaskController());
  taskLogCleanupThread = new UserLogCleaner(conf);
  taskLogCleanupThread.setClock(myClock);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  tt.setLocalizer(localizer);
  tt.setTaskLogCleanupThread(taskLogCleanupThread);
}

项目：hadoop-on-lustre 文件：DefaultTaskController.java

/**
 * Create all of the directories for the task and launches the child jvm.
 * @param user the user name
 * @param attemptId the attempt id
 * @throws IOException
 */
@Override
public int launchTask(String user, 
                                String jobId,
                                String attemptId,
                                List<String> setup,
                                List<String> jvmArguments,
                                File currentWorkDirectory,
                                String stdout,
                                String stderr) throws IOException {
  ShellCommandExecutor shExec = null;
  try {                 
    FileSystem localFs = FileSystem.getLocal(getConf());

    //create the attempt dirs
    new Localizer(localFs, 
        getConf().getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)).
        initializeAttemptDirs(user, jobId, attemptId);

    // create the working-directory of the task 
    if (!currentWorkDirectory.mkdir()) {
      throw new IOException("Mkdirs failed to create " 
                  + currentWorkDirectory.toString());
    }

    //mkdir the loglocation
    String logLocation = TaskLog.getAttemptDir(jobId, attemptId).toString();
    if (!localFs.mkdirs(new Path(logLocation))) {
      throw new IOException("Mkdirs failed to create " 
                 + logLocation);
    }
    //read the configuration for the job
    FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
    long logSize = 0; //TODO MAPREDUCE-1100
    // get the JVM command line.
    String cmdLine = 
      TaskLog.buildCommandLine(setup, jvmArguments,
          new File(stdout), new File(stderr), logSize, true);

    // write the command to a file in the
    // task specific cache directory
    // TODO copy to user dir
    Path p = new Path(allocator.getLocalPathForWrite(
        TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId),
        getConf()), COMMAND_FILE);

    String commandFile = writeCommand(cmdLine, rawFs, p);
    rawFs.setPermission(p, TaskController.TASK_LAUNCH_SCRIPT_PERMISSION);
    shExec = new ShellCommandExecutor(new String[]{
        "bash", "-c", commandFile},
        currentWorkDirectory);
    shExec.execute();
  } catch (Exception e) {
    if (shExec == null) {
      return -1;
    }
    int exitCode = shExec.getExitCode();
    LOG.warn("Exit code from task is : " + exitCode);
    LOG.info("Output from DefaultTaskController's launchTask follows:");
    logOutput(shExec.getOutput());
    return exitCode;
  }
  return 0;
}

项目：mapreduce-fork 文件：TaskTracker.java

Localizer getLocalizer() {
  return localizer;
}

项目：mapreduce-fork 文件：TaskTracker.java

void setLocalizer(Localizer l) {
  localizer = l;
}

项目：hortonworks-extension 文件：DefaultTaskController.java

/**
 * Create all of the directories for the task and launches the child jvm.
 * @param user the user name
 * @param attemptId the attempt id
 * @throws IOException
 */
@Override
public int launchTask(String user, 
                                String jobId,
                                String attemptId,
                                List<String> setup,
                                List<String> jvmArguments,
                                File currentWorkDirectory,
                                String stdout,
                                String stderr) throws IOException {
  ShellCommandExecutor shExec = null;
  try {                 
    FileSystem localFs = FileSystem.getLocal(getConf());

    //create the attempt dirs
    new Localizer(localFs, 
        getConf().getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)).
        initializeAttemptDirs(user, jobId, attemptId);

    // create the working-directory of the task 
    if (!currentWorkDirectory.mkdir()) {
      throw new IOException("Mkdirs failed to create " 
                  + currentWorkDirectory.toString());
    }

    //mkdir the loglocation
    String logLocation = TaskLog.getAttemptDir(jobId, attemptId).toString();
    if (!localFs.mkdirs(new Path(logLocation))) {
      throw new IOException("Mkdirs failed to create " 
                 + logLocation);
    }
    //read the configuration for the job
    FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
    long logSize = 0; //TODO MAPREDUCE-1100
    // get the JVM command line.
    String cmdLine = 
      TaskLog.buildCommandLine(setup, jvmArguments,
          new File(stdout), new File(stderr), logSize, true);

    // write the command to a file in the
    // task specific cache directory
    // TODO copy to user dir
    Path p = new Path(allocator.getLocalPathForWrite(
        TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId),
        getConf()), COMMAND_FILE);

    String commandFile = writeCommand(cmdLine, rawFs, p);
    rawFs.setPermission(p, TaskController.TASK_LAUNCH_SCRIPT_PERMISSION);
    shExec = new ShellCommandExecutor(new String[]{
        "bash", "-c", commandFile},
        currentWorkDirectory);
    shExec.execute();
  } catch (Exception e) {
    if (shExec == null) {
      return -1;
    }
    int exitCode = shExec.getExitCode();
    LOG.warn("Exit code from task is : " + exitCode);
    LOG.info("Output from DefaultTaskController's launchTask follows:");
    logOutput(shExec.getOutput());
    return exitCode;
  }
  return 0;
}

项目：hortonworks-extension 文件：DefaultTaskController.java

/**
 * Create all of the directories for the task and launches the child jvm.
 * @param user the user name
 * @param attemptId the attempt id
 * @throws IOException
 */
@Override
public int launchTask(String user, 
                                String jobId,
                                String attemptId,
                                List<String> setup,
                                List<String> jvmArguments,
                                File currentWorkDirectory,
                                String stdout,
                                String stderr) throws IOException {
  ShellCommandExecutor shExec = null;
  try {                 
    FileSystem localFs = FileSystem.getLocal(getConf());

    //create the attempt dirs
    new Localizer(localFs, 
        getConf().getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)).
        initializeAttemptDirs(user, jobId, attemptId);

    // create the working-directory of the task 
    if (!currentWorkDirectory.mkdir()) {
      throw new IOException("Mkdirs failed to create " 
                  + currentWorkDirectory.toString());
    }

    //mkdir the loglocation
    String logLocation = TaskLog.getAttemptDir(jobId, attemptId).toString();
    if (!localFs.mkdirs(new Path(logLocation))) {
      throw new IOException("Mkdirs failed to create " 
                 + logLocation);
    }
    //read the configuration for the job
    FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
    long logSize = 0; //TODO MAPREDUCE-1100
    // get the JVM command line.
    String cmdLine = 
      TaskLog.buildCommandLine(setup, jvmArguments,
          new File(stdout), new File(stderr), logSize, true);

    // write the command to a file in the
    // task specific cache directory
    // TODO copy to user dir
    Path p = new Path(allocator.getLocalPathForWrite(
        TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId),
        getConf()), COMMAND_FILE);

    String commandFile = writeCommand(cmdLine, rawFs, p);
    rawFs.setPermission(p, TaskController.TASK_LAUNCH_SCRIPT_PERMISSION);
    shExec = new ShellCommandExecutor(new String[]{
        "bash", "-c", commandFile},
        currentWorkDirectory);
    shExec.execute();
  } catch (Exception e) {
    if (shExec == null) {
      return -1;
    }
    int exitCode = shExec.getExitCode();
    LOG.warn("Exit code from task is : " + exitCode);
    LOG.info("Output from DefaultTaskController's launchTask follows:");
    logOutput(shExec.getOutput());
    return exitCode;
  }
  return 0;
}