public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException { float maxUsableSpacePercentagePerDisk = conf.getFloat( YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, YarnConfiguration.DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE); long minFreeSpacePerDiskMB = conf.getLong(YarnConfiguration.NM_MIN_PER_DISK_FREE_SPACE_MB, YarnConfiguration.DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB); localDirs = new DirectoryCollection( validatePaths(conf .getTrimmedStrings(YarnConfiguration.NM_LOCAL_DIRS)), maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB); logDirs = new DirectoryCollection( validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)), maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB); localDirsAllocator = new LocalDirAllocator( YarnConfiguration.NM_LOCAL_DIRS); logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS); }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile = ReflectionUtils.newInstance( conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, MROutputFiles.class, MapOutputFile.class), conf); this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
public OSSOutputStream(Configuration conf, OSSFileSystem fs, String bucket, String key, Progressable progress, FileSystem.Statistics statistics) throws IOException { this.bucket = bucket; this.key = key; this.progress = progress; this.fs = fs; this.statistics = statistics; lDirAlloc = new LocalDirAllocator(SmartOSSClientConfig.BUFFER_DIR); backupFile = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); closed = false; if (LOG.isDebugEnabled()) { LOG.debug("OutputStream for key '" + key + "' writing to tempfile: " + this.backupFile); } this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile)); }
/** * Default constructor * * @param account JOSS account object * @param url URL connection * @param contentTypeT content type * @param metadataT input metadata * @param connectionManager SwiftConnectionManager * @param fsT SwiftAPIClient * @throws IOException if error */ public SwiftNoStreamingOutputStream(JossAccount account, URL url, final String contentTypeT, Map<String, String> metadataT, SwiftConnectionManager connectionManager, SwiftAPIClient fsT) throws IOException { LOG.debug("SwiftNoStreamingOutputStream constructor entry for {}", url.toString()); mUrl = url; contentType = contentTypeT; mAccount = account; scm = connectionManager; metadata = metadataT; fs = fsT; try { mBackupFile = fs.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN); LOG.debug("OutputStream for key '{}' writing to tempfile: {}", mUrl.toString(), mBackupFile); mBackupOutputStream = new BufferedOutputStream(new FileOutputStream(mBackupFile), 32768); } catch (IOException e) { LOG.error(e.getMessage()); throw e; } }
public JobLocalizer(JobConf ttConf, String user, String jobid, String... localDirs) throws IOException { if (null == user) { throw new IOException("Cannot initialize for null user"); } this.user = user; if (null == jobid) { throw new IOException("Cannot initialize for null jobid"); } this.jobid = jobid; this.ttConf = new JobConf(ttConf); lfs = FileSystem.getLocal(this.ttConf).getRaw(); this.localDirs = createPaths(user, localDirs); this.ttConf.setStrings(JOB_LOCAL_CTXT, localDirs); Collections.shuffle(this.localDirs); lDirAlloc = new LocalDirAllocator(JOB_LOCAL_CTXT); JOBDIR = TaskTracker.JOBCACHE + Path.SEPARATOR + jobid; DISTDIR = JOBDIR + "/" + TaskTracker.DISTCACHEDIR; WORKDIR = JOBDIR + "/work"; JARDST = JOBDIR + "/" + TaskTracker.JARSDIR + "/job.jar"; JOBCONF = JOBDIR + "/" + TaskTracker.JOBFILE; JOBTOKEN = JOBDIR + "/" + TaskTracker.JOB_TOKEN_FILE; }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator("mapred.local.dir"); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions"); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
/** * Write the child's configuration to the disk and set it in configuration so * that the child can pick it up from there. * * @param lDirAlloc * @throws IOException */ void setupChildTaskConfiguration(LocalDirAllocator lDirAlloc) throws IOException { Path localTaskFile = lDirAlloc.getLocalPathForWrite(TaskTracker.getTaskConfFile( t.getUser(), t.getJobID().toString(), t.getTaskID().toString(), t .isTaskCleanupTask()), conf); // write the child's task configuration file to the local disk JobLocalizer.writeLocalJobFile(localTaskFile, conf); // Set the final job file in the task. The child needs to know the correct // path to job.xml. So set this path accordingly. t.setJobFile(localTaskFile.toString()); }
static void localizeTask(Task task, JobConf jobConf, String logLocation) throws IOException{ // Do the task-type specific localization task.localizeConfiguration(jobConf); // Write files required to re-run the task with IsolationRunner if (task.supportIsolationRunner(jobConf)) { task.writeFilesRequiredForRerun(jobConf); } //write the localized task jobconf LocalDirAllocator lDirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY); Path localTaskFile = lDirAlloc.getLocalPathForWrite(TaskTracker.JOBFILE, jobConf); JobLocalizer.writeLocalJobFile(localTaskFile, jobConf); task.setJobFile(localTaskFile.toString()); task.setConf(jobConf); }
@Override public void writeFilesRequiredForRerun(JobConf conf) throws IOException { // split.info file is used only by IsolationRunner. // Write the split file to the local disk if it is a normal map task (not a // job-setup or a job-cleanup task) and if the user wishes to run // IsolationRunner either by setting keep.failed.tasks.files to true or by // using keep.tasks.files.pattern if (isMapOrReduce()) { // localize the split meta-information Path localSplitMeta = new LocalDirAllocator("mapred.local.dir").getLocalPathForWrite( TaskTracker.getLocalSplitFile(conf.getUser(), getJobID() .toString(), getTaskID().toString()), conf); LOG.debug("Writing local split to " + localSplitMeta); DataOutputStream out = FileSystem.getLocal(conf).create(localSplitMeta); splitMetaInfo.write(out); out.close(); } }
@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // Check the permissions of the task-controller binary by running // it plainly. If permissions are correct, it returns an error // code 1, else it returns 24 or something else if some other bugs // are also present. String[] taskControllerCmd = new String[] { taskControllerExe }; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } } this.allocator = allocator; this.localStorage = localStorage; }
public TrackerDistributedCacheManager(Configuration conf, TaskController controller) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator("mapred.local.dir"); // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong ("local.cache.size", DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong ("mapreduce.tasktracker.cache.local.numberdirectories", DEFAULT_CACHE_SUBDIR_LIMIT); double cleanupPct = conf.getFloat("mapreduce.tasktracker.cache.local.keep.pct", DEFAULT_CACHE_KEEP_AROUND_PCT); this.allowedCacheSizeCleanupGoal = (long)(this.allowedCacheSize * cleanupPct); this.allowedCacheSubdirsCleanupGoal = (long)(this.allowedCacheSubdirs * cleanupPct); this.taskController = controller; this.cleanupThread = new CleanupThread(conf); }
private TaskTracker setupTaskTracker(Configuration conf) throws Exception { JobConf ttConf = new JobConf(conf); // Doesn't matter what we give here - we won't actually // connect to it. TaskTracker tt = new TaskTracker(); tt.setConf(ttConf); tt.setTaskController(Mockito.mock(TaskController.class)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalFileSystem localFs = FileSystem.getLocal(conf); LocalStorage localStorage = new LocalStorage(ttConf.getLocalDirs()); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); tt.setLocalFileSystem(localFs); tt.initializeDirectories(); return tt; }
public TestJvmManager() throws Exception { user = UserGroupInformation.getCurrentUser().getShortUserName(); tt = new TaskTracker(); ttConf = new JobConf(); ttConf.setLong("mapred.tasktracker.tasks.sleeptime-before-sigkill", 2000); tt.setConf(ttConf); tt.setMaxMapSlots(MAP_SLOTS); tt.setMaxReduceSlots(REDUCE_SLOTS); TaskController dtc; tt.setTaskController((dtc = new DefaultTaskController())); Configuration conf = new Configuration(); dtc.setConf(conf); LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY); tt.getTaskController().setup(ldirAlloc, new LocalStorage(ttConf.getLocalDirs())); JobID jobId = new JobID("test", 0); jvmManager = new JvmManager(tt); tt.setJvmManagerInstance(jvmManager); tt.setUserLogManager(new UserLogManager(ttConf)); tt.setCleanupThread(new InlineCleanupQueue()); }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalStorage localStorage = new LocalStorage(conf.getLocalDirs()); LocalFileSystem localFs = FileSystem.getLocal(conf); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); localizer = new Localizer(FileSystem.get(conf), conf .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, localStorage); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
@SuppressWarnings("deprecation") public CoronaJobTrackerRunner( TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker, JobConf ttConf, CoronaSessionInfo info, String originalPath, String releasePath) throws IOException { super(tip, task, tracker, ttConf); this.coronaSessionInfo = info; this.originalPath = originalPath; this.releasePath = releasePath; LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); workDir = new File(lDirAlloc.getLocalPathForWrite( TaskTracker.getLocalTaskDir( task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, conf). toString()); if (!workDir.mkdirs()) { if (!workDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + workDir.toString()); } } localizeTaskConfiguration(tracker, ttConf, workDir.toString(), task, task .getJobID()); }
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); this.sortAlgorithm = getSortAlgorithm(context.getQueryContext(), sortSpecs); LOG.info(sortAlgorithm.name() + " sort is selected"); }
public void cleanupTemporalDirectories() { if (deletionService == null) return; LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); try { Iterable<Path> iter = lDirAllocator.getAllLocalPathsToRead(".", systemConf); FileSystem localFS = FileSystem.getLocal(systemConf); for (Path path : iter) { PathData[] items = PathData.expandAsGlob(localFS.makeQualified(new Path(path, "*")).toString(), systemConf); ArrayList<Path> paths = new ArrayList<>(); for (PathData pd : items) { paths.add(pd.path); } if (paths.size() == 0) continue; deletionService.delete(null, paths.toArray(new Path[paths.size()])); } } catch (IOException e) { LOG.error(e.getMessage(), e); } }
public HashShuffleAppenderManager(TajoConf systemConf) throws IOException { this.systemConf = systemConf; // initialize LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); // initialize DFS and LocalFileSystems defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf); localFS = FileSystem.getLocal(systemConf); pageSize = systemConf.getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME) * StorageUnit.MB; Iterable<Path> allLocalPath = lDirAllocator.getAllLocalPathsToRead(".", systemConf); //add async hash shuffle writer for (Path path : allLocalPath) { temporalPaths.add(localFS.makeQualified(path).toString()); executors.put(temporalPaths.size() - 1, Executors.newSingleThreadExecutor()); } }
private ExternalSortExec(final TaskAttemptContext context, final AbstractStorageManager sm, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getConf().getLongVar(ConfVars.EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE) * 1048576L; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum); this.inMemoryTable = new ArrayList<Tuple>(100000); this.sortTmpDir = getExecutorTmpDir(); localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); localFS = new RawLocalFileSystem(); }
protected void cleanupTemporalDirectories() { if(deletionService == null) return; LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); try { Iterable<Path> iter = lDirAllocator.getAllLocalPathsToRead(".", systemConf); FileSystem localFS = FileSystem.getLocal(systemConf); for (Path path : iter){ PathData[] items = PathData.expandAsGlob(localFS.makeQualified(new Path(path, "*")).toString(), systemConf); ArrayList<Path> paths = new ArrayList<Path>(); for (PathData pd : items){ paths.add(pd.path); } if(paths.size() == 0) continue; deletionService.delete(null, paths.toArray(new Path[paths.size()])); } } catch (IOException e) { LOG.error(e.getMessage(), e); } }
public JobLocalizer(JobConf ttConf, String user, String jobid, String... localDirs) throws IOException { if (null == user) { throw new IOException("Cannot initialize for null user"); } this.user = user; if (null == jobid) { throw new IOException("Cannot initialize for null jobid"); } this.jobid = jobid; this.ttConf = ttConf; lfs = FileSystem.getLocal(ttConf).getRaw(); this.localDirs = createPaths(user, localDirs); ttConf.setStrings(JOB_LOCAL_CTXT, localDirs); Collections.shuffle(this.localDirs); lDirAlloc = new LocalDirAllocator(JOB_LOCAL_CTXT); JOBDIR = TaskTracker.JOBCACHE + Path.SEPARATOR + jobid; DISTDIR = JOBDIR + "/" + TaskTracker.DISTCACHEDIR; WORKDIR = JOBDIR + "/work"; JARDST = JOBDIR + "/" + TaskTracker.JARSDIR + "/job.jar"; JOBCONF = JOBDIR + "/" + TaskTracker.JOBFILE; JOBTOKEN = JOBDIR + "/" + TaskTracker.JOB_TOKEN_FILE; }
@Override public void localizeConfiguration(JobConf conf) throws IOException { super.localizeConfiguration(conf); // split.info file is used only by IsolationRunner. // Write the split file to the local disk if it is a normal map task (not a // job-setup or a job-cleanup task) and if the user wishes to run // IsolationRunner either by setting keep.failed.tasks.files to true or by // using keep.tasks.files.pattern if (supportIsolationRunner(conf) && isMapOrReduce()) { // localize the split meta-information Path localSplitMeta = new LocalDirAllocator("mapred.local.dir").getLocalPathForWrite( TaskTracker.getLocalSplitFile(conf.getUser(), getJobID() .toString(), getTaskID().toString()), conf); LOG.debug("Writing local split to " + localSplitMeta); DataOutputStream out = FileSystem.getLocal(conf).create(localSplitMeta); splitMetaInfo.write(out); out.close(); } }
@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // Check the permissions of the task-controller binary by running it plainly. // If permissions are correct, it returns an error code 1, else it returns // 24 or something else if some other bugs are also present. String[] taskControllerCmd = new String[] { taskControllerExe }; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } } this.allocator = allocator; this.localStorage = localStorage; }
public TrackerDistributedCacheManager(Configuration conf, TaskController controller ) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator("mapred.local.dir"); // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong ("local.cache.size", DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong ("mapreduce.tasktracker.local.cache.numberdirectories", DEFAULT_CACHE_SUBDIR_LIMIT); double cleanupPct = conf.getFloat("mapreduce.tasktracker.cache.local.keep.pct", DEFAULT_CACHE_KEEP_AROUND_PCT); this.allowedCacheSizeCleanupGoal = (long)(this.allowedCacheSize * cleanupPct); this.allowedCacheSubdirsCleanupGoal = (long)(this.allowedCacheSubdirs * cleanupPct); this.taskController = controller; this.cleanupThread = new CleanupThread(conf); }
protected void initMyTest() throws Exception { testDir.mkdirs(); mapredLocal.mkdirs(); createFakeTCScript(); conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, mapredLocal.toString()); // Set the task-controller binary path. conf.set("mapreduce.tasktracker.task-controller.exe", fakeTaskController.toString()); ltc = new LinuxTaskController(); ltc.setConf(conf); // LinuxTaskController runs task-controller in setup() with no // argument and expects 1 in return try { ltc.setup(new LocalDirAllocator(mapredLocal.toString()), new LocalStorage(new String[]{mapredLocal.toString()})); } catch (IOException ie) { fail("Error running task-controller from setup()."); } initialized = true; }
private void validateTaskControllerSetup(TaskController controller, boolean shouldFail) throws IOException { if (shouldFail) { // task controller setup should fail validating permissions. Throwable th = null; try { controller.setup( new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } catch (IOException ie) { th = ie; } assertNotNull("No exception during setup", th); assertTrue("Exception message does not contain exit code" + INVALID_TASKCONTROLLER_PERMISSIONS, th.getMessage().contains( "with exit code " + INVALID_TASKCONTROLLER_PERMISSIONS)); } else { controller.setup(new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY), new LocalStorage(controller.getConf().getStrings( JobConf.MAPRED_LOCAL_DIR_PROPERTY))); } }
