@Test /** Test that volumes specified as relative paths are handled properly * by MRAsyncDiskService (MAPREDUCE-1887). */ public void testVolumeNormalization() throws Throwable { LOG.info("TEST_ROOT_DIR is " + TEST_ROOT_DIR); String relativeTestRoot = relativeToWorking(TEST_ROOT_DIR); FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String [] vols = new String[] { relativeTestRoot + "/0", relativeTestRoot + "/1" }; // Put a file in one of the volumes to be cleared on startup. Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED); localFileSystem.mkdirs(delDir); localFileSystem.create(new Path(delDir, "foo")).close(); MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); makeSureCleanedUp(vols, service); }
/** * This test creates some directories inside the toBeDeleted directory and * then start the asyncDiskService. * AsyncDiskService will create tasks to delete the content inside the * toBeDeleted directories. */ @Test public void testMRAsyncDiskServiceStartupCleaning() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; String a = "a"; String b = "b"; String c = "b/c"; String d = "d"; // Create directories inside SUBDIR String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED; File fa = new File(vols[0] + suffix, a); File fb = new File(vols[1] + suffix, b); File fc = new File(vols[1] + suffix, c); File fd = new File(vols[1] + suffix, d); // Create the directories fa.mkdirs(); fb.mkdirs(); fc.mkdirs(); fd.mkdirs(); assertTrue(fa.exists()); assertTrue(fb.exists()); assertTrue(fc.exists()); assertTrue(fd.exists()); // Create the asyncDiskService which will delete all contents inside SUBDIR MRAsyncDiskService service = new MRAsyncDiskService( localFileSystem, vols); // Make sure everything is cleaned up makeSureCleanedUp(vols, service); }
private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service) throws Throwable { // Sleep at most 5 seconds to make sure the deleted items are all gone. service.shutdown(); if (!service.awaitTermination(5000)) { fail("MRAsyncDiskService is still not shutdown in 5 seconds!"); } // All contents should be gone by now. for (int i = 0; i < vols.length; i++) { File subDir = new File(vols[0]); String[] subDirContent = subDir.list(); assertEquals("Volume should contain a single child: " + MRAsyncDiskService.TOBEDELETED, 1, subDirContent.length); File toBeDeletedDir = new File(vols[0], MRAsyncDiskService.TOBEDELETED); String[] content = toBeDeletedDir.list(); assertNotNull("Cannot find " + toBeDeletedDir, content); assertEquals("" + toBeDeletedDir + " should be empty now.", 0, content.length); } }
/** * Delete a local path with asyncDiskService if available, * or otherwise synchronously with local file system. */ private static void deleteLocalPath(MRAsyncDiskService asyncDiskService, LocalFileSystem fs, Path path) throws IOException { boolean deleted = false; if (asyncDiskService != null) { // Try to delete using asyncDiskService String localPathToDelete = path.toUri().getPath(); deleted = asyncDiskService.moveAndDeleteAbsolutePath(localPathToDelete); if (!deleted) { LOG.warn("Cannot find DistributedCache path " + localPathToDelete + " on any of the asyncDiskService volumes!"); } } if (!deleted) { // If no asyncDiskService, we will delete the files synchronously fs.delete(path, true); } LOG.info("Deleted path " + path); }
private void initializeTracker() throws IOException { tracker.setIndexCache(new IndexCache(trackerFConf)); tracker.setTaskMemoryManagerEnabledFlag(); // for test case system FS is the local FS tracker.systemFS = FileSystem.getLocal(trackerFConf); tracker.setLocalFileSystem(tracker.systemFS); tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>(); tracker.runningJobs = new TreeMap<JobID, RunningJob>(); tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf)); tracker.getAsyncDiskService().cleanupAllVolumes(); // Set up TaskTracker instrumentation tracker.setTaskTrackerInstrumentation( TaskTracker.createInstrumentation(tracker, trackerFConf)); // setup task controller taskController = createTaskController(); taskController.setConf(trackerFConf); taskController.setup(); tracker.setTaskController(taskController); tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs, taskController)); }
@Test public void testToleratesSomeUnwritableVolumes() throws Throwable { FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); String[] vols = new String[]{TEST_ROOT_DIR + "/0", TEST_ROOT_DIR + "/1"}; assertTrue(new File(vols[0]).mkdirs()); assertEquals(0, FileUtil.chmod(vols[0], "400")); // read only try { new MRAsyncDiskService(localFileSystem, vols); } finally { FileUtil.chmod(vols[0], "755"); // make writable again } }
