@Test public void testDeleteOnExitPathHandling() throws IOException { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI chrootUri = URI.create("mockfs://foo/a/b"); ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem()) .getRawFileSystem(); // ensure delete propagates the correct path Path chrootPath = new Path("/c"); Path rawPath = new Path("/a/b/c"); chrootFs.delete(chrootPath, false); verify(mockFs).delete(eq(rawPath), eq(false)); reset(mockFs); // fake that the path exists for deleteOnExit FileStatus stat = mock(FileStatus.class); when(mockFs.getFileStatus(eq(rawPath))).thenReturn(stat); // ensure deleteOnExit propagates the correct path chrootFs.deleteOnExit(chrootPath); chrootFs.close(); verify(mockFs).delete(eq(rawPath), eq(true)); }
@Test(timeout = 30000) public void testCreateSnapshot() throws Exception { Path snapRootPath = new Path("/snapPath"); Path chRootedSnapRootPath = new Path("/a/b/snapPath"); Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI chrootUri = URI.create("mockfs://foo/a/b"); ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) .getRawFileSystem(); chrootFs.createSnapshot(snapRootPath, "snap1"); verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1"); }
@Test(timeout = 30000) public void testDeleteSnapshot() throws Exception { Path snapRootPath = new Path("/snapPath"); Path chRootedSnapRootPath = new Path("/a/b/snapPath"); Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI chrootUri = URI.create("mockfs://foo/a/b"); ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) .getRawFileSystem(); chrootFs.deleteSnapshot(snapRootPath, "snap1"); verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1"); }
@Test(timeout = 30000) public void testRenameSnapshot() throws Exception { Path snapRootPath = new Path("/snapPath"); Path chRootedSnapRootPath = new Path("/a/b/snapPath"); Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI chrootUri = URI.create("mockfs://foo/a/b"); ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) .getRawFileSystem(); chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName"); verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName"); }
@SuppressWarnings("deprecation") public void testSetupDistributedCacheConflictsFiles() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); when(mockFs.resolvePath(filePath)).thenReturn(filePath); when(mockFs.resolvePath(file2Path)).thenReturn(file2Path); DistributedCache.addCacheFile(file, conf); DistributedCache.addCacheFile(file2, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); assertEquals(1, localResources.size()); LocalResource lr = localResources.get("something"); //First one wins assertNotNull(lr); assertEquals(10l, lr.getSize()); assertEquals(10l, lr.getTimestamp()); assertEquals(LocalResourceType.FILE, lr.getType()); }
/** * Tests that ChRootedFileSystem delegates calls for every ACL method to the * underlying FileSystem with all Path arguments translated as required to * enforce chroot. */ @Test public void testAclMethodsPathTranslation() throws IOException { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI chrootUri = URI.create("mockfs://foo/a/b"); ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem()) .getRawFileSystem(); Path chrootPath = new Path("/c"); Path rawPath = new Path("/a/b/c"); List<AclEntry> entries = Collections.emptyList(); chrootFs.modifyAclEntries(chrootPath, entries); verify(mockFs).modifyAclEntries(rawPath, entries); chrootFs.removeAclEntries(chrootPath, entries); verify(mockFs).removeAclEntries(rawPath, entries); chrootFs.removeDefaultAcl(chrootPath); verify(mockFs).removeDefaultAcl(rawPath); chrootFs.removeAcl(chrootPath); verify(mockFs).removeAcl(rawPath); chrootFs.setAcl(chrootPath, entries); verify(mockFs).setAcl(rawPath, entries); chrootFs.getAclStatus(chrootPath); verify(mockFs).getAclStatus(rawPath); }
@SuppressWarnings("deprecation") public void testSetupDistributedCacheConflicts() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI archive = new URI("mockfs://mock/tmp/something.zip#something"); Path archivePath = new Path(archive); URI file = new URI("mockfs://mock/tmp/something.txt#something"); Path filePath = new Path(file); when(mockFs.resolvePath(archivePath)).thenReturn(archivePath); when(mockFs.resolvePath(filePath)).thenReturn(filePath); DistributedCache.addCacheArchive(archive, conf); conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); DistributedCache.addCacheFile(file, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); assertEquals(1, localResources.size()); LocalResource lr = localResources.get("something"); //Archive wins assertNotNull(lr); assertEquals(10l, lr.getSize()); assertEquals(10l, lr.getTimestamp()); assertEquals(LocalResourceType.ARCHIVE, lr.getType()); }
@SuppressWarnings("deprecation") @Test(timeout = 120000, expected = InvalidJobConfException.class) public void testSetupDistributedCacheConflicts() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI archive = new URI("mockfs://mock/tmp/something.zip#something"); Path archivePath = new Path(archive); URI file = new URI("mockfs://mock/tmp/something.txt#something"); Path filePath = new Path(file); when(mockFs.resolvePath(archivePath)).thenReturn(archivePath); when(mockFs.resolvePath(filePath)).thenReturn(filePath); DistributedCache.addCacheArchive(archive, conf); conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); DistributedCache.addCacheFile(file, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); }
@SuppressWarnings("deprecation") @Test(timeout = 120000, expected = InvalidJobConfException.class) public void testSetupDistributedCacheConflictsFiles() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); when(mockFs.resolvePath(filePath)).thenReturn(filePath); when(mockFs.resolvePath(file2Path)).thenReturn(file2Path); DistributedCache.addCacheFile(file, conf); DistributedCache.addCacheFile(file2, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); }
/** * Returns a DistributedFileSystem hosting the path supplied. */ protected DistributedFileSystem getDFS(Path p) throws IOException { FileSystem fs = p.getFileSystem(getConf()); DistributedFileSystem dfs = null; if (fs instanceof DistributedFileSystem) { dfs = (DistributedFileSystem) fs; } else if (fs instanceof FilterFileSystem) { FilterFileSystem ffs = (FilterFileSystem) fs; if (ffs.getRawFileSystem() instanceof DistributedFileSystem) { dfs = (DistributedFileSystem) ffs.getRawFileSystem(); } } return dfs; }
@Override protected void initNameNodes() throws IOException { if (this.fs instanceof FilterFileSystem) { this.client = ((DistributedFileSystem) ((FilterFileSystem) this.fs) .getRawFileSystem()).getClient().getNameNodeRPC(); } else { this.client = ((DistributedFileSystem) this.fs).getClient() .getNameNodeRPC(); } initNamenodeProtocol(false); }
public static DistributedFileSystem convertToDFS(FileSystem fs) { // for RaidDFS if (fs instanceof FilterFileSystem) { fs = ((FilterFileSystem) fs).getRawFileSystem(); } if (fs instanceof DistributedFileSystem) return (DistributedFileSystem) fs; else return null; }
protected DistributedFileSystem getDFS() throws IOException{ FileSystem fs = getFS(); DistributedFileSystem dfs = null; if (fs instanceof DistributedFileSystem) { dfs = (DistributedFileSystem) fs; } else { if (fs instanceof FilterFileSystem) { FilterFileSystem ffs = (FilterFileSystem) fs; if (ffs.getRawFileSystem() instanceof DistributedFileSystem) { dfs = (DistributedFileSystem) ffs.getRawFileSystem(); } } } return dfs; }
@SuppressWarnings("deprecation") public void testSetupDistributedCacheConflictsFiles() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file); when(mockFs.resolvePath(filePath)).thenReturn(filePath); when(mockFs.resolvePath(file2Path)).thenReturn(file2Path); DistributedCache.addCacheFile(file, conf); DistributedCache.addCacheFile(file2, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); assertEquals(1, localResources.size()); LocalResource lr = localResources.get("something"); //First one wins assertNotNull(lr); assertEquals(10l, lr.getSize()); assertEquals(10l, lr.getTimestamp()); assertEquals(LocalResourceType.FILE, lr.getType()); }