@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res; try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { if (createParent) { throw readOnlyMountTable("create", f); } else { throw e; } } assert(res.remainingPath != null); return res.targetFileSystem.createInternal(res.remainingPath, flag, absolutePermission, bufferSize, replication, blockSize, progress, checksumOpt, createParent); }
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException { out.println("Result of running LoadGenerator against fileSystem: " + FileContext.getFileContext().getDefaultFileSystem().getUri()); if (numOfOps[OPEN] != 0) { out.println("Average open execution time: " + (double)executionTime[OPEN]/numOfOps[OPEN] + "ms"); } if (numOfOps[LIST] != 0) { out.println("Average list execution time: " + (double)executionTime[LIST]/numOfOps[LIST] + "ms"); } if (numOfOps[DELETE] != 0) { out.println("Average deletion execution time: " + (double)executionTime[DELETE]/numOfOps[DELETE] + "ms"); out.println("Average create execution time: " + (double)executionTime[CREATE]/numOfOps[CREATE] + "ms"); out.println("Average write_close execution time: " + (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms"); } if (totalTime != 0) { out.println("Average operations per second: " + (double)totalOps/totalTime +"ops/s"); } out.println(); }
private void doAppLogAggregationPostCleanUp() { // Remove the local app-log-dirs List<Path> localAppLogDirs = new ArrayList<Path>(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logPath = new Path(rootLogDir, applicationId); try { // check if log dir exists lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue); continue; } catch (IOException fe) { continue; } } if (localAppLogDirs.size() > 0) { this.delService.delete(this.userUgi.getShortUserName(), null, localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } }
/** * Creates a new instance. * * @param configuration the hadoop configuration * @param pathBase base path for all non-absolute location created through this {@link LocationFactory}. */ public FileContextLocationFactory(final Configuration configuration, String pathBase) { this.configuration = configuration; this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase); int maxCacheSize = configuration.getInt(Configs.Keys.FILE_CONTEXT_CACHE_MAX_SIZE, Configs.Defaults.FILE_CONTEXT_CACHE_MAX_SIZE); this.fileContextCache = CacheBuilder .newBuilder() .weakKeys() .weakValues() .maximumSize(maxCacheSize) .build(new CacheLoader<UserGroupInformation, FileContext>() { @Override public FileContext load(UserGroupInformation ugi) throws Exception { return ugi.doAs(new PrivilegedExceptionAction<FileContext>() { @Override public FileContext run() throws UnsupportedFileSystemException { return FileContext.getFileContext(configuration); } }); } }); }
private void doAppLogAggregationPostCleanUp() { // Remove the local app-log-dirs List<Path> localAppLogDirs = new ArrayList<Path>(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logPath = new Path(rootLogDir, userFolder + Path.SEPARATOR + applicationId); try { // check if log dir exists lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue); continue; } catch (IOException fe) { continue; } } if (localAppLogDirs.size() > 0) { this.delService.delete(this.userUgi.getShortUserName(), null, localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } }
/** * This constructor has the signature needed by * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. * * @param theUri which must be that of ViewFs * @param conf * @throws IOException * @throws URISyntaxException */ ViewFs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { super(theUri, FsConstants.VIEWFS_SCHEME, false, -1); creationTime = Time.now(); ugi = UserGroupInformation.getCurrentUser(); config = conf; // Now build client side view (i.e. client side mount table) from config. String authority = theUri.getAuthority(); fsState = new InodeTree<AbstractFileSystem>(conf, authority) { @Override protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { String pathString = uri.getPath(); if (pathString.isEmpty()) { pathString = "/"; } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), new Path(pathString)); } @Override protected AbstractFileSystem getTargetFileSystem( final INodeDir<AbstractFileSystem> dir) throws URISyntaxException { return new InternalDirOfViewFs(dir, creationTime, ugi, getUri()); } @Override protected AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList) throws URISyntaxException, UnsupportedFileSystemException { throw new UnsupportedFileSystemException("mergefs not implemented yet"); // return MergeFs.createMergeFs(mergeFsURIList, config); } }; }
@Override public FileStatus getFileLinkStatus(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(getUriPath(f), false); // do not follow mount link return res.targetFileSystem.getFileLinkStatus(res.remainingPath); }
@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { throw readOnlyMountTable("create", f); }
@Test(expected=FileAlreadyExistsException.class) public void testInvalidConfig() throws IOException, URISyntaxException { Configuration conf = new Configuration(); ConfigUtil.addLink(conf, "/internalDir/linkToDir2", new Path("file:///dir2").toUri()); ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3", new Path("file:///dir3").toUri()); class Foo { }; new InodeTree<Foo>(conf, null) { @Override protected Foo getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { return null; } @Override protected Foo getTargetFileSystem( org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir) throws URISyntaxException { return null; } @Override protected Foo getTargetFileSystem(URI[] mergeFsURIList) throws URISyntaxException, UnsupportedFileSystemException { return null; } }; }
@BeforeClass public static void setup() throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { localFS = FileContext.getLocalFSFileContext(); localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true); localFSDirBase.mkdirs(); // Do not start cluster here }
private void submitDirForDeletion(String userName, Path dir) { try { lfs.getFileStatus(dir); delService.delete(userName, dir, new Path[] {}); } catch (UnsupportedFileSystemException ue) { LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue); } catch (IOException ie) { // ignore return; } }
@Override protected void serviceInit(Configuration conf) throws Exception { try { //TODO Is this required? FileContext.getLocalFSFileContext(conf); } catch (UnsupportedFileSystemException e) { throw new YarnRuntimeException("Failed to start ContainersLauncher", e); } super.serviceInit(conf); }
@Override @SuppressWarnings("unchecked") public void run() { List<Path> localAppLogDirs = new ArrayList<Path>(); FileContext lfs = getLocalFileContext(getConfig()); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logDir = new Path(rootLogDir, applicationId.toString()); try { lfs.getFileStatus(logDir); localAppLogDirs.add(logDir); } catch (UnsupportedFileSystemException ue) { LOG.warn("Unsupported file system used for log dir " + logDir, ue); continue; } catch (IOException ie) { continue; } } // Inform the application before the actual delete itself, so that links // to logs will no longer be there on NM web-UI. NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle( new ApplicationEvent(this.applicationId, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED)); if (localAppLogDirs.size() > 0) { NonAggregatingLogHandler.this.delService.delete(user, null, (Path[]) localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } try { NonAggregatingLogHandler.this.stateStore.removeLogDeleter( this.applicationId); } catch (IOException e) { LOG.error("Error removing log deletion state", e); } }
public DockerContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
static final FileContext getLfs() { try { return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
public DefaultContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
public BaseContainerManagerTest() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); localDir = new File("target", this.getClass().getSimpleName() + "-localDir") .getAbsoluteFile(); localLogDir = new File("target", this.getClass().getSimpleName() + "-localLogDir") .getAbsoluteFile(); remoteLogDir = new File("target", this.getClass().getSimpleName() + "-remoteLogDir") .getAbsoluteFile(); tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir"); }
private static final FileContext getLfs() { try { return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
@Before public void setup() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); tmpDir.mkdirs(); logsDir.mkdirs(); remoteLogsDir.mkdirs(); nmLocalDir.mkdirs(); // Construct the Container-id cId = createContainerId(); }
@Before public void setup() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); tmpDir.mkdirs(); logsDir.mkdirs(); remoteLogsDir.mkdirs(); nmLocalDir.mkdirs(); syncBarrier = new CyclicBarrier(2); }
public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical) { super(LocalContainerLauncher.class.getName()); this.context = context; this.umbilical = umbilical; // umbilical: MRAppMaster creates (taskAttemptListener), passes to us // (TODO/FIXME: pointless to use RPC to talk to self; should create // LocalTaskAttemptListener or similar: implement umbilical protocol // but skip RPC stuff) try { curFC = FileContext.getFileContext(curDir.toURI()); } catch (UnsupportedFileSystemException ufse) { LOG.error("Local filesystem " + curDir.toURI().toString() + " is unsupported?? (should never happen)"); } // Save list of files/dirs that are supposed to be present so can delete // any extras created by one task before starting subsequent task. Note // that there's no protection against deleted or renamed localization; // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). File[] curLocalFiles = curDir.listFiles(); localizedFiles = new HashSet<File>(curLocalFiles.length); for (int j = 0; j < curLocalFiles.length; ++j) { localizedFiles.add(curLocalFiles[j]); } // Relocalization note/future FIXME (per chrisdo, 20110315): At moment, // full localization info is in AppSubmissionContext passed from client to // RM and then to NM for AM-container launch: no difference between AM- // localization and MapTask- or ReduceTask-localization, so can assume all // OK. Longer-term, will need to override uber-AM container-localization // request ("needed resources") with union of regular-AM-resources + task- // resources (and, if maps and reduces ever differ, then union of all three // types), OR will need localizer service/API that uber-AM can request // after running (e.g., "localizeForTask()" or "localizeForMapTask()"). }
/** * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)} * but allowing injecting {@link ClientCache}. Enable mocking and testing. * @param conf the configuration object * @param resMgrDelegate the resource manager delegate * @param clientCache the client cache object. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, ClientCache clientCache) { this.conf = conf; try { this.resMgrDelegate = resMgrDelegate; this.clientCache = clientCache; this.defaultFileContext = FileContext.getFileContext(this.conf); } catch (UnsupportedFileSystemException ufe) { throw new RuntimeException("Error in instantiating YarnClient", ufe); } }
/** * Get default file system URI for the cluster (used to ensure consistency * of history done/staging locations) over different context * * @return Default file context */ private static FileContext getDefaultFileContext() { // If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore // ignore it. This prevents defaulting history paths to file system specified // by core-default.xml which would not make sense in any case. For a test // case to exploit this functionality it should create core-site.xml FileContext fc = null; Configuration defaultConf = new Configuration(); String[] sources; sources = defaultConf.getPropertySources( CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); if (sources != null && (!Arrays.asList(sources).contains("core-default.xml") || sources.length > 1)) { try { fc = FileContext.getFileContext(defaultConf); LOG.info("Default file system [" + fc.getDefaultFileSystem().getUri() + "]"); } catch (UnsupportedFileSystemException e) { LOG.error("Unable to create default file context [" + defaultConf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) + "]", e); } } else { LOG.info("Default file system is set solely " + "by core-default.xml therefore - ignoring"); } return fc; }
public HPCLogAggregateHandler(String applicationId, String user) { super(HPCLogAggregateHandler.class.getName()); this.applicationId = applicationId; this.user = user; try { this.lfs = FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
