private static void validateDestination( String src, String dst, INode srcInode) throws IOException { String error; if (srcInode.isSymlink() && dst.equals(srcInode.asSymlink().getSymlinkString())) { throw new FileAlreadyExistsException("Cannot rename symlink " + src + " to its target " + dst); } // dst cannot be a directory or a file under src if (dst.startsWith(src) && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) { error = "Rename destination " + dst + " is a directory or file under source " + src; NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new IOException(error); } }
@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res; try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { if (createParent) { throw readOnlyMountTable("create", f); } else { throw e; } } assert(res.remainingPath != null); return res.targetFileSystem.createInternal(res.remainingPath, flag, absolutePermission, bufferSize, replication, blockSize, progress, checksumOpt, createParent); }
@Test public void testCreateFileOverExistingFileNoOverwrite() throws Throwable { describe("Verify overwriting an existing file fails"); Path path = path("testCreateFileOverExistingFileNoOverwrite"); byte[] data = dataset(256, 'a', 'z'); writeDataset(getFileSystem(), path, data, data.length, 1024, false); byte[] data2 = dataset(10 * 1024, 'A', 'Z'); try { writeDataset(getFileSystem(), path, data2, data2.length, 1024, false); fail("writing without overwrite unexpectedly succeeded"); } catch (FileAlreadyExistsException expected) { //expected handleExpectedException(expected); } catch (IOException relaxed) { handleRelaxedException("Creating a file over a file with overwrite==false", "FileAlreadyExistsException", relaxed); } }
/** * Create or update an entry * @param path path * @param data data * @param acl ACL for path -used when creating a new entry * @param overwrite enable overwrite * @throws IOException * @return true if the entry was created, false if it was simply updated. */ public boolean zkSet(String path, CreateMode mode, byte[] data, List<ACL> acl, boolean overwrite) throws IOException { Preconditions.checkArgument(data != null, "null data"); checkServiceLive(); if (!zkPathExists(path)) { zkCreate(path, mode, data, acl); return true; } else { if (overwrite) { zkUpdate(path, data); return false; } else { throw new FileAlreadyExistsException(path); } } }
@Test public void testOverwrite() throws Throwable { ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0); ServiceRecord resolved1 = operations.resolve(ENTRY_PATH); resolved1.description = "resolved1"; try { operations.bind(ENTRY_PATH, resolved1, 0); fail("overwrite succeeded when it should have failed"); } catch (FileAlreadyExistsException expected) { // expected } // verify there's no changed ServiceRecord resolved2 = operations.resolve(ENTRY_PATH); assertMatches(written, resolved2); operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE); ServiceRecord resolved3 = operations.resolve(ENTRY_PATH); assertMatches(resolved1, resolved3); }
private void mkdir(FileSystem fs, Path path, FsPermission fsp) throws IOException { if (!fs.exists(path)) { try { fs.mkdirs(path, fsp); FileStatus fsStatus = fs.getFileStatus(path); LOG.info("Perms after creating " + fsStatus.getPermission().toShort() + ", Expected: " + fsp.toShort()); if (fsStatus.getPermission().toShort() != fsp.toShort()) { LOG.info("Explicitly setting permissions to : " + fsp.toShort() + ", " + fsp); fs.setPermission(path, fsp); } } catch (FileAlreadyExistsException e) { LOG.info("Directory: [" + path + "] already exists."); } } }
private void mkdir(FileContext fc, Path path, FsPermission fsp) throws IOException { if (!fc.util().exists(path)) { try { fc.mkdir(path, fsp, true); FileStatus fsStatus = fc.getFileStatus(path); LOG.info("Perms after creating " + fsStatus.getPermission().toShort() + ", Expected: " + fsp.toShort()); if (fsStatus.getPermission().toShort() != fsp.toShort()) { LOG.info("Explicitly setting permissions to : " + fsp.toShort() + ", " + fsp); fc.setPermission(path, fsp); } } catch (FileAlreadyExistsException e) { LOG.info("Directory: [" + path + "] already exists."); } } }
private void makeDoneSubdir(Path path) throws IOException { try { doneDirFc.getFileStatus(path); existingDoneSubdirs.add(path); } catch (FileNotFoundException fnfE) { try { FsPermission fsp = new FsPermission( JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION); doneDirFc.mkdir(path, fsp, true); FileStatus fsStatus = doneDirFc.getFileStatus(path); LOG.info("Perms after creating " + fsStatus.getPermission().toShort() + ", Expected: " + fsp.toShort()); if (fsStatus.getPermission().toShort() != fsp.toShort()) { LOG.info("Explicitly setting permissions to : " + fsp.toShort() + ", " + fsp); doneDirFc.setPermission(path, fsp); } existingDoneSubdirs.add(path); } catch (FileAlreadyExistsException faeE) { // Nothing to do. } } }
/** * Creates a symbolic link. * * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean) */ public void createSymlink(String target, String link, boolean createParent) throws IOException { TraceScope scope = getPathTraceScope("createSymlink", target); try { FsPermission dirPerm = FsPermission.getDefault().applyUMask(dfsClientConf.uMask); namenode.createSymlink(target, link, dirPerm, createParent); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, NSQuotaExceededException.class, DSQuotaExceededException.class, UnresolvedPathException.class, SnapshotAccessControlException.class); } finally { scope.close(); } }
/** * Rename file or directory. * @see ClientProtocol#rename2(String, String, Options.Rename...) */ public void rename(String src, String dst, Options.Rename... options) throws IOException { checkOpen(); TraceScope scope = getSrcDstTraceScope("rename2", src, dst); try { namenode.rename2(src, dst, options); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, SafeModeException.class, NSQuotaExceededException.class, UnresolvedPathException.class, SnapshotAccessControlException.class); } finally { scope.close(); } }
/** * Create a new file entry in the namespace. * * For description of parameters and exceptions thrown see * {@link ClientProtocol#create}, except it returns valid file status upon * success */ HdfsFileStatus startFile(String src, PermissionStatus permissions, String holder, String clientMachine, EnumSet<CreateFlag> flag, boolean createParent, short replication, long blockSize, CryptoProtocolVersion[] supportedVersions, boolean logRetryCache) throws AccessControlException, SafeModeException, FileAlreadyExistsException, UnresolvedLinkException, FileNotFoundException, ParentNotDirectoryException, IOException { HdfsFileStatus status = null; try { status = startFileInt(src, permissions, holder, clientMachine, flag, createParent, replication, blockSize, supportedVersions, logRetryCache); } catch (AccessControlException e) { logAuditEvent(false, "create", src); throw e; } return status; }
/** * create a directory at path specified by parent */ private static INodesInPath unprotectedMkdir(FSDirectory fsd, long inodeId, INodesInPath parent, byte[] name, PermissionStatus permission, List<AclEntry> aclEntries, long timestamp) throws QuotaExceededException, AclException, FileAlreadyExistsException { assert fsd.hasWriteLock(); assert parent.getLastINode() != null; if (!parent.getLastINode().isDirectory()) { throw new FileAlreadyExistsException("Parent path is not a directory: " + parent.getPath() + " " + DFSUtil.bytes2String(name)); } final INodeDirectory dir = new INodeDirectory(inodeId, name, permission, timestamp); INodesInPath iip = fsd.addLastINode(parent, dir, true); if (iip != null && aclEntries != null) { AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID); } return iip; }
@Override public void rename2(String src, String dst, Rename... options) throws AccessControlException, DSQuotaExceededException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, IOException { boolean overwrite = false; if (options != null) { for (Rename option : options) { if (option == Rename.OVERWRITE) { overwrite = true; } } } Rename2RequestProto req = Rename2RequestProto.newBuilder(). setSrc(src). setDst(dst).setOverwriteDest(overwrite). build(); try { rpcProxy.rename2(null, req); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Override public boolean mkdirs(String src, FsPermission masked, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, IOException { MkdirsRequestProto req = MkdirsRequestProto.newBuilder() .setSrc(src) .setMasked(PBHelper.convert(masked)) .setCreateParent(createParent).build(); try { return rpcProxy.mkdirs(null, req).getResult(); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Override public void createSymlink(String target, String link, FsPermission dirPerm, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, IOException { CreateSymlinkRequestProto req = CreateSymlinkRequestProto.newBuilder() .setTarget(target) .setLink(link) .setDirPerm(PBHelper.convert(dirPerm)) .setCreateParent(createParent) .build(); try { rpcProxy.createSymlink(null, req); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if (exists(f) && !overwrite) { throw new FileAlreadyExistsException("File already exists: " + f); } if(LOG.isDebugEnabled()) { LOG.debug("Creating new file '" + f + "' in S3"); } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); return new FSDataOutputStream(new NativeS3FsOutputStream(getConf(), store, key, progress, bufferSize), statistics); }
private boolean mkdir(Path f) throws IOException { try { FileStatus fileStatus = getFileStatus(f); if (fileStatus.isFile()) { throw new FileAlreadyExistsException(String.format( "Can't make directory for path '%s' since it is a file.", f)); } } catch (FileNotFoundException e) { if(LOG.isDebugEnabled()) { LOG.debug("Making dir '" + f + "' in S3"); } String key = pathToKey(f) + FOLDER_SUFFIX; store.storeEmptyFile(key); } return true; }
/** * @param permission Currently ignored. */ @Override public FSDataOutputStream create(Path file, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { INode inode = store.retrieveINode(makeAbsolute(file)); if (inode != null) { if (overwrite) { delete(file, true); } else { throw new FileAlreadyExistsException("File already exists: " + file); } } else { Path parent = file.getParent(); if (parent != null) { if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } } } return new FSDataOutputStream (new S3OutputStream(getConf(), store, makeAbsolute(file), blockSize, progress, bufferSize), statistics); }
/** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * @param f the file name to open * @param permission * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { String key = pathToKey(f); if (!overwrite && exists(f)) { throw new FileAlreadyExistsException(f + " already exists"); } if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) { return new FSDataOutputStream(new S3AFastOutputStream(s3, this, bucket, key, progress, statistics, cannedACL, serverSideEncryptionAlgorithm, partSize, (long)multiPartThreshold, threadPoolExecutor), statistics); } // We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file return new FSDataOutputStream(new S3AOutputStream(getConf(), transfers, this, bucket, key, progress, cannedACL, statistics, serverSideEncryptionAlgorithm), null); }
/** * Creates a symbolic link. * * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean) */ public void createSymlink(String target, String link, boolean createParent) throws IOException { checkOpen(); try (TraceScope ignored = newPathTraceScope("createSymlink", target)) { final FsPermission dirPerm = applyUMask(null); namenode.createSymlink(target, link, dirPerm, createParent); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, NSQuotaExceededException.class, DSQuotaExceededException.class, QuotaByStorageTypeExceededException.class, UnresolvedPathException.class, SnapshotAccessControlException.class); } }
/** * Rename file or directory. * @see ClientProtocol#rename2(String, String, Options.Rename...) */ public void rename(String src, String dst, Options.Rename... options) throws IOException { checkOpen(); try (TraceScope ignored = newSrcDstTraceScope("rename2", src, dst)) { namenode.rename2(src, dst, options); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, QuotaByStorageTypeExceededException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, SafeModeException.class, NSQuotaExceededException.class, UnresolvedPathException.class, SnapshotAccessControlException.class); } }
/** * Same {{@link #mkdirs(String, FsPermission, boolean)} except * that the permissions has already been masked against umask. */ public boolean primitiveMkdir(String src, FsPermission absPermission, boolean createParent) throws IOException { checkOpen(); if (absPermission == null) { absPermission = applyUMask(null); } LOG.debug("{}: masked={}", src, absPermission); try (TraceScope ignored = tracer.newScope("mkdir")) { return namenode.mkdirs(src, absPermission, createParent); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, InvalidPathException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, SafeModeException.class, NSQuotaExceededException.class, DSQuotaExceededException.class, QuotaByStorageTypeExceededException.class, UnresolvedPathException.class, SnapshotAccessControlException.class); } }
private static void validateDestination( String src, String dst, INode srcInode) throws IOException { String error; if (srcInode.isSymlink() && dst.equals(srcInode.asSymlink().getSymlinkString())) { throw new FileAlreadyExistsException("Cannot rename symlink " + src + " to its target " + dst); } // dst cannot be a directory or a file under src if (dst.startsWith(src) && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) { error = "Rename destination " + dst + " is a directory or file under source " + src; NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new IOException(error); } if (FSDirectory.isExactReservedName(src) || FSDirectory.isExactReservedName(dst)) { error = "Cannot rename to or from /.reserved"; throw new InvalidPathException(error); } }
/** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * @param f the file name to open * @param permission * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { String key = pathToKey(f); if (!overwrite && exists(f)) { throw new FileAlreadyExistsException(f + " already exists"); } if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) { return new FSDataOutputStream(new S3AFastOutputStream(s3, this, bucket, key, progress, statistics, cannedACL, serverSideEncryptionAlgorithm, partSize, multiPartThreshold, threadPoolExecutor), statistics); } // We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file return new FSDataOutputStream(new S3AOutputStream(getConf(), transfers, this, bucket, key, progress, cannedACL, statistics, serverSideEncryptionAlgorithm), null); }
/** * Tests CreateOperation with fencing resolution for lower-epoch creation. */ @Test public void testLowerEpochFencedOut() throws Exception { @Cleanup val fs = new MockFileSystem(); val context1 = newContext(1, fs); val context2 = newContext(2, fs); // This should wipe out the file created by the first call. new CreateOperation(SEGMENT_NAME, context2).call(); // This should fail because we attempt to create the segment using a lower epoch. AssertExtensions.assertThrows( "Lower epoch segment creation did not fail.", new CreateOperation(SEGMENT_NAME, context1)::call, ex -> ex instanceof StorageNotPrimaryException || ex instanceof FileAlreadyExistsException); }
