/** * Upload part of a larger file. * * @param path destination path * @param partNumber item number in the path * @param inputStream input data * @param length length of the data * @throws IOException on a problem */ public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException { String stringPath = path.toUri().toString(); String partitionFilename = SwiftUtils.partitionFilenameFromNumber( partNumber); if (stringPath.endsWith("/")) { stringPath = stringPath.concat(partitionFilename); } else { stringPath = stringPath.concat("/").concat(partitionFilename); } swiftRestClient.upload( new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length); }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testChildOfProbe() throws Throwable { SwiftObjectPath parent = new SwiftObjectPath("container", "/parent"); SwiftObjectPath parent2 = new SwiftObjectPath("container", "/parent2"); SwiftObjectPath child = new SwiftObjectPath("container", "/parent/child"); SwiftObjectPath sibling = new SwiftObjectPath("container", "/parent/sibling"); SwiftObjectPath grandchild = new SwiftObjectPath("container", "/parent/child/grandchild"); assertParentOf(parent, child); assertParentOf(parent, grandchild); assertParentOf(child, grandchild); assertParentOf(parent, parent); assertNotParentOf(child, parent); assertParentOf(child, child); assertNotParentOf(parent, parent2); assertNotParentOf(grandchild, parent); }
/** * Create the URI needed to query the location of an object * @param path object path to retrieve information about * @return the URI for the location operation * @throws SwiftException if the URI could not be constructed */ private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException { URI uri; String dataLocationURI = objectLocationURI.toString(); try { if (path.toString().startsWith("/")) { dataLocationURI = dataLocationURI.concat(path.toUriPath()); } else { dataLocationURI = dataLocationURI.concat("/").concat(path.toUriPath()); } uri = new URI(dataLocationURI); } catch (URISyntaxException e) { throw new SwiftException(e); } return uri; }
/** * Upload part of a larger file. * * @param path destination path * @param partNumber item number in the path * @param inputStream input data * @param length length of the data * @throws IOException on a problem */ public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException { String stringPath = path.toUri().getPath(); String partitionFilename = SwiftUtils.partitionFilenameFromNumber( partNumber); if (stringPath.endsWith("/")) { stringPath = stringPath.concat(partitionFilename); } else { stringPath = stringPath.concat("/").concat(partitionFilename); } swiftRestClient.upload( new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length); }
/** * deletes object from Swift * * @param status FileStatus to delete * @return true if the path was deleted by this specific operation. * @throws IOException on a failure */ public boolean deleteObject(FileStatus status) throws IOException { SwiftObjectPath swiftObjectPath; if (status.isDir()) { swiftObjectPath = toDirPath(status.getPath()); } else { swiftObjectPath = toObjectPath(status.getPath()); } if (!SwiftUtils.isRootDir(swiftObjectPath)) { return swiftRestClient.delete(swiftObjectPath); } else { if (LOG.isDebugEnabled()) { LOG.debug("Not deleting root directory entry"); } return true; } }
/** * List all segments in dynamic large object. * * @param file SwiftFileStatus of large object * @param newest ask for the newest, or can some out of date data work? * @return the file statuses, or an empty array if there are no segments * @throws IOException on IO problems */ public FileStatus[] listSegments(FileStatus file, boolean newest) throws IOException { SwiftObjectPath prefix = ((SwiftFileStatus)file).getDLOPrefix(); if (prefix == null) { return new FileStatus[0]; } final List<FileStatus> objects; objects = listDirectory(prefix, true, newest, false); final ArrayList<FileStatus> segments; segments = new ArrayList<FileStatus>(objects.size()); for (FileStatus status : objects) { if (!status.isDir()) { segments.add(status); } } return segments.toArray(new FileStatus[segments.size()]); }
private String[] getRawObjectNames() throws Exception { SwiftRestClient client; client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf()); SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/")); byte[] bytes = client.listDeepObjectsInDirectory(path, true, true); final CollectionType collectionType = JSONUtil.getJsonMapper(). getTypeFactory().constructCollectionType(List.class, SwiftObjectFileStatus.class); final List<SwiftObjectFileStatus> fileStatusList = JSONUtil.toObject(new String(bytes), collectionType); final ArrayList<String> objects = new ArrayList(); for (SwiftObjectFileStatus status : fileStatusList) { if (status.getName() != null) { objects.add(status.getName()); } else if (status.getSubdir() != null) { objects.add(status.getSubdir()); } } return objects.toArray(new String[objects.size()]); }
/** * Make an HTTP GET request to Swift to get a range of data in the object. * * @param path path to object * @param offset offset from file beginning * @param length file length * @return The input stream -which must be closed afterwards. * @throws IOException Problems * @throws SwiftException swift specific error * @throws FileNotFoundException path is not there */ public HttpBodyContent getData(SwiftObjectPath path, long offset, long length) throws IOException { if (offset < 0) { throw new SwiftException("Invalid offset: " + offset + " in getDataAsInputStream( path=" + path + ", offset=" + offset + ", length =" + length + ")"); } if (length <= 0) { throw new SwiftException("Invalid length: " + length + " in getDataAsInputStream( path="+ path + ", offset=" + offset + ", length ="+ length + ")"); } final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN, offset, offset + length - 1); if (LOG.isDebugEnabled()) { LOG.debug("getData:" + range); } return getData(path, new Header(HEADER_RANGE, range), SwiftRestClient.NEWEST); }
/** * Find objects in a directory * * @param path path prefix * @param requestHeaders optional request headers * @return byte[] file data or null if the object was not found * @throws IOException on IO Faults * @throws FileNotFoundException if nothing is at the end of the URI -that is, * the directory is empty */ public byte[] listDeepObjectsInDirectory(SwiftObjectPath path, boolean listDeep, final Header... requestHeaders) throws IOException { preRemoteCommand("listDeepObjectsInDirectory"); String endpoint = getEndpointURI().toString(); StringBuilder dataLocationURI = new StringBuilder(); dataLocationURI.append(endpoint); String object = path.getObject(); if (object.startsWith("/")) { object = object.substring(1); } if (!object.endsWith("/")) { object = object.concat("/"); } if (object.equals("/")) { object = ""; } dataLocationURI = dataLocationURI.append("/") .append(path.getContainer()) .append("/?prefix=") .append(object) .append("&format=json"); //in listing deep set param to false if (listDeep == false) { dataLocationURI.append("&delimiter=/"); } return findObjects(dataLocationURI.toString(), requestHeaders); }
/** * Create a container -if it already exists, do nothing * * @param containerName the container name * @throws IOException IO problems * @throws SwiftBadRequestException invalid container name * @throws SwiftInvalidResponseException error from the server */ public void createContainer(String containerName) throws IOException { SwiftObjectPath objectPath = new SwiftObjectPath(containerName, ""); try { //see if the data is there headRequest("createContainer", objectPath, NEWEST); } catch (FileNotFoundException ex) { int status = 0; try { status = putRequest(objectPath); } catch (FileNotFoundException e) { //triggered by a very bad container name. //re-insert the 404 result into the status status = SC_NOT_FOUND; } if (status == SC_BAD_REQUEST) { throw new SwiftBadRequestException( "Bad request -authentication failure or bad container name?", status, "PUT", null); } if (!isStatusCodeExpected(status, SC_OK, SC_CREATED, SC_ACCEPTED, SC_NO_CONTENT)) { throw new SwiftInvalidResponseException("Couldn't create container " + containerName + " for storing data in Swift." + " Try to create container " + containerName + " manually ", status, "PUT", null); } else { throw ex; } } }
/** * Converts Swift path to URI to make request. * This is public for unit testing * * @param path path to object * @param endpointURI damain url e.g. http://domain.com * @return valid URI for object * @throws SwiftException */ public static URI pathToURI(SwiftObjectPath path, URI endpointURI) throws SwiftException { checkNotNull(endpointURI, "Null Endpoint -client is not authenticated"); String dataLocationURI = endpointURI.toString(); try { dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath())); return new URI(dataLocationURI); } catch (URISyntaxException e) { throw new SwiftException("Failed to create URI from " + dataLocationURI, e); } }
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws IOException { Header[] headers; if (newest) { headers = swiftRestClient.headRequest("getObjectMetadata-newest", objectPath, SwiftRestClient.NEWEST); } else { headers = swiftRestClient.headRequest("getObjectMetadata", objectPath); } return headers; }
/** * deletes object from Swift * * @param path path to delete * @return true if the path was deleted by this specific operation. * @throws IOException on a failure */ public boolean deleteObject(Path path) throws IOException { SwiftObjectPath swiftObjectPath = toObjectPath(path); if (!SwiftUtils.isRootDir(swiftObjectPath)) { return swiftRestClient.delete(swiftObjectPath); } else { if (LOG.isDebugEnabled()) { LOG.debug("Not deleting root directory entry"); } return true; } }
/** * Does the object exist * * @param path swift object path * @return true if the metadata of an object could be retrieved * @throws IOException IO problems other than FileNotFound, which * is downgraded to an object does not exist return code */ public boolean objectExists(SwiftObjectPath path) throws IOException { try { Header[] headers = swiftRestClient.headRequest("objectExists", path, SwiftRestClient.NEWEST); //no headers is treated as a missing file return headers.length != 0; } catch (FileNotFoundException e) { return false; } }
/** * Debug action to dump directory statuses to the debug log * * @param message explanation * @param objectPath object path (can be null) * @param statuses listing output */ private void logDirectory(String message, SwiftObjectPath objectPath, Iterable<FileStatus> statuses) { if (LOG.isDebugEnabled()) { LOG.debug(message + ": listing of " + objectPath); for (FileStatus fileStatus : statuses) { LOG.debug(fileStatus.getPath()); } } }
/** * Copy an object then, if the copy worked, delete it. * If the copy failed, the source object is not deleted. * * @param srcObject source object path * @param destObject destination object path * @throws IOException IO problems */ private void copyThenDeleteObject(SwiftObjectPath srcObject, SwiftObjectPath destObject) throws IOException { //do the copy copyObject(srcObject, destObject); //getting here means the copy worked swiftRestClient.delete(srcObject); }
/** * Copy an object * @param srcObject source object path * @param destObject destination object path * @throws IOException IO problems */ private void copyObject(SwiftObjectPath srcObject, SwiftObjectPath destObject) throws IOException { if (srcObject.isEqualToOrParentOf(destObject)) { throw new SwiftException( "Can't copy " + srcObject + " onto " + destObject); } //do the copy boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject); if (!copySucceeded) { throw new SwiftException("Copy of " + srcObject + " to " + destObject + "failed"); } }
/** * Builds a hadoop-Path from a swift path, inserting the URI authority * of this FS instance * @param path swift object path * @return Hadoop path * @throws SwiftException if the URI couldn't be created. */ private Path getCorrectSwiftPath(SwiftObjectPath path) throws SwiftException { try { final URI fullUri = new URI(uri.getScheme(), uri.getAuthority(), path.getObject(), null, null); return new Path(fullUri); } catch (URISyntaxException e) { throw new SwiftException("Specified path " + path + " is incorrect", e); } }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testPutAndDelete() throws Throwable { assumeEnabled(); SwiftRestClient client = createClient(); client.authenticate(); Path path = new Path("restTestPutAndDelete"); SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path); byte[] stuff = new byte[1]; stuff[0] = 'a'; client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length); //check file exists Duration head = new Duration(); Header[] responseHeaders = client.headRequest("expect success", sobject, SwiftRestClient.NEWEST); head.finished(); LOG.info("head request duration " + head); for (Header header: responseHeaders) { LOG.info(header.toString()); } //delete the file client.delete(sobject); //check file is gone try { Header[] headers = client.headRequest("expect fail", sobject, SwiftRestClient.NEWEST); Assert.fail("Expected deleted file, but object is still present: " + sobject); } catch (FileNotFoundException e) { //expected } for (DurationStats stats: client.getOperationStatistics()) { LOG.info(stats); } }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParsePath() throws Exception { final String pathString = "/home/user/files/file1"; final Path path = new Path(pathString); final URI uri = new URI("http://container.localhost"); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), pathString); assertEquals(expected, actual); }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParseUrlPath() throws Exception { final String pathString = "swift://container.service1/home/user/files/file1"; final URI uri = new URI(pathString); final Path path = new Path(pathString); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), "/home/user/files/file1"); assertEquals(expected, actual); }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testHandleUrlAsPath() throws Exception { final String hostPart = "swift://container.service1"; final String pathPart = "/home/user/files/file1"; final String uriString = hostPart + pathPart; final SwiftObjectPath expected = new SwiftObjectPath(uriString, pathPart); final SwiftObjectPath actual = new SwiftObjectPath(uriString, uriString); assertEquals(expected, actual); }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParseAuthenticatedUrl() throws Exception { final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx"; final URI uri = new URI(pathString); final Path path = new Path(pathString); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), "/home/tom/documents/finance.docx"); assertEquals(expected, actual); }
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testConvertToPath() throws Throwable { String initialpath = "/dir/file1"; Path ipath = new Path(initialpath); SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath), ipath); URI endpoint = new URI(ENDPOINT); URI uri = SwiftRestClient.pathToURI(objectPath, endpoint); LOG.info("Inital Hadoop Path =" + initialpath); LOG.info("Merged URI=" + uri); }
