Java 类com.amazonaws.services.s3.model.ListMultipartUploadsRequest 实例源码

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest)
        throws SdkClientException, AmazonServiceException {
    listMultipartUploadsRequest = beforeClientExecution(listMultipartUploadsRequest);
    rejectNull(listMultipartUploadsRequest,
        "The request parameter must be specified when listing multipart uploads");

    rejectNull(listMultipartUploadsRequest.getBucketName(),
        "The bucket name parameter must be specified when listing multipart uploads");

    Request<ListMultipartUploadsRequest> request = createRequest(listMultipartUploadsRequest.getBucketName(), null, listMultipartUploadsRequest, HttpMethodName.GET);
    request.addParameter("uploads", null);

    if (listMultipartUploadsRequest.getKeyMarker() != null) request.addParameter("key-marker", listMultipartUploadsRequest.getKeyMarker());
    if (listMultipartUploadsRequest.getMaxUploads() != null) request.addParameter("max-uploads", listMultipartUploadsRequest.getMaxUploads().toString());
    if (listMultipartUploadsRequest.getUploadIdMarker() != null) request.addParameter("upload-id-marker", listMultipartUploadsRequest.getUploadIdMarker());
    if (listMultipartUploadsRequest.getDelimiter() != null) request.addParameter("delimiter", listMultipartUploadsRequest.getDelimiter());
    if (listMultipartUploadsRequest.getPrefix() != null) request.addParameter("prefix", listMultipartUploadsRequest.getPrefix());
    if (listMultipartUploadsRequest.getEncodingType() != null) request.addParameter("encoding-type", listMultipartUploadsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListMultipartUploadsResultUnmarshaller(), listMultipartUploadsRequest.getBucketName(), null);
}

/**
 * <p>
 * Aborts any multipart uploads that were initiated before the specified date.
 * </p>
 * <p>
 * This method is useful for cleaning up any interrupted multipart uploads.
 * <code>TransferManager</code> attempts to abort any failed uploads,
 * but in some cases this may not be possible, such as if network connectivity
 * is completely lost.
 * </p>
 *
 * @param bucketName
 *            The name of the bucket containing the multipart uploads to
 *            abort.
 * @param date
 *            The date indicating which multipart uploads should be aborted.
 */
public void abortMultipartUploads(String bucketName, Date date)
        throws AmazonServiceException, AmazonClientException {
    MultipartUploadListing uploadListing = s3.listMultipartUploads(appendSingleObjectUserAgent(
            new ListMultipartUploadsRequest(bucketName)));
    do {
        for (MultipartUpload upload : uploadListing.getMultipartUploads()) {
            if (upload.getInitiated().compareTo(date) < 0) {
                s3.abortMultipartUpload(appendSingleObjectUserAgent(new AbortMultipartUploadRequest(
                        bucketName, upload.getKey(), upload.getUploadId())));
            }
        }

        ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(bucketName)
            .withUploadIdMarker(uploadListing.getNextUploadIdMarker())
            .withKeyMarker(uploadListing.getNextKeyMarker());
        uploadListing = s3.listMultipartUploads(appendSingleObjectUserAgent(request));
    } while (uploadListing.isTruncated());
}

/**
 * Deletes all existing buckets
 */
@After
public void cleanupFilestore() {
  for (final Bucket bucket : s3Client.listBuckets()) {
    if (!INITIAL_BUCKET_NAMES.contains(bucket.getName())) {
      s3Client.listMultipartUploads(new ListMultipartUploadsRequest(bucket.getName()))
          .getMultipartUploads()
          .forEach(upload ->
              s3Client.abortMultipartUpload(
                  new AbortMultipartUploadRequest(bucket.getName(), upload.getKey(),
                      upload.getUploadId()))
          );
      s3Client.deleteBucket(bucket.getName());
    }
  }
}

/**
 * Tests if not yet completed / aborted multipart uploads are listed.
 *
 * @throws Exception not expected
 */
@Test
public void shouldListMultipartUploads() throws Exception {
  s3Client.createBucket(BUCKET_NAME);

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));

  final InitiateMultipartUploadResult initiateMultipartUploadResult = s3Client
      .initiateMultipartUpload(new InitiateMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME));
  final String uploadId = initiateMultipartUploadResult.getUploadId();

  final MultipartUploadListing listing =
      s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME));
  assertThat(listing.getMultipartUploads(), is(not(empty())));
  assertThat(listing.getBucketName(), equalTo(BUCKET_NAME));
  assertThat(listing.getMultipartUploads(), hasSize(1));
  final MultipartUpload upload = listing.getMultipartUploads().get(0);
  assertThat(upload.getUploadId(), equalTo(uploadId));
  assertThat(upload.getKey(), equalTo(UPLOAD_FILE_NAME));
}

/**
 * Tests if a multipart upload can be aborted.
 *
 * @throws Exception not expected
 */
@Test
public void shouldAbortMultipartUpload() throws Exception {
  s3Client.createBucket(BUCKET_NAME);

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));

  final InitiateMultipartUploadResult initiateMultipartUploadResult = s3Client
      .initiateMultipartUpload(new InitiateMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME));
  final String uploadId = initiateMultipartUploadResult.getUploadId();

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(not(empty())));

  s3Client.abortMultipartUpload(
      new AbortMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadId));

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));
}

private void abortAllMultipartUploads() {
    ListMultipartUploadsRequest multipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
    amazonS3.listMultipartUploads(multipartUploadsRequest)
            .getMultipartUploads()
            .forEach(upload -> {
                AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(bucketName, upload.getKey(), upload.getUploadId());
                amazonS3.abortMultipartUpload(request);
            });
}

@Test
public void should_have_permission_to_list_own_multipart_uploads() throws Exception {
    remoteTestBucket.givenMultipartUploadExists(TEST_VIDEO_STORAGE_BUCKET, TEST_USERNAME + "/upload.txt");
    ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(TEST_VIDEO_STORAGE_BUCKET);
    request.setPrefix(TEST_USERNAME);

    expectNoException();
    MultipartUploadListing multipartUploadListing = federatedS3Client.listMultipartUploads(request);
    assertThat(multipartUploadListing.getMultipartUploads().size(), is(1));
}

@Ignore("AWS bug. See: https://forums.aws.amazon.com/thread.jspa?threadID=158131")
@Test
public void should_not_have_permission_to_list_other_multipart_uploads() throws Exception {
    remoteTestBucket.givenMultipartUploadExists(TEST_VIDEO_STORAGE_BUCKET, OTHER_USERNAME + "/upload.txt");
    ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(TEST_VIDEO_STORAGE_BUCKET);
    request.setPrefix(OTHER_USERNAME);

    expectForbiddenException();
    federatedS3Client.listMultipartUploads(request);
}

/**
 * {@inheritDoc} <p/> <p> Since a multipart upload in progress does not exist when in-memory, this method simply returns a preconfigured list. </p> <p>
 * Returns a mock {@link MultipartUploadListing} based on the parameters and hints provided. By default returns a mock listing as defiend by {@link
 * #getMultipartUploadListing()}. </p> <p> This operation takes the following hints when suffixed in listMultipartUploadsRequest.bucketName: <dl> <p/>
 * <dt>MOCK_S3_BUCKET_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> <dt>MOCK_S3_BUCKET_NAME_TRUNCATED_MULTIPART_LISTING</dt>
 * <dd>Returns the listing as if it is truncated. See below for details.</dd> <p/> </dl> </p>
 */
@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest, AmazonS3 s3Client)
{
    if (listMultipartUploadsRequest.getBucketName().equals(MOCK_S3_BUCKET_NAME_SERVICE_EXCEPTION))
    {
        throw new AmazonServiceException(null);
    }
    else if (listMultipartUploadsRequest.getBucketName().equals(MOCK_S3_BUCKET_NAME_TRUNCATED_MULTIPART_LISTING))
    {
        MultipartUploadListing multipartUploadListing = getMultipartUploadListing();

        // If listing request does not have upload ID marker set, mark the listing as truncated - this is done to truncate the multipart listing just once.
        if (listMultipartUploadsRequest.getUploadIdMarker() == null)
        {
            multipartUploadListing.setNextUploadIdMarker("TEST_UPLOAD_MARKER_ID");
            multipartUploadListing.setNextKeyMarker("TEST_KEY_MARKER_ID");
            multipartUploadListing.setTruncated(true);
        }

        return multipartUploadListing;
    }
    else
    {
        return getMultipartUploadListing();
    }
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest request) throws AmazonClientException, AmazonServiceException {
    return delegate.listMultipartUploads(request);
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest request) throws SdkClientException, AmazonServiceException {
  return call(() -> getDelegate().listMultipartUploads(request));
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) throws AmazonClientException, AmazonServiceException {
    throw new UnsupportedOperationException();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest, AmazonS3 s3Client)
{
    return s3Client.listMultipartUploads(listMultipartUploadsRequest);
}

@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate)
{
    // Create an Amazon S3 client.
    AmazonS3Client s3Client = getAmazonS3(params);
    int abortedMultipartUploadsCount = 0;

    try
    {
        // List upload markers. Null implies initial list request.
        String uploadIdMarker = null;
        String keyMarker = null;

        boolean truncated;
        do
        {
            // Create the list multipart request, optionally using the last markers.
            ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
            request.setUploadIdMarker(uploadIdMarker);
            request.setKeyMarker(keyMarker);

            // Request the multipart upload listing.
            MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);

            for (MultipartUpload upload : uploadListing.getMultipartUploads())
            {
                if (upload.getInitiated().compareTo(thresholdDate) < 0)
                {
                    // Abort the upload.
                    s3Operations.abortMultipartUpload(TransferManager
                        .appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())),
                        s3Client);

                    // Log the information about the aborted multipart upload.
                    LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(),
                        params.getS3BucketName(), upload.getInitiated());

                    // Increment the counter.
                    abortedMultipartUploadsCount++;
                }
            }

            // Determine whether there are more uploads to list.
            truncated = uploadListing.isTruncated();
            if (truncated)
            {
                // Record the list markers.
                uploadIdMarker = uploadListing.getNextUploadIdMarker();
                keyMarker = uploadListing.getNextKeyMarker();
            }
        }
        while (truncated);
    }
    finally
    {
        // Shutdown the Amazon S3 client instance to release resources.
        s3Client.shutdown();
    }

    return abortedMultipartUploadsCount;
}

@Test
public void testAbortMultipartUploadsAssertTruncatedResult()
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    try
    {
        String s3BucketName = "s3BucketName";
        String uploadKey = "uploadKey";
        String uploadId = "uploadId";
        Date uploadInitiated = new Date(0);

        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        Date thresholdDate = new Date(1);

        when(mockS3Operations.listMultipartUploads(any(), any())).then(new Answer<MultipartUploadListing>()
        {
            @Override
            public MultipartUploadListing answer(InvocationOnMock invocation) throws Throwable
            {
                ListMultipartUploadsRequest listMultipartUploadsRequest = invocation.getArgument(0);
                String keyMarker = listMultipartUploadsRequest.getKeyMarker();
                String uploadIdMarker = listMultipartUploadsRequest.getUploadIdMarker();

                MultipartUploadListing multipartUploadListing = new MultipartUploadListing();
                if (keyMarker == null || uploadIdMarker == null)
                {
                    multipartUploadListing.setNextKeyMarker("nextKeyMarker");
                    multipartUploadListing.setNextUploadIdMarker("nextUploadIdMarker");
                    multipartUploadListing.setTruncated(true);
                }
                else
                {
                    assertEquals("nextKeyMarker", keyMarker);
                    assertEquals("nextUploadIdMarker", uploadIdMarker);

                    MultipartUpload multipartUpload = new MultipartUpload();
                    multipartUpload.setUploadId(uploadId);
                    multipartUpload.setKey(uploadKey);
                    multipartUpload.setInitiated(uploadInitiated);
                    multipartUploadListing.getMultipartUploads().add(multipartUpload);
                }
                return multipartUploadListing;
            }
        });

        assertEquals(1, s3Dao.abortMultipartUploads(s3FileTransferRequestParamsDto, thresholdDate));

        // Assert listMultipartUploads() is called twice due to truncation
        verify(mockS3Operations, times(2)).listMultipartUploads(any(), any());

        /*
         * Assert that S3Operations.abortMultipartUpload is called exactly ONCE with arguments matching the given ArgumentMatcher
         */
        verify(mockS3Operations).abortMultipartUpload(argThat(
            argument -> Objects.equal(s3BucketName, argument.getBucketName()) && Objects.equal(uploadKey, argument.getKey()) &&
                Objects.equal(uploadId, argument.getUploadId())), any());

        // Assert that no other interactions occur with the mock
        verifyNoMoreInteractions(mockS3Operations);
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest request)
        throws AmazonClientException
{
    return null;
}

@Test
public void testMultipartUploadAbort() throws Exception {
    String blobName = "multipart-upload-abort";
    ByteSource byteSource = TestUtils.randomByteSource().slice(
            0, context.getBlobStore().getMinimumMultipartPartSize());

    InitiateMultipartUploadResult result = client.initiateMultipartUpload(
            new InitiateMultipartUploadRequest(containerName, blobName));

    // TODO: google-cloud-storage and openstack-swift cannot list multipart
    // uploads
    MultipartUploadListing multipartListing = client.listMultipartUploads(
            new ListMultipartUploadsRequest(containerName));
    if (blobStoreType.equals("azureblob")) {
        // Azure does not create a manifest during initiate multi-part
        // upload.  Instead the first part creates this.
        assertThat(multipartListing.getMultipartUploads()).isEmpty();
    } else {
        assertThat(multipartListing.getMultipartUploads()).hasSize(1);
    }

    PartListing partListing = client.listParts(new ListPartsRequest(
            containerName, blobName, result.getUploadId()));
    assertThat(partListing.getParts()).isEmpty();

    client.uploadPart(new UploadPartRequest()
            .withBucketName(containerName)
            .withKey(blobName)
            .withUploadId(result.getUploadId())
            .withPartNumber(1)
            .withPartSize(byteSource.size())
            .withInputStream(byteSource.openStream()));

    multipartListing = client.listMultipartUploads(
            new ListMultipartUploadsRequest(containerName));
    assertThat(multipartListing.getMultipartUploads()).hasSize(1);

    partListing = client.listParts(new ListPartsRequest(
            containerName, blobName, result.getUploadId()));
    assertThat(partListing.getParts()).hasSize(1);

    client.abortMultipartUpload(new AbortMultipartUploadRequest(
            containerName, blobName, result.getUploadId()));

    multipartListing = client.listMultipartUploads(
            new ListMultipartUploadsRequest(containerName));
    if (blobStoreType.equals("azureblob")) {
        // Azure does not support explicit abort.  It automatically
        // removes incomplete multi-part uploads after 7 days.
        assertThat(multipartListing.getMultipartUploads()).hasSize(1);
    } else {
        assertThat(multipartListing.getMultipartUploads()).isEmpty();
    }

    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();
}

@Override
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest request) throws AmazonClientException,
    AmazonServiceException {
  // TODO Auto-generated method stub
  return null;
}

/**
 * Lists in-progress multipart uploads. An in-progress multipart upload is a multipart upload that has been initiated, but has not yet been completed or
 * aborted. This operation returns at most 1,000 multipart uploads in the response by default.
 *
 * @param listMultipartUploadsRequest the request object that specifies all the parameters of this operation
 * @param s3Client the {@link AmazonS3} implementation to use
 *
 * @return the listing of multipart uploads from Amazon S3
 */
public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest, AmazonS3 s3Client);