Java 类com.amazonaws.services.s3.model.S3ObjectSummary 实例源码

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

private boolean isUploadFile(Iterator<S3ObjectSummary> iter, String path, String hash) {
    while (iter.hasNext()) {
        S3ObjectSummary fileS3 = iter.next();
        // Filename should look like this:
        // a/b
        if (!fileS3.getKey().equals(path)) {
            // If this is another file, then continue!
            continue;
        }
        // Remove the file from the S3 list as it does not need to be processed further
        iter.remove();
        // Upload if the hashes differ
        return StringUtils.isNullOrEmpty(hash) || !fileS3.getETag().equals(hash);
    }
    return true;
}

@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

  Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
  Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

  return objectStream.map(p -> {
    Integer version = getVersionIfMatch(p.getKey());
    if (version == null) {
      return null;
    }
    return new Pair<>(version, p);
  }).filter(Objects::nonNull)
      .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst())
            .reversed()).map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}

@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}

private boolean processObjects(List<S3ObjectSummary> objects) {
    Logger.Debug("Scanning next batch of %s ", objects.size());
    objects
        .parallelStream()
        .filter(this::shouldEnqueue)
        .forEach(object -> {
            numSeen.incrementAndGet();
            String path = object.getBucketName() + "/" + object.getKey();
            Logger.Info("Posting: %s", path);

            SendMessageRequest msg = new SendMessageRequest()
                .withQueueUrl(queueUrl)
                .withMessageBody(path);
            sqs.sendMessage(msg);
        });
    if (max > -1L && numSeen.incrementAndGet() > max) {
        Logger.Info("Added max jobs, quitting");
        return false;
    }
    return true;
}

@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
    List<AwsFileMiniModel> files = new ArrayList();

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    ObjectListing objectListing;

    do {
        objectListing = s3client.listObjects(listObjectsRequest);

        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
                    + " (date = " + objectSummary.getLastModified() + ")");
            files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
        }
        listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());

    return files;

}

/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}

static void deletePrefix(AmazonS3 s3Client, String bucketName, String prefix) {
    List<DeleteObjectsRequest.KeyVersion> trashKeys = new ArrayList<>();
    DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, prefix)) {
        trashKeys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        if (trashKeys.size() == BULK_DELETE_SIZE) {
            deleteObjectsRequest.setKeys(trashKeys);
            s3Client.deleteObjects(deleteObjectsRequest);
            trashKeys.clear();
        }
    }
    if (!trashKeys.isEmpty()) {
        deleteObjectsRequest.setKeys(trashKeys);
        s3Client.deleteObjects(deleteObjectsRequest);
    }
}

@Override
protected void copy(RuntimeExecutionTrace source, RuntimeExecutionTrace target,
        RuntimeAnnotatedExecutionTrace absoluteSource, RuntimeAnnotatedExecutionTrace absoluteTarget)
        throws IOException {
    String sourcePrefix = toS3Path(source).getPrefixForChildren();
    String targetPrefix = toS3Path(target).getPrefixForChildren();
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, sourcePrefix)) {
        if (summary.getKey().startsWith(sourcePrefix)) {
            String relativeKey = summary.getKey().substring(sourcePrefix.length());
            s3Client.copyObject(bucketName, summary.getKey(), bucketName, targetPrefix + relativeKey);
        } else {
            log.error(String.format(
                "S3Objects.withPrefix() returned unexpected key '%s' when asked for prefix '%s'.",
                summary.getKey(), sourcePrefix
            ));
        }
    }
}

@Override
protected void tearDown() throws Exception {
    final File theFile = new File(TEST_FILE_NAME);
    theFile.delete();

    if (s3Account != null) {
        final AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(s3Account.keyId, s3Account.secretKey));
        String hash = s3Account.getHashedName();
        if (s3.doesBucketExist(hash)) {
            for (S3ObjectSummary summary : s3.listObjects(hash).getObjectSummaries()) {
                s3.deleteObject(s3Account.getHashedName(), summary.getKey());
            }

        }
    }

    super.tearDown();
}

/**
 * Gets the metadata for a single table in this stash.  This is similar to getting the splits for the table
 * except that it exposes lower level information about the underlying S3 files.  For clients who will use
 * their own system for reading the files from S3, such as source files for a map-reduce job, this method provides
 * the necessary information.  For simply iterating over the stash contents using either {@link #scan(String)}
 * or {@link #getSplits(String)} in conjunction with {@link #getSplit(StashSplit)} is preferred.
 */
public StashTableMetadata getTableMetadata(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashFileMetadata> filesBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        filesBuilder.add(new StashFileMetadata(_bucket, objectSummary.getKey(), objectSummary.getSize()));
    }

    List<StashFileMetadata> files = filesBuilder.build();

    // Get the prefix arbitrarily from the first file.
    String prefix = files.get(0).getKey();
    prefix = prefix.substring(0, prefix.lastIndexOf('/') + 1);

    return new StashTableMetadata(_bucket, prefix, table, files);
}

private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}

@Test
public void testCompressFile() throws Exception {
  String avroCodec = "snappy";
  localProps.put(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG, avroCodec);
  setUp();
  task = new S3SinkTask(connectorConfig, context, storage, partitioner, format, SYSTEM_TIME);

  List<SinkRecord> sinkRecords = createRecords(7);
  // Perform write
  task.put(sinkRecords);
  task.close(context.assignment());
  task.stop();

  List<S3ObjectSummary> summaries = listObjects(S3_TEST_BUCKET_NAME, "/", s3);
  for(S3ObjectSummary summary: summaries){
    InputStream in = s3.getObject(summary.getBucketName(), summary.getKey()).getObjectContent();
    DatumReader<Object> reader = new GenericDatumReader<>();
    DataFileStream<Object> streamReader = new DataFileStream<>(in, reader);
    // make sure that produced Avro file has proper codec set
    Assert.assertEquals(avroCodec, streamReader.getMetaString(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG));
    streamReader.close();
  }

  long[] validOffsets = {0, 3, 6};
  verify(sinkRecords, validOffsets);
}

public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}

@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}

@Override
public Iterable<BinaryKey> getAllBinaryKeys() throws BinaryStoreException {
    try {
        final Iterator<S3ObjectSummary> objectsIterator =
            S3Objects.inBucket(s3Client, bucketName).iterator();
        // Lambda to hand back BinaryKeys rather than S3ObjectSummaries
        return () -> {
            return new Iterator<BinaryKey>() {
                @Override
                public boolean hasNext() {
                    return objectsIterator.hasNext();
                }

                @Override
                public BinaryKey next() {
                    S3ObjectSummary object = objectsIterator.next();
                    return new BinaryKey(object.getKey());
                }
            };
        };
    } catch (AmazonClientException e) {
        throw new BinaryStoreException(e);
    }
}

public static void main(String[] args) throws Exception {
    // create the AWS S3 Client
AmazonS3 s3 = AWSS3Factory.getS3Client();

      // delete the demo bucket and all its content
for (S3ObjectSummary summary : s3.listObjects(AWSS3Factory.S3_BUCKET).getObjectSummaries())
{
    s3.deleteObject(AWSS3Factory.S3_BUCKET, summary.getKey());
    System.out.println(String.format("Deleted [%s/%s]", AWSS3Factory.S3_BUCKET, summary.getKey()));
}
s3.deleteBucket(AWSS3Factory.S3_BUCKET);

                // print bucket key/value and content for validation
    System.out.println( String.format("deleted bucket [%s]",
            AWSS3Factory.S3_BUCKET));
  }

public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}

@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}

private void subscribe(S3ObjectSummary os, Observable<? extends Object> result) {

        result.subscribe(new Subscriber<Object>() {
            @Override
            public void onCompleted() {
                if (s3ObjectsInPipeline.remove(os)) {
                    processingState.completed(os);
                    release();
                }
            }

            @Override
            public void onError(Throwable e) {
                if (s3ObjectsInPipeline.remove(os)) {
                    release();
                }

                if (LOGGER.isTraceEnabled()) e.printStackTrace();
            }

            @Override
            public void onNext(Object event) { }
        });

    }

@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();

    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }

    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);

    return response;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();

    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }

    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);

    return response;
}

@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}

public List<String> listKeysInOutputBucket(String bucketName) {
    List<String> videoKeys = new ArrayList<>();

    ListObjectsRequest listFirstLevelKeyRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withDelimiter("/");

    for (String commonPrefix : s3client.listObjects(listFirstLevelKeyRequest).getCommonPrefixes()) {

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(commonPrefix)
        .withDelimiter("/");

        for (S3ObjectSummary file : s3client.listObjects(listObjectsRequest).getObjectSummaries()) {
            videoKeys.add(file.getKey());
        }
    }
    return videoKeys;       
}

@Test
public void testListBucket() {
    final String dataBucket = "noaa-nexrad-level2";
    String[] key = {"2010/01/01", "2010/07/14"};
    String jobID = null; 
    AWSInterface awsInterface = new AWSInterface(jobID, null); 
    AWSAnonInterface awsAnonInterface = new AWSAnonInterface();
    List<S3ObjectSummary> summaries; 
    int[] output = new int[2];
    for(int i = 0 ; i < 2; i++) {
        summaries = awsAnonInterface.ListBucket(dataBucket, key[i]);
        output[i] = summaries.size();
        summaries.clear();
    }
    int[] answer = {14104, 33468};
    assertArrayEquals(answer, output);
    System.out.println("ListBucket() is ok");
}

@Test
public void testValidateCopiedS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}

@Test
public void testValidateCopiedS3FilesActualFileSizeMismatch() throws IOException
{
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));

    // Try to validate S3 files when expected file size does not match actual file size.
    try
    {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
            new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE,
                SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Specified file size of %d bytes for copied \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.",
                FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}

@Test
public void testValidateRegisteredS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}

@Test
public void testValidateRegisteredS3FilesUnexpectedS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected S3 file exists.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating registered S3 files. Business object data {%s}", TARGET_S3_KEY,
                STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}

@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Arrays.asList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}

@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
    ObjectListing listing = s3.listObjects(
            new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

    S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
    if (summary == null) {
        return Record.noFile(uri, path);
    }

    long time = summary.getLastModified().getTime();
    long size = summary.getSize();
    boolean directory = summary.getKey().endsWith("/");
    return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}

@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}

public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}

public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}