/** * Stores the {@link InputStream} as an object in the S3 bucket. * * @param keyName The requested key name for the object. * @param inStream The {@link InputStream} to write out to an object in S3. * @param size The size of the {@link InputStream}. * @return A {@link CompletableFuture} that will eventually contain the S3 object key. */ @Override public CompletableFuture<String> store(String keyName, InputStream inStream, long size) { final String bucketName = environment.getProperty(Constants.BUCKET_NAME_ENV_VARIABLE); final String kmsKey = environment.getProperty(Constants.KMS_KEY_ENV_VARIABLE); if (Strings.isNullOrEmpty(bucketName) || Strings.isNullOrEmpty(kmsKey)) { API_LOG.warn("No bucket name is specified or no KMS key specified."); return CompletableFuture.completedFuture(""); } ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); s3ObjectMetadata.setContentLength(size); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, inStream, s3ObjectMetadata) .withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsKey)); API_LOG.info("Writing object {} to S3 bucket {}", keyName, bucketName); return actOnItem(putObjectRequest); }
/** * Tests if Object can be uploaded with KMS */ @Test public void shouldUploadWithEncryption() { final File uploadFile = new File(UPLOAD_FILE_NAME); final String objectKey = UPLOAD_FILE_NAME; s3Client.createBucket(BUCKET_NAME); final PutObjectRequest putObjectRequest = new PutObjectRequest(BUCKET_NAME, objectKey, uploadFile); putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_ENC_KEYREF)); s3Client.putObject(putObjectRequest); final GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(BUCKET_NAME, objectKey); final ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getObjectMetadataRequest); assertThat(objectMetadata.getContentLength(), is(uploadFile.length())); }
/** * Tests if Object can be uploaded with wrong KMS Key */ @Test public void shouldNotUploadStreamingWithWrongEncryptionKey() { final byte[] bytes = UPLOAD_FILE_NAME.getBytes(); final InputStream stream = new ByteArrayInputStream(bytes); final String objectKey = UUID.randomUUID().toString(); s3Client.createBucket(BUCKET_NAME); final ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(bytes.length); final PutObjectRequest putObjectRequest = new PutObjectRequest(BUCKET_NAME, objectKey, stream, metadata); putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_WRONG_KEYREF)); thrown.expect(AmazonS3Exception.class); thrown.expectMessage(containsString("Status Code: 400; Error Code: KMS.NotFoundException")); s3Client.putObject(putObjectRequest); }
/** * Puts an Object; Copies that object to a new bucket; Downloads the object from the new * bucket; compares checksums * of original and copied object * * @throws Exception if an Exception occurs */ @Test public void shouldCopyObjectEncrypted() throws Exception { final File uploadFile = new File(UPLOAD_FILE_NAME); final String sourceKey = UPLOAD_FILE_NAME; final String destinationBucketName = "destinationBucket"; final String destinationKey = "copyOf/" + sourceKey; s3Client.putObject(new PutObjectRequest(BUCKET_NAME, sourceKey, uploadFile)); final CopyObjectRequest copyObjectRequest = new CopyObjectRequest(BUCKET_NAME, sourceKey, destinationBucketName, destinationKey); copyObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_ENC_KEYREF)); final CopyObjectResult copyObjectResult = s3Client.copyObject(copyObjectRequest); final ObjectMetadata metadata = s3Client.getObjectMetadata(destinationBucketName, destinationKey); final InputStream uploadFileIS = new FileInputStream(uploadFile); final String uploadHash = HashUtil.getDigest(TEST_ENC_KEYREF, uploadFileIS); assertThat("ETag should match", copyObjectResult.getETag(), is(uploadHash)); assertThat("Files should have the same length", metadata.getContentLength(), is(uploadFile.length())); }
/** * Tests that an object wont be copied with wrong encryption Key * * @throws Exception if an Exception occurs */ @Test public void shouldNotObjectCopyWithWrongEncryptionKey() { final File uploadFile = new File(UPLOAD_FILE_NAME); final String sourceKey = UPLOAD_FILE_NAME; final String destinationBucketName = "destinationBucket"; final String destinationKey = "copyOf" + sourceKey; s3Client.putObject(new PutObjectRequest(BUCKET_NAME, sourceKey, uploadFile)); final CopyObjectRequest copyObjectRequest = new CopyObjectRequest(BUCKET_NAME, sourceKey, destinationBucketName, destinationKey); copyObjectRequest .setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_WRONG_KEYREF)); thrown.expect(AmazonS3Exception.class); thrown.expectMessage(containsString("Status Code: 400; Error Code: KMS.NotFoundException")); s3Client.copyObject(copyObjectRequest); }
private static void populateSSE_KMS(Request<?> request, SSEAwsKeyManagementParams sseParams) { if (sseParams != null) { addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION, sseParams.getEncryption()); addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID, sseParams.getAwsKmsKeyId()); } }
/** * Tests if Object can be uploaded with wrong KMS Key */ @Test public void shouldNotUploadWithWrongEncryptionKey() { final File uploadFile = new File(UPLOAD_FILE_NAME); final String objectKey = UPLOAD_FILE_NAME; s3Client.createBucket(BUCKET_NAME); final PutObjectRequest putObjectRequest = new PutObjectRequest(BUCKET_NAME, objectKey, uploadFile); putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_WRONG_KEYREF)); thrown.expect(AmazonS3Exception.class); thrown.expectMessage(containsString("Status Code: 400; Error Code: KMS.NotFoundException")); s3Client.putObject(putObjectRequest); }
private static SSEAwsKeyManagementParams toSSEAwsKeyManagementParams(final EncryptionKey encryptionKey) { if (encryptionKey != null && encryptionKey.getId() != null && EncryptionKeyType.KMS.toString().equals(encryptionKey.getType())) { return new SSEAwsKeyManagementParams(encryptionKey.getId()); } return new SSEAwsKeyManagementParams(); }
@Test public void uploadFileSuccess() throws IOException { TestUtils.initializeTestingFolders(); final File compressedFile = CompressionTools.compressFile( "ZipProject", PATH_TO_COMPRESS, CompressionType.Zip, null); PublisherTools.uploadFile( compressedFile, mockArtifact, CompressionType.Zip, null, // No custom encryption key mockS3Client, null); // Listener final InOrder inOrder = inOrder(mockS3Client); inOrder.verify(mockS3Client, times(1)).initiateMultipartUpload(initiateCaptor.capture()); // Total size is less than 5MB, should only be one upload inOrder.verify(mockS3Client, times(1)).uploadPart(any(UploadPartRequest.class)); inOrder.verify(mockS3Client, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); assertContainsIgnoreCase("[AWS CodePipeline Plugin] Uploading artifact:", outContent.toString()); assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString()); final InitiateMultipartUploadRequest request = initiateCaptor.getValue(); final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams(); assertNotNull(encryptionParams); assertNull(encryptionParams.getAwsKmsKeyId()); assertEquals("aws:kms", encryptionParams.getEncryption()); compressedFile.delete(); TestUtils.cleanUpTestingFolders(); }
@Test public void uploadWithCustomKmsEncryptionKey() throws IOException { TestUtils.initializeTestingFolders(); when(mockEncryptionKey.getId()).thenReturn("KMS-KEY-ARN"); when(mockEncryptionKey.getType()).thenReturn(EncryptionKeyType.KMS.toString()); final File compressedFile = CompressionTools.compressFile( "ZipProject", PATH_TO_COMPRESS, CompressionType.Zip, null); PublisherTools.uploadFile( compressedFile, mockArtifact, CompressionType.Zip, mockEncryptionKey, mockS3Client, null); // Listener verify(mockS3Client).initiateMultipartUpload(initiateCaptor.capture()); assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString()); final InitiateMultipartUploadRequest request = initiateCaptor.getValue(); final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams(); assertNotNull(encryptionParams); assertEquals("KMS-KEY-ARN", encryptionParams.getAwsKmsKeyId()); assertEquals("aws:kms", encryptionParams.getEncryption()); compressedFile.delete(); TestUtils.cleanUpTestingFolders(); }
@Test public void uploadWithUnknownEncryptionKeyType() throws IOException { TestUtils.initializeTestingFolders(); when(mockEncryptionKey.getId()).thenReturn("KMS-KEY-ARN"); when(mockEncryptionKey.getType()).thenReturn("Custom"); final File compressedFile = CompressionTools.compressFile( "ZipProject", PATH_TO_COMPRESS, CompressionType.Zip, null); PublisherTools.uploadFile( compressedFile, mockArtifact, CompressionType.Zip, mockEncryptionKey, mockS3Client, null); // Listener verify(mockS3Client).initiateMultipartUpload(initiateCaptor.capture()); assertContainsIgnoreCase("[AWS CodePipeline Plugin] Upload successful", outContent.toString()); final InitiateMultipartUploadRequest request = initiateCaptor.getValue(); final SSEAwsKeyManagementParams encryptionParams = request.getSSEAwsKeyManagementParams(); assertNotNull(encryptionParams); assertNull(encryptionParams.getAwsKmsKeyId()); assertEquals("aws:kms", encryptionParams.getEncryption()); compressedFile.delete(); TestUtils.cleanUpTestingFolders(); }
/** * not sure the method is called one or two times, it depend on the platform */ @Override public Result close() throws IOException { if (closed) { return result; } closed = true; try { if (writer != null) { writer.flush(); writer.close(); } S3DatasetProperties data_set = properties.getDatasetProperties(); PutObjectRequest request = new PutObjectRequest(data_set.bucket.getValue(), data_set.object.getValue(), data_file); Boolean serverSideEnc = data_set.encryptDataAtRest.getValue(); if (serverSideEnc != null && serverSideEnc) { request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(data_set.kmsForDataAtRest.getValue())); } s3_client.putObject(request); } finally { writer = null; data_file.delete(); if (s3_client != null) { s3_client.shutdown(); s3_client = null; } } result.successCount = result.totalCount; return result; }
private void enableKmsEncryption(PutObjectRequest uploadRequest) { String keyId = mConfig.getAwsSseKmsKey(); if (!keyId.isEmpty()) { uploadRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(keyId)); } else { uploadRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams()); } }
@NotNull private static SSEAwsKeyManagementParams getSSEAwsKeyManagementParams(@Nullable EncryptionKey encryptionKey) { return encryptionKey == null || encryptionKey.getId() == null || encryptionKey.getType() == null || !EncryptionKeyType.KMS.toString().equals(encryptionKey.getType()) ? new SSEAwsKeyManagementParams() : new SSEAwsKeyManagementParams(encryptionKey.getId()); }
@Override public S3FileTransferResultsDto copyFile(final S3FileCopyRequestParamsDto params) throws InterruptedException { LOGGER .info("Copying S3 object... sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\"", params.getSourceObjectKey(), params.getSourceBucketName(), params.getTargetObjectKey(), params.getTargetBucketName()); // Perform the copy. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { // Create a copy request. CopyObjectRequest copyObjectRequest = new CopyObjectRequest(params.getSourceBucketName(), params.getSourceObjectKey(), params.getTargetBucketName(), params.getTargetObjectKey()); // If KMS Key ID is specified, set the AWS Key Management System parameters to be used to encrypt the object. if (StringUtils.isNotBlank(params.getKmsKeyId())) { copyObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(params.getKmsKeyId())); } // Otherwise, specify the server-side encryption algorithm for encrypting the object using AWS-managed keys. else { ObjectMetadata metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); copyObjectRequest.setNewObjectMetadata(metadata); } return s3Operations.copyFile(copyObjectRequest, transferManager); } }); LOGGER.info("Copied S3 object. sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\" " + "totalBytesTransferred={} transferDuration=\"{}\"", params.getSourceObjectKey(), params.getSourceBucketName(), params.getTargetObjectKey(), params.getTargetBucketName(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }