@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
/** * Cleans up the local temp directory and S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the local directory. FileUtils.deleteDirectory(localTempPath.toFile()); // Clean up the destination S3 folder. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(testS3KeyPrefix); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); }
/** * Cleans up the local temp directory and S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the local directory. FileUtils.deleteDirectory(localTempPath.toFile()); // Clean up the destination S3 folders. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); for (String keyPrefix : Arrays.asList(testS3KeyPrefix, TEST_S3_KEY_PREFIX)) { // Since the key prefix represents a directory, we add a trailing '/' character to it. s3FileTransferRequestParamsDto.setS3KeyPrefix(keyPrefix + "/"); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } }
/** * Cleans up the destination S3 key prefixes used by the uploader unit tests. */ protected void cleanupS3() { // Delete the test business object data versions. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(); for (String s3KeyPrefix : new String[] {S3_TEST_PARENT_PATH_V0, S3_TEST_PARENT_PATH_V1, S3_TEST_PATH_V0, S3_TEST_PATH_V1, S3_SIMPLE_TEST_PATH}) { s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix); if (!s3Service.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) { s3Service.deleteDirectory(s3FileTransferRequestParamsDto); } } }
@Test(expected = ObjectNotFoundException.class) public void testValidateS3FileObjectNotFoundException() throws IOException, InterruptedException { // Try to validate a non-existing S3 file. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.validateS3File(s3FileTransferRequestParamsDto, FILE_SIZE_1_KB); }
@Test(expected = IllegalArgumentException.class) public void testValidateS3FileRuntimeExceptionFileSizeDoesNotMatch() throws IOException, InterruptedException { // Put a 1 KB file in S3. PutObjectRequest putObjectRequest = new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null); s3Operations.putObject(putObjectRequest, null); // Try to validate an S3 file by specifying an incorrect file size. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.validateS3File(s3FileTransferRequestParamsDto, FILE_SIZE_1_KB + 999); }
@Test public void testValidateS3File() throws IOException, InterruptedException { // Put a 1 KB file in S3. PutObjectRequest putObjectRequest = new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null); s3Operations.putObject(putObjectRequest, null); // Validate the S3 file. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.validateS3File(s3FileTransferRequestParamsDto, FILE_SIZE_1_KB); }
/** * Uploads and registers a version if of the test business object data that will be used as a parent. * * @param s3KeyPrefix the destination S3 key prefix that must comply with the S3 naming conventions including the expected data version value * @param dataBridgeWebClient the databridge web client instance */ protected void uploadAndRegisterTestDataParent(String s3KeyPrefix, DataBridgeWebClient dataBridgeWebClient) throws Exception { uploadTestDataFilesToS3(s3KeyPrefix, testManifestFiles, new ArrayList<String>()); UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto(TEST_PARENT_PARTITION_VALUE, TEST_SUB_PARTITION_VALUES, false); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/"); BusinessObjectData businessObjectData = dataBridgeWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, true); BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData); dataBridgeWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, s3FileTransferRequestParamsDto, StorageEntity.MANAGED_STORAGE); dataBridgeWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID); // Clean up the local input directory used for the test data files upload. FileUtils.cleanDirectory(LOCAL_TEMP_PATH_INPUT.toFile()); }
@Test public void testS3FileExistsKeyNoExists() { S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); params.setS3KeyPrefix(TARGET_S3_KEY); Assert.assertFalse(s3Dao.s3FileExists(params)); }
/** * Test "key not found" scenario for the getObjectMetadata S3Dao operation. */ @Test public void testGetObjectMetadataS3KeyNoExists() { // Try to retrieve S3 object metadata for a non-existing S3 key. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); assertNull(s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto)); }
/** * Test "bucket not found" scenario for the getObjectMetadata S3Dao operation. */ @Test public void testGetObjectMetadataS3BucketNoExists() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); // Try to retrieve S3 object metadata when S3 bucket does not exist. s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); assertNull(s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto)); }
@Test public void testListDirectoryNoSuchBucket() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); try { s3Dao.listDirectory(s3FileTransferRequestParamsDto); fail("expected an IllegalArgumentException to be thrown, but no exceptions thrown"); } catch (Exception e) { assertEquals("thrown exception type", IllegalArgumentException.class, e.getClass()); assertEquals("thrown exception message", "The specified bucket '" + MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION + "' does not exist.", e.getMessage()); } }
@Test public void testListVersionsNoSuchBucket() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); try { s3Dao.listVersions(s3FileTransferRequestParamsDto); fail("expected an IllegalArgumentException to be thrown, but no exceptions thrown"); } catch (Exception e) { assertEquals("thrown exception type", IllegalArgumentException.class, e.getClass()); assertEquals("thrown exception message", "The specified bucket '" + MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION + "' does not exist.", e.getMessage()); } }
/** * Throws the exception as-is without wrapping if the exception is of type AmazonServiceException or children. */ @Test public void testS3FileExistsGenericAmazonError() { try { S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR); params.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.s3FileExists(params); Assert.fail("expected AmazonServiceException to be thrown, but no exceptions were thrown"); } catch (Exception e) { Assert.assertEquals("thrown exception type", AmazonServiceException.class, e.getClass()); } }
/** * Test that we are able to perform the deleteDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDeleteDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Validate that S3 directory is not empty. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); Assert.assertTrue(s3Dao.listDirectory(s3FileTransferRequestParamsDto).size() > 0); // Delete directory from S3 using s3Dao. s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); // Validate that S3 directory got deleted. Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier. */ @Test public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException { // Create a zero size local file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the zero bytes upload. Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * The method is successful when both bucket and key exists. */ @Test public void testS3FileExists() { String expectedKey = "foo"; String expectedValue = "bar"; ByteArrayInputStream inputStream = new ByteArrayInputStream((expectedKey + "=" + expectedValue).getBytes()); PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, inputStream, new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); params.setS3KeyPrefix(TARGET_S3_KEY); Assert.assertTrue(s3Dao.s3FileExists(params)); }