/** * Deletes a source S3 object based on the given bucket name and file path. * * @param s3BucketName the S3 bucket name * @param storageFilePath the storage file path * @param businessObjectDataKey the business object key */ private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) { // Delete the file from S3 if storage file information exists. if (!StringUtils.isEmpty(storageFilePath)) { try { // Delete the source file from S3. AwsParamsDto awsParams = awsHelper.getAwsParamsDto(); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath) .withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e); } } }
/** * Deletes a source S3 object based on the given bucket name and file path. * * @param s3BucketName the S3 bucket name * @param storageFilePath the storage file path * @param businessObjectDataKey the business object key */ private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) { // Delete the file from S3 if storage file information exists. if (!StringUtils.isEmpty(storageFilePath)) { try { // Delete the source file from S3. AwsParamsDto awsParams = awsHelper.getAwsParamsDto(); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath) .withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e); } } }
@Test public void testListVersionsRootKeyPrefix() { for (String s3KeyPrefix : Arrays.asList(null, BLANK_TEXT, "/")) { try { s3Dao.listVersions(S3FileTransferRequestParamsDto.builder().withS3KeyPrefix(s3KeyPrefix).build()); fail("Should throw an IllegalArgumentException when S3 key prefix specifies a root directory."); } catch (IllegalArgumentException e) { assertEquals("Listing of S3 versions from root directory is not allowed.", e.getMessage()); } } }
/** * Returns an S3 file transfer request parameters DTO instance initialized using hard coded test values. This DTO is required for testing and clean up * activities. * * @return the newly created S3 file transfer request parameters DTO */ public S3FileTransferRequestParamsDto getTestS3FileTransferRequestParamsDto() { String s3BucketName = storageDaoTestHelper.getS3ManagedBucketName(); return S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(AbstractDaoTest.TEST_S3_KEY_PREFIX).build(); }
@Test public void testDeleteDirectoryRootKeyPrefix() { for (String s3KeyPrefix : Arrays.asList(null, BLANK_TEXT, "/")) { try { s3Dao.deleteDirectory(S3FileTransferRequestParamsDto.builder().withS3KeyPrefix(s3KeyPrefix).build()); fail("Should throw an IllegalArgumentException when S3 key prefix specifies a root directory."); } catch (IllegalArgumentException e) { assertEquals("Deleting from root directory is not allowed.", e.getMessage()); } } }
@Test public void testListDirectoryRootKeyPrefix() { for (String s3KeyPrefix : Arrays.asList(null, BLANK_TEXT, "/")) { try { s3Dao.listDirectory(S3FileTransferRequestParamsDto.builder().withS3KeyPrefix(s3KeyPrefix).build()); fail("Should throw an IllegalArgumentException when S3 key prefix specifies a root directory."); } catch (IllegalArgumentException e) { assertEquals("Listing of S3 objects from root directory is not allowed.", e.getMessage()); } } }
/** * Cleans up the local temp directory and S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the local directory. FileUtils.deleteDirectory(localTempPath.toFile()); // Delete test files from S3 storage. Since test S3 key prefix represents a directory, we add a trailing '/' character to it. for (S3FileTransferRequestParamsDto params : Arrays.asList(s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()) .withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3ExternalBucketName()).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/") .build())) { if (!s3Dao.listDirectory(params).isEmpty()) { s3Dao.deleteDirectory(params); } } s3Operations.rollback(); }
/** * Cleans up the S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the source and target S3 folders. Since test S3 key prefix represents a directory, we add a trailing '/' character to it. for (S3FileTransferRequestParamsDto params : Arrays.asList( S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()) .withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3ExternalBucketName()).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/") .build())) { if (!s3Dao.listDirectory(params).isEmpty()) { s3Dao.deleteDirectory(params); } } s3Operations.rollback(); }
/** * Returns an S3 file transfer request parameters DTO instance initialized using hard coded test values. This DTO is required for testing and clean up * activities. * * @param s3KeyPrefix the S3 key prefix * * @return the newly created S3 file transfer request parameters DTO */ protected S3FileTransferRequestParamsDto getTestS3FileTransferRequestParamsDto(String s3KeyPrefix) { return S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix).withAwsAccessKeyId(S3_ACCESS_KEY) .withAwsSecretKey(S3_SECRET_KEY).withHttpProxyHost(HTTP_PROXY_HOST).withHttpProxyPort(HTTP_PROXY_PORT) .withLocalPath(LOCAL_TEMP_PATH_INPUT.toString()).build(); }
/** * Delete the source file from S3 * * @param completeUploadSingleParamsDto the DTO that contains complete upload single message parameters */ protected void deleteSourceFileFromS3Impl(CompleteUploadSingleParamsDto completeUploadSingleParamsDto) { try { // Delete the source file from S3. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()) .withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()) .withHttpProxyHost(completeUploadSingleParamsDto.getAwsParams().getHttpProxyHost()) .withHttpProxyPort(completeUploadSingleParamsDto.getAwsParams().getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { // Log the error if failed to delete the file from source S3 bucket. LOGGER.error("Failed to delete the upload single file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBucketName(), jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), e); } }
/** * Delete the source file from S3 * * @param completeUploadSingleParamsDto the DTO that contains complete upload single message parameters */ protected void deleteSourceFileFromS3Impl(CompleteUploadSingleParamsDto completeUploadSingleParamsDto) { try { // Delete the source file from S3. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()) .withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()) .withHttpProxyHost(completeUploadSingleParamsDto.getAwsParams().getHttpProxyHost()) .withHttpProxyPort(completeUploadSingleParamsDto.getAwsParams().getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { // Log the error if failed to delete the file from source S3 bucket. LOGGER.error("Failed to delete the upload single file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBucketName(), jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), e); } }
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(S3_BUCKET_NAME + "/" + TEST_S3_KEY_PREFIX + "/").build();
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(S3_BUCKET_NAME + "/" + TEST_S3_KEY_PREFIX + "/").build();
S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
S3FileTransferRequestParamsDto params = S3FileTransferRequestParamsDto.builder().withLocalPath(argParser.getStringValue(localPathOpt)) .withAwsAccessKeyId(argParser.getStringValue(s3AccessKeyOpt)).withAwsSecretKey(argParser.getStringValue(s3SecretKeyOpt)) .withS3Endpoint(argParser.getStringValue(s3EndpointOpt)).withMaxThreads(maxThreads).withHttpProxyHost(argParser.getStringValue(httpProxyHostOpt))
S3FileTransferRequestParamsDto.builder().withLocalPath(argParser.getStringValue(localPathOpt)).withUseRrs(argParser.getBooleanValue(rrsOpt)) .withAwsAccessKeyId(argParser.getStringValue(s3AccessKeyOpt)).withAwsSecretKey(argParser.getStringValue(s3SecretKeyOpt)) .withS3Endpoint(argParser.getStringValue(s3EndpointOpt)).withMaxThreads(maxThreads)
S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()).withS3KeyPrefix(filePath).build());