@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
theS3Endpoint = this.getS3Endpoint(); strategy.appendField(locator, this, "s3Endpoint", buffer, theS3Endpoint, (this.s3Endpoint!= null)); theS3BucketName = this.getS3BucketName(); strategy.appendField(locator, this, "s3BucketName", buffer, theS3BucketName, (this.s3BucketName!= null)); theS3KeyPrefix = this.getS3KeyPrefix(); strategy.appendField(locator, this, "s3KeyPrefix", buffer, theS3KeyPrefix, (this.s3KeyPrefix!= null)); theLocalPath = this.getLocalPath(); strategy.appendField(locator, this, "localPath", buffer, theLocalPath, (this.localPath!= null)); theFiles = (((this.files!= null)&&(!this.files.isEmpty()))?this.getFiles():null); strategy.appendField(locator, this, "files", buffer, theFiles, ((this.files!= null)&&(!this.files.isEmpty()))); theRecursive = this.isRecursive(); strategy.appendField(locator, this, "recursive", buffer, theRecursive, (this.recursive!= null)); theUseRrs = this.isUseRrs(); strategy.appendField(locator, this, "useRrs", buffer, theUseRrs, (this.useRrs!= null)); theMaxThreads = this.getMaxThreads(); strategy.appendField(locator, this, "maxThreads", buffer, theMaxThreads, (this.maxThreads!= null)); theKmsKeyId = this.getKmsKeyId(); strategy.appendField(locator, this, "kmsKeyId", buffer, theKmsKeyId, (this.kmsKeyId!= null)); theSocketTimeout = this.getSocketTimeout(); strategy.appendField(locator, this, "socketTimeout", buffer, theSocketTimeout, (this.socketTimeout!= null));
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadFileList(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.getFiles(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
public Object copyTo(ObjectLocator locator, Object target, CopyStrategy2 strategy) { final Object draftCopy = ((target == null)?createNewInstance():target); super.copyTo(locator, draftCopy, strategy); if (draftCopy instanceof S3FileTransferRequestParamsDto) { if (s3EndpointShouldBeCopiedAndSet == Boolean.TRUE) { String sourceS3Endpoint; sourceS3Endpoint = this.getS3Endpoint(); String copyS3Endpoint = ((String) strategy.copy(LocatorUtils.property(locator, "s3Endpoint", sourceS3Endpoint), sourceS3Endpoint, (this.s3Endpoint!= null))); copy.setS3Endpoint(copyS3Endpoint); } else { if (s3EndpointShouldBeCopiedAndSet == Boolean.FALSE) { if (s3BucketNameShouldBeCopiedAndSet == Boolean.TRUE) { String sourceS3BucketName; sourceS3BucketName = this.getS3BucketName(); String copyS3BucketName = ((String) strategy.copy(LocatorUtils.property(locator, "s3BucketName", sourceS3BucketName), sourceS3BucketName, (this.s3BucketName!= null))); copy.setS3BucketName(copyS3BucketName); } else { if (s3BucketNameShouldBeCopiedAndSet == Boolean.FALSE) { if (s3KeyPrefixShouldBeCopiedAndSet == Boolean.TRUE) { String sourceS3KeyPrefix; sourceS3KeyPrefix = this.getS3KeyPrefix(); String copyS3KeyPrefix = ((String) strategy.copy(LocatorUtils.property(locator, "s3KeyPrefix", sourceS3KeyPrefix), sourceS3KeyPrefix, (this.s3KeyPrefix!= null))); copy.setS3KeyPrefix(copyS3KeyPrefix); } else { if (s3KeyPrefixShouldBeCopiedAndSet == Boolean.FALSE) { if (localPathShouldBeCopiedAndSet == Boolean.TRUE) {
List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles()); params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() params.setKmsKeyId(kmsKeyId); params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS)); params.setS3BucketName(s3BucketName); params.setS3KeyPrefix(s3KeyPrefix + "/"); params.setFiles(sourceFiles); .format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(), params.getS3KeyPrefix())); .format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(), params.getS3BucketName())); s3Service.deleteDirectoryIgnoreException(params);
s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() targetLocalDirectory = Paths.get(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix()).toFile(); if (!targetLocalDirectory.isDirectory()) s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixInformation.getS3KeyPrefix() + "/"); s3FileTransferRequestParamsDto.setMaxThreads(adjustIntegerValue(s3FileTransferRequestParamsDto.getMaxThreads(), MIN_THREADS, MAX_THREADS)); s3FileTransferRequestParamsDto.setRecursive(true); cleanUpTargetLocalDirectoryOnFailure = true; s3Service.downloadDirectory(s3FileTransferRequestParamsDto); storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit);
/** * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.downloadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size()); // Validate that we have the directory downloaded from S3. for (String file : LOCAL_FILES) { Assert.assertTrue(Paths.get(localTempPath.toString(), TEST_S3_KEY_PREFIX, file).toFile().isFile()); } }
@Test public void testS3FileExistsKeyNoExists() { S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); params.setS3KeyPrefix(TARGET_S3_KEY); Assert.assertFalse(s3Dao.s3FileExists(params)); }
S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setLocalPath(localPath.toString()); s3FileTransferRequestParamsDto.setMaxThreads(1);
@Test public void testTagVersionsNoS3VersionSummaries() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); // Create an S3 file transfer request parameters DTO to tag S3 objects. S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto(); s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY); s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY); s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN); // Create an S3 object tag. Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE); // Call the method under test with a list of S3 version summaries passed as null. s3DaoImpl.tagVersions(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, null, tag); // Verify the external calls. verifyNoMoreInteractionsHelper(); }
@Test public void testListDirectoryNoSuchBucket() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); try { s3Dao.listDirectory(s3FileTransferRequestParamsDto); fail("expected an IllegalArgumentException to be thrown, but no exceptions thrown"); } catch (Exception e) { assertEquals("thrown exception type", IllegalArgumentException.class, e.getClass()); assertEquals("thrown exception message", "The specified bucket '" + MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION + "' does not exist.", e.getMessage()); } }
/** * Test that we are able to perform the uploadFile S3Dao operation with the specified maximum number of threads set to some value. */ @Test public void testUploadFileWithMaxThreadsSet() throws IOException, InterruptedException { // Create local test file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); s3FileTransferRequestParamsDto.setMaxThreads(3); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); }
@Override public void validateS3File(S3FileTransferRequestParamsDto params, Long fileSizeInBytes) throws RuntimeException { ObjectMetadata objectMetadata = getObjectMetadata(params); if (objectMetadata == null) { throw new ObjectNotFoundException(String.format("File not found at s3://%s/%s location.", params.getS3BucketName(), params.getS3KeyPrefix())); } Assert.isTrue(fileSizeInBytes == null || Objects.equals(fileSizeInBytes, objectMetadata.getContentLength()), String .format("Specified file size (%d bytes) does not match to the actual file size (%d bytes) reported by S3 for s3://%s/%s file.", fileSizeInBytes, objectMetadata.getContentLength(), params.getS3BucketName(), params.getS3KeyPrefix())); }
/** * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadFile() throws IOException, InterruptedException { // Upload local file to s3Dao. testUploadFile(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Destination local file. File destinationLocalFile = Paths.get(localTempPath.toString(), LOCAL_FILE).toFile(); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(destinationLocalFile.getPath()); S3FileTransferResultsDto results = s3Dao.downloadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate that we have the file downloaded from S3. Assert.assertTrue(destinationLocalFile.isFile()); }
/** * Deletes a source S3 object based on the given bucket name and file path. * * @param s3BucketName the S3 bucket name * @param storageFilePath the storage file path * @param businessObjectDataKey the business object key */ private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) { // Delete the file from S3 if storage file information exists. if (!StringUtils.isEmpty(storageFilePath)) { try { // Delete the source file from S3. AwsParamsDto awsParams = awsHelper.getAwsParamsDto(); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath) .withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e); } } }
s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setFiles(requestFileList); S3FileTransferResultsDto results = s3Dao.uploadFileList(s3FileTransferRequestParamsDto);
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
/** * Create an instance of {@link S3FileTransferRequestParamsDto } * */ public S3FileTransferRequestParamsDto createS3FileTransferRequestParamsDto() { return new S3FileTransferRequestParamsDto(); }