public StringBuilder appendFields(ObjectLocator locator, StringBuilder buffer, ToStringStrategy2 strategy) { { Long theTotalFilesTransferred; theTotalFilesTransferred = this.getTotalFilesTransferred(); strategy.appendField(locator, this, "totalFilesTransferred", buffer, theTotalFilesTransferred, (this.totalFilesTransferred!= null)); } { Long theTotalBytesTransferred; theTotalBytesTransferred = this.getTotalBytesTransferred(); strategy.appendField(locator, this, "totalBytesTransferred", buffer, theTotalBytesTransferred, (this.totalBytesTransferred!= null)); } { Long theDurationMillis; theDurationMillis = this.getDurationMillis(); strategy.appendField(locator, this, "durationMillis", buffer, theDurationMillis, (this.durationMillis!= null)); } return buffer; }
params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES_SUBSET.size());
@Override public S3FileTransferResultsDto downloadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 directory to the local system... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Note that the directory download always recursively copies sub-directories. // To not recurse, we would have to list the files on S3 (AmazonS3Client.html#listObjects) and manually copy them one at a time. // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER.info("Downloaded S3 directory to the local system. " + "s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public S3FileTransferResultsDto uploadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Uploading local directory to S3... localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } }); LOGGER.info("Uploaded local directory to S3. " + "localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public S3FileTransferResultsDto downloadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 directory to the local system... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Note that the directory download always recursively copies sub-directories. // To not recurse, we would have to list the files on S3 (AmazonS3Client.html#listObjects) and manually copy them one at a time. // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER.info("Downloaded S3 directory to the local system. " + "s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
/** * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadFile() throws IOException, InterruptedException { // Upload local file to s3Dao. testUploadFile(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Destination local file. File destinationLocalFile = Paths.get(localTempPath.toString(), LOCAL_FILE).toFile(); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(destinationLocalFile.getPath()); S3FileTransferResultsDto results = s3Dao.downloadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate that we have the file downloaded from S3. Assert.assertTrue(destinationLocalFile.isFile()); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.downloadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size()); // Validate that we have the directory downloaded from S3. for (String file : LOCAL_FILES) { Assert.assertTrue(Paths.get(localTempPath.toString(), TEST_S3_KEY_PREFIX, file).toFile().isFile()); } }
@Override public S3FileTransferResultsDto uploadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Uploading local directory to S3... localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } }); LOGGER.info("Uploaded local directory to S3. " + "localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testUploadDirectory() throws IOException, InterruptedException { // Create local test files. for (String file : LOCAL_FILES) { createLocalFile(localTempPath.toString(), file, FILE_SIZE_1_KB); } // Upload test file to s3Dao. // Since the S3 key prefix represents a directory, we add a trailing '/' character to it. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size()); // Build a list of expected S3 key values. List<String> expectedKeys = new ArrayList<>(); for (String file : LOCAL_FILES) { expectedKeys.add(TEST_S3_KEY_PREFIX + "/" + file.replaceAll("\\\\", "/")); } // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, expectedKeys); }
public int hashCode(ObjectLocator locator, HashCodeStrategy2 strategy) { int currentHashCode = 1; { Long theTotalFilesTransferred; theTotalFilesTransferred = this.getTotalFilesTransferred(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "totalFilesTransferred", theTotalFilesTransferred), currentHashCode, theTotalFilesTransferred, (this.totalFilesTransferred!= null)); } { Long theTotalBytesTransferred; theTotalBytesTransferred = this.getTotalBytesTransferred(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "totalBytesTransferred", theTotalBytesTransferred), currentHashCode, theTotalBytesTransferred, (this.totalBytesTransferred!= null)); } { Long theDurationMillis; theDurationMillis = this.getDurationMillis(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "durationMillis", theDurationMillis), currentHashCode, theDurationMillis, (this.durationMillis!= null)); } return currentHashCode; }
/** * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier. */ @Test public void testUploadFile() throws IOException, InterruptedException { // Create local test file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); }
/** * Test that we are able to upload a zero byte file to S3 using our DAO tier. */ @Test public void testUploadFileZeroBytes() throws IOException, InterruptedException { // Create an empty local file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_0_BYTE); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); }
/** * Test that we are able to perform the uploadFile S3Dao operation utilizing Reduced Redundancy Storage (RRS) storage option. */ @Test public void testUploadFileUseRrs() throws IOException, InterruptedException { // Create local test file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); s3FileTransferRequestParamsDto.setUseRrs(true); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); // TODO: Validate Reduced Redundancy Storage (RRS) storage option. }
/** * Test that we are able to perform the uploadFile S3Dao operation with the specified maximum number of threads set to some value. */ @Test public void testUploadFileWithMaxThreadsSet() throws IOException, InterruptedException { // Create local test file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); s3FileTransferRequestParamsDto.setMaxThreads(3); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier. */ @Test public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException { // Create a zero size local file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the zero bytes upload. Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with more than 1000 files using our DAO tier. This test is needed to * make AmazonS3.listObjects() to start truncating ObjectListing that it returns. */ @Test public void testUploadDirectoryLargeNumberOfFiles() throws IOException, InterruptedException { final int NUM_OF_FILES = 1001; // Create local files. for (int i = 0; i < NUM_OF_FILES; i++) { File targetFile = createLocalFile(localTempPath.toString(), String.format("%04d_%s", i, LOCAL_FILE), FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); } // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == NUM_OF_FILES); // Validate the empty folder upload. Assert.assertEquals(NUM_OF_FILES, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * Test S3 file copy without a KMS ID specified. */ @Test public void testCopyFileNoKmsId() throws InterruptedException { // Put a 1 byte file in S3. s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null); // Perform an S3 file copy operation when KMS ID value is not specified. for (String kmsId : Arrays.asList(BLANK_TEXT, null)) { S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto(); transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName()); transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName()); transferDto.setSourceObjectKey(TARGET_S3_KEY); transferDto.setTargetObjectKey(TARGET_S3_KEY); transferDto.setKmsKeyId(kmsId); S3FileTransferResultsDto resultsDto = s3Dao.copyFile(transferDto); assertEquals(Long.valueOf(1L), resultsDto.getTotalFilesTransferred()); } }
/** * Test S3 file copy without any errors. */ @Test public void testCopyFile() throws InterruptedException { // Put a 1 byte file in S3. s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null); S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto(); transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName()); transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName()); transferDto.setSourceObjectKey(TARGET_S3_KEY); transferDto.setTargetObjectKey(TARGET_S3_KEY); transferDto.setKmsKeyId(MockS3OperationsImpl.MOCK_KMS_ID); S3FileTransferResultsDto resultsDto = s3Dao.copyFile(transferDto); assertEquals(Long.valueOf(1L), resultsDto.getTotalFilesTransferred()); }