public StringBuilder appendFields(ObjectLocator locator, StringBuilder buffer, ToStringStrategy2 strategy) { { Long theTotalFilesTransferred; theTotalFilesTransferred = this.getTotalFilesTransferred(); strategy.appendField(locator, this, "totalFilesTransferred", buffer, theTotalFilesTransferred, (this.totalFilesTransferred!= null)); } { Long theTotalBytesTransferred; theTotalBytesTransferred = this.getTotalBytesTransferred(); strategy.appendField(locator, this, "totalBytesTransferred", buffer, theTotalBytesTransferred, (this.totalBytesTransferred!= null)); } { Long theDurationMillis; theDurationMillis = this.getDurationMillis(); strategy.appendField(locator, this, "durationMillis", buffer, theDurationMillis, (this.durationMillis!= null)); } return buffer; }
/** * Create an instance of {@link S3FileTransferResultsDto } * */ public S3FileTransferResultsDto createS3FileTransferResultsDto() { return new S3FileTransferResultsDto(); }
public Object copyTo(ObjectLocator locator, Object target, CopyStrategy2 strategy) { final Object draftCopy = ((target == null)?createNewInstance():target); if (draftCopy instanceof S3FileTransferResultsDto) { final S3FileTransferResultsDto copy = ((S3FileTransferResultsDto) draftCopy); if (totalFilesTransferredShouldBeCopiedAndSet == Boolean.TRUE) { Long sourceTotalFilesTransferred; sourceTotalFilesTransferred = this.getTotalFilesTransferred(); Long copyTotalFilesTransferred = ((Long) strategy.copy(LocatorUtils.property(locator, "totalFilesTransferred", sourceTotalFilesTransferred), sourceTotalFilesTransferred, (this.totalFilesTransferred!= null))); copy.setTotalFilesTransferred(copyTotalFilesTransferred); } else { if (totalFilesTransferredShouldBeCopiedAndSet == Boolean.FALSE) { if (totalBytesTransferredShouldBeCopiedAndSet == Boolean.TRUE) { Long sourceTotalBytesTransferred; sourceTotalBytesTransferred = this.getTotalBytesTransferred(); Long copyTotalBytesTransferred = ((Long) strategy.copy(LocatorUtils.property(locator, "totalBytesTransferred", sourceTotalBytesTransferred), sourceTotalBytesTransferred, (this.totalBytesTransferred!= null))); copy.setTotalBytesTransferred(copyTotalBytesTransferred); } else { if (totalBytesTransferredShouldBeCopiedAndSet == Boolean.FALSE) { if (durationMillisShouldBeCopiedAndSet == Boolean.TRUE) { Long sourceDurationMillis; sourceDurationMillis = this.getDurationMillis(); Long copyDurationMillis = ((Long) strategy.copy(LocatorUtils.property(locator, "durationMillis", sourceDurationMillis), sourceDurationMillis, (this.durationMillis!= null))); copy.setDurationMillis(copyDurationMillis); } else { if (durationMillisShouldBeCopiedAndSet == Boolean.FALSE) {
/** * Logs overall transfer rate for an S3 file transfer operation. * * @param s3FileTransferResultsDto the DTO for the S3 file transfer operation results */ private void logOverallTransferRate(S3FileTransferResultsDto s3FileTransferResultsDto) { if (LOGGER.isInfoEnabled()) { NumberFormat formatter = new DecimalFormat("#0.00"); LOGGER.info("overallTransferRateKiloBytesPerSecond={} overallTransferRateMegaBitsPerSecond={}", formatter.format(awsHelper .getTransferRateInKilobytesPerSecond(s3FileTransferResultsDto.getTotalBytesTransferred(), s3FileTransferResultsDto.getDurationMillis())), formatter.format(awsHelper .getTransferRateInMegabitsPerSecond(s3FileTransferResultsDto.getTotalBytesTransferred(), s3FileTransferResultsDto.getDurationMillis()))); } }
Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES_SUBSET.size());
S3FileTransferResultsDto results = new S3FileTransferResultsDto(); results.setDurationMillis(stopWatch.getTime()); results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred()); results.setTotalFilesTransferred(1L); results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size()); results.setTotalFilesTransferred((long) listDirectory(params).size());
/** * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadFile() throws IOException, InterruptedException { // Upload local file to s3Dao. testUploadFile(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Destination local file. File destinationLocalFile = Paths.get(localTempPath.toString(), LOCAL_FILE).toFile(); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(destinationLocalFile.getPath()); S3FileTransferResultsDto results = s3Dao.downloadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate that we have the file downloaded from S3. Assert.assertTrue(destinationLocalFile.isFile()); }
S3FileTransferResultsDto results = new S3FileTransferResultsDto(); results.setDurationMillis(stopWatch.getTime()); results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred()); results.setTotalFilesTransferred(1L); results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size()); results.setTotalFilesTransferred((long) listDirectory(params).size());
/** * Logs overall transfer rate for an S3 file transfer operation. * * @param s3FileTransferResultsDto the DTO for the S3 file transfer operation results */ private void logOverallTransferRate(S3FileTransferResultsDto s3FileTransferResultsDto) { if (LOGGER.isInfoEnabled()) { NumberFormat formatter = new DecimalFormat("#0.00"); LOGGER.info("overallTransferRateKiloBytesPerSecond={} overallTransferRateMegaBitsPerSecond={}", formatter.format(awsHelper .getTransferRateInKilobytesPerSecond(s3FileTransferResultsDto.getTotalBytesTransferred(), s3FileTransferResultsDto.getDurationMillis())), formatter.format(awsHelper .getTransferRateInMegabitsPerSecond(s3FileTransferResultsDto.getTotalBytesTransferred(), s3FileTransferResultsDto.getDurationMillis()))); } }
public int hashCode(ObjectLocator locator, HashCodeStrategy2 strategy) { int currentHashCode = 1; { Long theTotalFilesTransferred; theTotalFilesTransferred = this.getTotalFilesTransferred(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "totalFilesTransferred", theTotalFilesTransferred), currentHashCode, theTotalFilesTransferred, (this.totalFilesTransferred!= null)); } { Long theTotalBytesTransferred; theTotalBytesTransferred = this.getTotalBytesTransferred(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "totalBytesTransferred", theTotalBytesTransferred), currentHashCode, theTotalBytesTransferred, (this.totalBytesTransferred!= null)); } { Long theDurationMillis; theDurationMillis = this.getDurationMillis(); currentHashCode = strategy.hashCode(LocatorUtils.property(locator, "durationMillis", theDurationMillis), currentHashCode, theDurationMillis, (this.durationMillis!= null)); } return currentHashCode; }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDownloadDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Clean up the local directory, so we can test the download. FileUtils.deleteDirectory(localTempPath.toFile()); // Create local temp directory - this also validates that clean up was really executed. Assert.assertTrue(localTempPath.toFile().mkdir()); // Execute download. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.downloadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size()); // Validate that we have the directory downloaded from S3. for (String file : LOCAL_FILES) { Assert.assertTrue(Paths.get(localTempPath.toString(), TEST_S3_KEY_PREFIX, file).toFile().isFile()); } }
@Override public S3FileTransferResultsDto downloadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 file... s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER .info("Downloaded S3 file to the local system. s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
public Object createNewInstance() { return new S3FileTransferResultsDto(); }
lhsTotalFilesTransferred = this.getTotalFilesTransferred(); Long rhsTotalFilesTransferred; rhsTotalFilesTransferred = that.getTotalFilesTransferred(); if (!strategy.equals(LocatorUtils.property(thisLocator, "totalFilesTransferred", lhsTotalFilesTransferred), LocatorUtils.property(thatLocator, "totalFilesTransferred", rhsTotalFilesTransferred), lhsTotalFilesTransferred, rhsTotalFilesTransferred, (this.totalFilesTransferred!= null), (that.totalFilesTransferred!= null))) { return false; lhsTotalBytesTransferred = this.getTotalBytesTransferred(); Long rhsTotalBytesTransferred; rhsTotalBytesTransferred = that.getTotalBytesTransferred(); if (!strategy.equals(LocatorUtils.property(thisLocator, "totalBytesTransferred", lhsTotalBytesTransferred), LocatorUtils.property(thatLocator, "totalBytesTransferred", rhsTotalBytesTransferred), lhsTotalBytesTransferred, rhsTotalBytesTransferred, (this.totalBytesTransferred!= null), (that.totalBytesTransferred!= null))) { return false; lhsDurationMillis = this.getDurationMillis(); Long rhsDurationMillis; rhsDurationMillis = that.getDurationMillis(); if (!strategy.equals(LocatorUtils.property(thisLocator, "durationMillis", lhsDurationMillis), LocatorUtils.property(thatLocator, "durationMillis", rhsDurationMillis), lhsDurationMillis, rhsDurationMillis, (this.durationMillis!= null), (that.durationMillis!= null))) { return false;
/** * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testUploadDirectory() throws IOException, InterruptedException { // Create local test files. for (String file : LOCAL_FILES) { createLocalFile(localTempPath.toString(), file, FILE_SIZE_1_KB); } // Upload test file to s3Dao. // Since the S3 key prefix represents a directory, we add a trailing '/' character to it. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size()); // Build a list of expected S3 key values. List<String> expectedKeys = new ArrayList<>(); for (String file : LOCAL_FILES) { expectedKeys.add(TEST_S3_KEY_PREFIX + "/" + file.replaceAll("\\\\", "/")); } // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, expectedKeys); }
@Override public S3FileTransferResultsDto downloadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 file... s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER .info("Downloaded S3 file to the local system. s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public S3FileTransferResultsDto build() { if (_storedValue == null) { return this.init(new S3FileTransferResultsDto()); } else { return ((S3FileTransferResultsDto) _storedValue); } }
@Override public S3FileTransferResultsDto downloadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 directory to the local system... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Note that the directory download always recursively copies sub-directories. // To not recurse, we would have to list the files on S3 (AmazonS3Client.html#listObjects) and manually copy them one at a time. // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER.info("Downloaded S3 directory to the local system. " + "s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
/** * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier. */ @Test public void testUploadFile() throws IOException, InterruptedException { // Create local test file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB); Assert.assertTrue(targetFile.isFile()); Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB); // Upload test file to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath()); S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the file upload. s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY)); }
@Override public S3FileTransferResultsDto uploadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Uploading local file to S3... localPath=\"{}\" s3Key=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { // Get a handle to the local file. File localFile = new File(params.getLocalPath()); // Create and prepare the metadata. ObjectMetadata metadata = new ObjectMetadata(); prepareMetadata(params, metadata); // Create a put request and a transfer manager with the parameters and the metadata. PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), params.getS3KeyPrefix(), localFile); putObjectRequest.setMetadata(metadata); return s3Operations.upload(putObjectRequest, transferManager); } }); LOGGER.info("Uploaded local file to the S3. localPath=\"{}\" s3Key=\"{}\" s3BucketName=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }