/** * Logs all files found in the specified S3 location. * * @param params the S3 file transfer request parameters */ private void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params) { List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params); LOGGER.info( String.format("Found %d keys with prefix \"%s\" in bucket \"%s\":", s3ObjectSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName())); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { LOGGER.info(String.format(" s3://%s/%s", params.getS3BucketName(), s3ObjectSummary.getKey())); } } }
@Override public void validateS3File(S3FileTransferRequestParamsDto params, Long fileSizeInBytes) throws RuntimeException { ObjectMetadata objectMetadata = getObjectMetadata(params); if (objectMetadata == null) { throw new ObjectNotFoundException(String.format("File not found at s3://%s/%s location.", params.getS3BucketName(), params.getS3KeyPrefix())); } Assert.isTrue(fileSizeInBytes == null || Objects.equals(fileSizeInBytes, objectMetadata.getContentLength()), String .format("Specified file size (%d bytes) does not match to the actual file size (%d bytes) reported by S3 for s3://%s/%s file.", fileSizeInBytes, objectMetadata.getContentLength(), params.getS3BucketName(), params.getS3KeyPrefix())); }
@Override public void validateS3File(S3FileTransferRequestParamsDto params, Long fileSizeInBytes) throws RuntimeException { ObjectMetadata objectMetadata = getObjectMetadata(params); if (objectMetadata == null) { throw new ObjectNotFoundException(String.format("File not found at s3://%s/%s location.", params.getS3BucketName(), params.getS3KeyPrefix())); } Assert.isTrue(fileSizeInBytes == null || Objects.equals(fileSizeInBytes, objectMetadata.getContentLength()), String .format("Specified file size (%d bytes) does not match to the actual file size (%d bytes) reported by S3 for s3://%s/%s file.", fileSizeInBytes, objectMetadata.getContentLength(), params.getS3BucketName(), params.getS3KeyPrefix())); }
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadFileList(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.getFiles(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.uploadFileList(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.getFiles(), new ObjectMetadataProvider() { @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { prepareMetadata(params, metadata); } }, transferManager); } });
@Override public boolean s3FileExists(S3FileTransferRequestParamsDto params) throws RuntimeException { AmazonS3Client s3Client = getAmazonS3(params); try { S3Object s3Object = getS3Object(s3Client, params.getS3BucketName(), params.getS3KeyPrefix(), false); return (s3Object != null); } finally { s3Client.shutdown(); } }
@Override public boolean s3FileExists(S3FileTransferRequestParamsDto params) throws RuntimeException { AmazonS3Client s3Client = getAmazonS3(params); try { S3Object s3Object = getS3Object(s3Client, params.getS3BucketName(), params.getS3KeyPrefix(), false); return (s3Object != null); } finally { s3Client.shutdown(); } }
@Override public S3FileTransferResultsDto downloadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 file... s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER .info("Downloaded S3 file to the local system. s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public S3FileTransferResultsDto downloadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 file... s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER .info("Downloaded S3 file to the local system. s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
@Override public Transfer performTransfer(TransferManager transferManager) { // Get a handle to the local file. File localFile = new File(params.getLocalPath()); // Create and prepare the metadata. ObjectMetadata metadata = new ObjectMetadata(); prepareMetadata(params, metadata); // Create a put request and a transfer manager with the parameters and the metadata. PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), params.getS3KeyPrefix(), localFile); putObjectRequest.setMetadata(metadata); return s3Operations.upload(putObjectRequest, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { // Get a handle to the local file. File localFile = new File(params.getLocalPath()); // Create and prepare the metadata. ObjectMetadata metadata = new ObjectMetadata(); prepareMetadata(params, metadata); // Create a put request and a transfer manager with the parameters and the metadata. PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), params.getS3KeyPrefix(), localFile); putObjectRequest.setMetadata(metadata); return s3Operations.upload(putObjectRequest, transferManager); } });
@Override public S3FileTransferResultsDto downloadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException { LOGGER.info("Downloading S3 directory to the local system... s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath()); // Note that the directory download always recursively copies sub-directories. // To not recurse, we would have to list the files on S3 (AmazonS3Client.html#listObjects) and manually copy them one at a time. // Perform the transfer. S3FileTransferResultsDto results = performTransfer(params, new Transferer() { @Override public Transfer performTransfer(TransferManager transferManager) { return s3Operations.downloadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager); } }); LOGGER.info("Downloaded S3 directory to the local system. " + "s3KeyPrefix=\"{}\" s3BucketName=\"{}\" localDirectory=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis())); logOverallTransferRate(results); return results; }
/** * Test S3 exception handling in the getObjectMetadata S3Dao operation. */ @Test public void testGetObjectMetadataServiceException() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); // Validate that S3 service exception is handled correctly when retrieving S3 object metadata. try { s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto); fail("Should throw an IllegalStateException when Amazon service exception occurs."); } catch (IllegalStateException e) { assertEquals(String.format("Failed to get S3 metadata for object key \"%s\" from bucket \"%s\". " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", s3FileTransferRequestParamsDto.getS3KeyPrefix(), s3FileTransferRequestParamsDto.getS3BucketName()), e.getMessage()); } }
/** * Creates an object in S3 with the prefix constructed from the given parameters. The object's full path will be {prefix}/{UUID} * * @param businessObjectFormatEntity business object format * @param request request with partition values and storage * @param businessObjectDataVersion business object data version to put */ public void createS3Object(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataInvalidateUnregisteredRequest request, int businessObjectDataVersion) { StorageEntity storageEntity = storageDao.getStorageByName(request.getStorageName()); String s3BucketName = storageHelper.getS3BucketAccessParams(storageEntity).getS3BucketName(); BusinessObjectDataKey businessObjectDataKey = getBusinessObjectDataKey(request); businessObjectDataKey.setBusinessObjectDataVersion(businessObjectDataVersion); String s3KeyPrefix = s3KeyPrefixHelper .buildS3KeyPrefix(AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataKey, storageEntity.getName()); String s3ObjectKey = s3KeyPrefix + "/test"; PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, s3ObjectKey, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); }