public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, boolean resumeOnRetry) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, resumeOnRetry, null); }
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, KeyFilter filter) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, false, filter); }
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, false); }
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
public static void downloadDir(String bucket_name, String key_prefix, String dir_path, boolean pause) { System.out.println("downloading to directory: " + dir_path + (pause ? " (pause)" : "")); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileDownload xfer = xfer_mgr.downloadDirectory( bucket_name, key_prefix, new File(dir_path)); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
@Override public MultipleFileDownload downloadDirectory(String s3BucketName, String s3KeyPrefix, File destinationDirectory, TransferManager transferManager) { return transferManager.downloadDirectory(s3BucketName, s3KeyPrefix, destinationDirectory); }
@Override public MultipleFileDownload downloadDirectory(String s3BucketName, String s3KeyPrefix, File destinationDirectory, TransferManager transferManager) { return transferManager.downloadDirectory(s3BucketName, s3KeyPrefix, destinationDirectory); }
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, false); }
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, KeyFilter filter) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, false, filter); }
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, boolean resumeOnRetry) { return downloadDirectory(bucketName, keyPrefix, destinationDirectory, resumeOnRetry, null); }
private void doItForJob(String jobId) throws InterruptedException { logger.info("Beginning download of "+jobId); String bucketName= getLast(on("/").omitEmptyStrings().split(awsLogUri)); Transfer that=transferManager.downloadDirectory(bucketName, jobId, new File(localLogTarget)); that.waitForCompletion(); } }
private Transfer download(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); Assert.state(payload instanceof File, () -> "For the 'DOWNLOAD' operation the 'payload' must be of " + "'java.io.File' type, but gotten: [" + payload.getClass() + ']'); File targetFile = (File) payload; String bucket = obtainBucket(requestMessage); String key = null; if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } else { key = targetFile.getName(); } Assert.state(key != null, () -> "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (targetFile.isDirectory()) { return this.transferManager.downloadDirectory(bucket, key, targetFile); } else { if (this.s3ProgressListener != null) { return this.transferManager.download(new GetObjectRequest(bucket, key), targetFile, this.s3ProgressListener); } else { return this.transferManager.download(bucket, key, targetFile); } } }
private Transfer download(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); Assert.state(payload instanceof File, "For the 'DOWNLOAD' operation the 'payload' must be of " + "'java.io.File' type, but gotten: [" + payload.getClass() + ']'); File targetFile = (File) payload; String bucket = obtainBucket(requestMessage); String key = null; if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } else { key = targetFile.getName(); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (targetFile.isDirectory()) { return this.transferManager.downloadDirectory(bucket, key, targetFile); } else { if (this.s3ProgressListener != null) { return this.transferManager.download(new GetObjectRequest(bucket, key), targetFile, this.s3ProgressListener); } else { return this.transferManager.download(bucket, key, targetFile); } } }
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) { final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile); fileDownload.waitForCompletion(); RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription()); return null; } else { final Download download = mgr.download(this.bucket, this.path, localFile); download.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription()); } }); download.waitForCompletion(); return null; } }