public static void showTransferProgress(Transfer xfer) { // print the transfer's human-readable description System.out.println(xfer.getDescription()); // print an empty progress bar... printProgressBar(0.0); // update the progress bar while the xfer is ongoing. do { try { Thread.sleep(100); } catch (InterruptedException e) { return; } // Note: so_far and total aren't used, they're just for // documentation purposes. TransferProgress progress = xfer.getProgress(); long so_far = progress.getBytesTransferred(); long total = progress.getTotalBytesToTransfer(); double pct = progress.getPercentTransferred(); eraseProgressBar(); printProgressBar(pct); } while (xfer.isDone() == false); // print the final state of the transfer. TransferState xfer_state = xfer.getState(); System.out.println(": " + xfer_state); }
void copyComplete() { markAllDone(); transfer.setState(TransferState.Completed); // Since the copy has completed we can assume all bytes were successfully transferred // This is required since there are no progress updates available during server-side // copying of data. transfer.getProgress().updateProgress(transfer.getProgress().getTotalBytesToTransfer()); // AmazonS3Client takes care of all the events for single part uploads, // so we only need to send a completed event for multipart uploads. if (multipartCopyCallable.isMultipartCopy()) { publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT); } }
/** * Logs transfer progress for an S3 file transfer operation. * * @param transferProgress the progress of an S3 transfer operation */ private void logTransferProgress(TransferProgress transferProgress) { // If the total bytes to transfer is set to 0, we do not log the transfer progress. if (LOGGER.isInfoEnabled() && transferProgress.getTotalBytesToTransfer() > 0) { NumberFormat formatter = new DecimalFormat("#0.0"); LOGGER.info("progressBytesTransferred={} totalBytesToTransfer={} progressPercentTransferred={}", transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer(), formatter.format(transferProgress.getPercentTransferred())); } }
/** * Logs transfer progress for an S3 file transfer operation. * * @param transferProgress the progress of an S3 transfer operation */ private void logTransferProgress(TransferProgress transferProgress) { // If the total bytes to transfer is set to 0, we do not log the transfer progress. if (LOGGER.isInfoEnabled() && transferProgress.getTotalBytesToTransfer() > 0) { NumberFormat formatter = new DecimalFormat("#0.0"); LOGGER.info("progressBytesTransferred={} totalBytesToTransfer={} progressPercentTransferred={}", transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer(), formatter.format(transferProgress.getPercentTransferred())); } }
private Metrics gatherAllCopyResults() { AtomicLong bytesReplicated = new AtomicLong(0); registerRunningMetrics(bytesReplicated); for (Copy copyJob : copyJobs) { try { copyJob.waitForCompletion(); long alreadyReplicated = bytesReplicated.addAndGet(copyJob.getProgress().getTotalBytesToTransfer()); if (totalBytesToReplicate > 0) { LOG .info("Replicating...': {}% complete", String.format("%.0f", (alreadyReplicated / (double) totalBytesToReplicate) * 100.0)); } } catch (InterruptedException e) { throw new CircusTrainException(e); } } ImmutableMap<String, Long> metrics = ImmutableMap .of(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name(), totalBytesToReplicate); return new S3S3CopierMetrics(metrics, bytesReplicated.get()); }
private Metrics gatherAllCopyResults() { AtomicLong bytesReplicated = new AtomicLong(0); registerRunningMetrics(bytesReplicated); for (Copy copyJob : copyJobs) { try { copyJob.waitForCompletion(); long alreadyReplicated = bytesReplicated.addAndGet(copyJob.getProgress().getTotalBytesToTransfer()); if (totalBytesToReplicate > 0) { LOG .info("Replicating...': {}% complete", String.format("%.0f", (alreadyReplicated / (double) totalBytesToReplicate) * 100.0)); } } catch (InterruptedException e) { throw new CircusTrainException(e); } } ImmutableMap<String, Long> metrics = ImmutableMap .of(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name(), totalBytesToReplicate); return new S3S3CopierMetrics(metrics, bytesReplicated.get()); }
@Override public void transferStateChanged(Transfer transfer, TransferState state) { if (state == TransferState.Completed) { // NOTE: running progress doesn't seem to be reported correctly. // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment // so just printing total bytes when completed. LOG .debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey, transfer.getProgress().getTotalBytesToTransfer()); } } }
@Override public void transferStateChanged(Transfer transfer, TransferState state) { if (state == TransferState.Completed) { // NOTE: running progress doesn't seem to be reported correctly. // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment // so just printing total bytes when completed. LOG .debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey, transfer.getProgress().getTotalBytesToTransfer()); } } }
+ "/" + upload.getProgress() .getTotalBytesToTransfer() + " bytes transferred...\r"); Thread.sleep(statusUpdatePeriodInMs); + "/" + upload.getProgress() .getTotalBytesToTransfer() + " bytes transferred...\n"); } else {
getLog().debug("Transferring " + transfer.getProgress().getTotalBytesToTransfer() + " bytes..."); transfer.waitForCompletion(); getLog().info("Transferred " + transfer.getProgress().getBytesTransferred() + " bytes.");
@Override public void progressChanged(ProgressEvent progressEvent) { final ProgressEventType eventType = progressEvent.getEventType(); if (eventType == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT || eventType == ProgressEventType.TRANSFER_COMPLETED_EVENT) { long now = System.currentTimeMillis(); if (now > lastLogTime + LOG_INTERVAL_SECONDS * 1000 || eventType == ProgressEventType.TRANSFER_COMPLETED_EVENT) { TransferProgress transferProgress = upload.getProgress(); double durationSec = (now - beginTime) / 1000D; LOG.info("{} transferred {} of {} ({} percent), duration {}, speed {})", upload.getDescription(), human(transferProgress.getBytesTransferred(), "B"), human(transferProgress.getTotalBytesToTransfer(), "B"), transferProgress.getPercentTransferred(), human(durationSec, "s"), human(transferProgress.getBytesTransferred() / durationSec, "B/sec")); lastLogTime = now; } } else if (eventType == ProgressEventType.TRANSFER_FAILED_EVENT) { LOG.error("{}: TRANSFER FAILED.", upload.getDescription()); } } }
@Override public void progressChanged(ProgressEvent progressEvent) { final ProgressEventType eventType = progressEvent.getEventType(); if (eventType == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT || eventType == ProgressEventType.TRANSFER_COMPLETED_EVENT) { long now = System.currentTimeMillis(); if (now > lastLogTime + LOG_INTERVAL_SECONDS * 1000 || eventType == ProgressEventType.TRANSFER_COMPLETED_EVENT) { TransferProgress transferProgress = upload.getProgress(); double durationSec = (now - beginTime) / 1000D; LOG.info("{} transferred {} of {} ({} percent), duration {}, speed {})", upload.getDescription(), human(transferProgress.getBytesTransferred(), "B"), human(transferProgress.getTotalBytesToTransfer(), "B"), transferProgress.getPercentTransferred(), human(durationSec, "s"), human(transferProgress.getBytesTransferred() / durationSec, "B/sec")); lastLogTime = now; } } else if (eventType == ProgressEventType.TRANSFER_FAILED_EVENT) { LOG.error("{}: TRANSFER FAILED.", upload.getDescription()); } } }
private void submitCopyJobsFromListing( AmazonS3URI sourceS3Uri, final AmazonS3URI targetS3Uri, ListObjectsRequest request, ObjectListing listing) { LOG .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(), request.getPrefix()); List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries(); for (final S3ObjectSummary s3ObjectSummary : objectSummaries) { String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey()); final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName; LOG .info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); if (s3s3CopierOptions.getCannedAcl() != null) { copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl()); } applyObjectMetadata(copyObjectRequest); TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary, targetS3Uri, targetKey); Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener); totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer(); copyJobs.add(copy); } }
private void submitCopyJobsFromListing( AmazonS3URI sourceS3Uri, final AmazonS3URI targetS3Uri, ListObjectsRequest request, ObjectListing listing) { LOG .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(), request.getPrefix()); List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries(); for (final S3ObjectSummary s3ObjectSummary : objectSummaries) { String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey()); final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName; LOG .info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); if (s3s3CopierOptions.getCannedAcl() != null) { copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl()); } applyObjectMetadata(copyObjectRequest); TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary, targetS3Uri, targetKey); Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener); totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer(); copyJobs.add(copy); } }
Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(), String .format("Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).", transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer()));
Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(), String .format("Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).", transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer()));