@Override public void setNextMarker() { listObjectsRequest.setMarker(objectListing.getNextMarker()); }
private ObjectListing getObjectListingChunkV1(ListObjectsRequest request) throws IOException { ObjectListing result; try { // Query S3 for the next batch of objects. result = mClient.listObjects(request); // Advance the request continuation token to the next set of objects. request.setMarker(result.getNextMarker()); } catch (AmazonClientException e) { throw new IOException(e); } return result; }
/** * Creates a new {@link ListObjectsRequest} object using the previous object listing. * @return A new {@link ListObjectsRequest} object using the previous object listing. */ public ListObjectsRequest toListObjectsRequest() { return new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys())) .withEncodingType(previousObjectListing.getEncodingType()); }
&& objectListing.getNextMarker() == null) {
@Override public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws SdkClientException, AmazonServiceException { listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest); rejectNull(listNextBatchOfObjectsRequest, "The request object parameter must be specified when listing the next batch of objects in a bucket"); ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) { LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName)); List<S3ObjectSummary> objects = new ArrayList<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(bucketName); listObjectsRequest.withPrefix(directory); ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); if(ol.isTruncated()) { do { objects.addAll(ol.getObjectSummaries()); listObjectsRequest.setMarker(ol.getNextMarker()); ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); } while (ol.isTruncated()); } else { objects.addAll(ol.getObjectSummaries()); } if (objects.isEmpty()) { return emptyList(); } return unmodifiableList(objects); }
/** * Creates a new {@link ListObjectsRequest} object using the previous object listing. * @return A new {@link ListObjectsRequest} object using the previous object listing. */ public ListObjectsRequest toListObjectsRequest() { return new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys())) .withEncodingType(previousObjectListing.getEncodingType()); }
&& objectListing.getNextMarker() == null) {
@Override public void setNextMarker() { listObjectsRequest.setMarker(objectListing.getNextMarker()); }
@Override public ObjectListing listNextBatchOfObjects( ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws AmazonClientException, AmazonServiceException { final ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { final ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }
@Override protected T computeNext() { if (queue.isEmpty()) { loadBatch(); } while (queue.isEmpty() && prevObjectListing.getNextMarker() != null) { LOG.debug("Queue is empty, but there is more data in the S3 bucket"); loadBatch(); } if (!queue.isEmpty()) { return transformer.apply(queue.remove()); } return endOfData(); }
@Override protected T computeNext() { if (queue.isEmpty()) { loadBatch(); } while (queue.isEmpty() && prevObjectListing.getNextMarker() != null) { LOG.debug("Queue is empty, but there is more data in the S3 bucket"); loadBatch(); } if (!queue.isEmpty()) { return transformer.apply(queue.remove()); } return endOfData(); }
ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName); ObjectListing listing = null; while((listing == null) || (request.getMarker() != null)) { listing = s3Client.listObjects(request); // do stuff with listing request.setMarker(listing.getNextMarker()); }
// We only want the keys that are in the folder ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName("my-bucket") .withPrefix("path/to/folder/"); ObjectListing objectListing; // Iterate over all the matching keys do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Apply the ACL s3.setObjectAcl(bucketName, key, acl); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
/** * Creates a new {@link ListObjectsRequest} object using the previous object listing. * @return A new {@link ListObjectsRequest} object using the previous object listing. */ public ListObjectsRequest toListObjectsRequest() { return new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys())) .withEncodingType(previousObjectListing.getEncodingType()); }
/** * Creates a new {@link ListObjectsRequest} object using the previous object listing. * @return A new {@link ListObjectsRequest} object using the previous object listing. */ public ListObjectsRequest toListObjectsRequest() { return new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys())) .withEncodingType(previousObjectListing.getEncodingType()); }
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("m"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
CustomObjectListing(ObjectListing rawListing, List<S3ObjectSummary> customListing, List<String> customPrefixes) { super(); this.customListing = customListing; this.customPrefixes = customPrefixes; this.setBucketName(rawListing.getBucketName()); this.setCommonPrefixes(rawListing.getCommonPrefixes()); this.setDelimiter(rawListing.getDelimiter()); this.setEncodingType(rawListing.getEncodingType()); this.setMarker(rawListing.getMarker()); this.setMaxKeys(rawListing.getMaxKeys()); this.setNextMarker(rawListing.getNextMarker()); this.setPrefix(rawListing.getPrefix()); this.setTruncated(rawListing.isTruncated()); }
@Override public ObjectListing listNextBatchOfObjects( ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws AmazonClientException, AmazonServiceException { final ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { final ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }
private ObjectListing copyOf(com.amazonaws.services.s3.model.ObjectListing mutable) { ObjectListing.Builder builder = ObjectListing.builder(); builder.withBucket(mutable.getBucketName()); builder.withCommonPrefixes(mutable.getCommonPrefixes()); builder.withDelimiter(fromTrimToNull(mutable.getDelimiter())); builder.withEncoding(fromTrimToNull(mutable.getEncodingType())); builder.withIsTruncated(mutable.isTruncated()); builder.withMarker(fromTrimToNull(mutable.getMarker())); builder.withMaxKeys(mutable.getMaxKeys()); builder.withNextMarker(fromTrimToNull(mutable.getNextMarker())); builder.withPrefix(fromTrimToNull(mutable.getPrefix())); builder.withSummaries(getSummaries(mutable.getObjectSummaries())); return builder.build(); }