/** * Sets the optional marker parameter indicating where in the bucket to begin * listing. * Returns this {@link ListObjectsRequest}, enabling additional method * calls to be chained together. * The list will only include keys that occur lexicographically * after the marker. * * @param marker * The optional parameter indicating where in the bucket to begin * listing. The list will only include keys that occur * lexicographically after the marker. * * @return This {@link ListObjectsRequest}, enabling additional method * calls to be chained together. * * @see ListObjectsRequest#getMarker() * @see ListObjectsRequest#setMarker(String) */ public ListObjectsRequest withMarker(String marker) { setMarker(marker); return this; }
@Override public void setNextMarker() { listObjectsRequest.setMarker(objectListing.getNextMarker()); }
private ObjectListing getObjectListingChunkV1(ListObjectsRequest request) throws IOException { ObjectListing result; try { // Query S3 for the next batch of objects. result = mClient.listObjects(request); // Advance the request continuation token to the next set of objects. request.setMarker(result.getNextMarker()); } catch (AmazonClientException e) { throw new IOException(e); } return result; }
private void initListing() { ListObjectsRequest listReq = new ListObjectsRequest(); listReq.setBucketName(bucket); listReq.setPrefix(prefix); if (StringUtils.isNotBlank(delimiter)) listReq.setDelimiter(delimiter); if (StringUtils.isNotBlank(marker)) listReq.setMarker(marker); objectListing = s3Client.listObjects(listReq); }
/** * Constructs a new {@link ListObjectsRequest} object and * initializes all required and optional object fields. * * @param bucketName * The name of the bucket whose objects are to be listed. * @param prefix * The prefix restricting what keys will be listed. * @param marker * The key marker indicating where listing results should begin. * @param delimiter * The delimiter for condensing common prefixes in the returned * listing results. * @param maxKeys * The maximum number of results to return. * * @see ListObjectsRequest#ListObjectsRequest() */ public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys) { setBucketName(bucketName); setPrefix(prefix); setMarker(marker); setDelimiter(delimiter); setMaxKeys(maxKeys); }
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
/** * Sets the optional marker parameter indicating where in the bucket to * begin listing. Returns this {@link ListObjectsRequest}, enabling * additional method calls to be chained together. The list will only * include keys that occur lexicographically after the marker. * * @param marker The optional parameter indicating where in the bucket to * begin listing. The list will only include keys that occur * lexicographically after the marker. * @return This {@link ListObjectsRequest}, enabling additional method calls * to be chained together. * @see ListObjectsRequest#getMarker() * @see ListObjectsRequest#setMarker(String) */ public ListObjectsRequest withMarker(String marker) { setMarker(marker); return this; }
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) { LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName)); List<S3ObjectSummary> objects = new ArrayList<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(bucketName); listObjectsRequest.withPrefix(directory); ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); if(ol.isTruncated()) { do { objects.addAll(ol.getObjectSummaries()); listObjectsRequest.setMarker(ol.getNextMarker()); ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); } while (ol.isTruncated()); } else { objects.addAll(ol.getObjectSummaries()); } if (objects.isEmpty()) { return emptyList(); } return unmodifiableList(objects); }
/** * Constructs a new {@link ListObjectsRequest} object and initializes all * required and optional object fields. * * @param bucketName The name of the bucket whose objects are to be listed. * @param prefix The prefix restricting what keys will be listed. * @param marker The key marker indicating where listing results should * begin. * @param delimiter The delimiter for condensing common prefixes in the * returned listing results. * @param maxKeys The maximum number of results to return. * @see ListObjectsRequest#ListObjectsRequest() */ public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys) { setBucketName(bucketName); setPrefix(prefix); setMarker(marker); setDelimiter(delimiter); setMaxKeys(maxKeys); }
/** * Sets the optional marker parameter indicating where in the bucket to * begin listing. Returns this {@link ListObjectsRequest}, enabling * additional method calls to be chained together. The list will only * include keys that occur lexicographically after the marker. * * @param marker The optional parameter indicating where in the bucket to * begin listing. The list will only include keys that occur * lexicographically after the marker. * @return This {@link ListObjectsRequest}, enabling additional method calls * to be chained together. * @see ListObjectsRequest#getMarker() * @see ListObjectsRequest#setMarker(String) */ public ListObjectsRequest withMarker(String marker) { setMarker(marker); return this; }
@Override public void setNextMarker() { listObjectsRequest.setMarker(objectListing.getNextMarker()); }
ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName); ObjectListing listing = null; while((listing == null) || (request.getMarker() != null)) { listing = s3Client.listObjects(request); // do stuff with listing request.setMarker(listing.getNextMarker()); }
private static String lastKey = ""; private static String preLastKey = ""; ... do{ preLastKey = lastKey; AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); String bucketName = "bucketname"; ListObjectsRequest lstRQ = new ListObjectsRequest().withBucketName(bucketName).withPrefix(""); lstRQ.setMarker(lastKey); ObjectListing objectListing = s3.listObjects(lstRQ); // loop and get file on S3 for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // get oject and do something..... } }while(lastKey != preLastKey);
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withDelimiter("/") .withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add); listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (final AmazonServiceException e) { log.error("Could not list common prefixes in S3", e); } return commonPrefixes; }
// We only want the keys that are in the folder ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName("my-bucket") .withPrefix("path/to/folder/"); ObjectListing objectListing; // Iterate over all the matching keys do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Apply the ACL s3.setObjectAcl(bucketName, key, acl); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("m"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
public List<String> listS3Objects(final String bucketName, final String prefix) { final List<String> s3Objects = Lists.newArrayList(); try { logger.info("Listing objects"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/") .withBucketName(bucketName).withPrefix( prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { if (objectSummary.getKey().equals(prefix)) { continue; } s3Objects.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException e) { logger.error("Error Message: " + e.getMessage()); } return s3Objects; }
@Override public ObjectListing getObjectListing(ListObjectsRequest request) { com.amazonaws.services.s3.model.ListObjectsRequest mutable = new com.amazonaws.services.s3.model.ListObjectsRequest(); mutable.setBucketName(request.getBucket()); mutable.setDelimiter(request.getDelimiter().orNull()); mutable.setEncodingType(request.getEncoding().orNull()); mutable.setMarker(request.getMarker().orNull()); mutable.setMaxKeys(request.getMax()); mutable.setPrefix(request.getPrefix().orNull()); return copyOf(client.listObjects(mutable)); }
public List<String> list(Run build, String bucket, String expandedFilter) { AmazonS3Client s3client = getClient(); String buildName = build.getDisplayName(); int buildID = build.getNumber(); Destination dest = new Destination(bucket, "jobs/" + buildName + "/" + buildID + "/" + name); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(dest.bucketName) .withPrefix(dest.objectName); List<String> files = Lists.newArrayList(); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { GetObjectRequest req = new GetObjectRequest(dest.bucketName, summary.getKey()); files.add(req.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return files; }
ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { /** Default Metadata **/ Date dtLastModified = objectSummary.getLastModified(); String sEtag = objectSummary.getETag(); long lSize = objectSummary.getSize(); String sStorageClass = objectSummary.getStorageClass(); /** To get user defined metadata **/ ObjectMetadata objectMetadata = s3client.getObjectMetadata(bucketName, objectSummary.getKey()); Map userMetadataMap = objectMetadata.getUserMetadata(); Map rowMetadataMap = objectMetadata.getRawMetadata(); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());