private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
if ( listObjectsResponse == null ) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else {
@Override @Nullable protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException { String delimiter = recursive ? "" : PATH_SEPARATOR; key = PathUtils.normalizePath(key, PATH_SEPARATOR); // In case key is root (empty string) do not normalize prefix. key = key.equals(PATH_SEPARATOR) ? "" : key; if (mConf.isSet(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1) && mConf .get(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1).equals(Boolean.toString(true))) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ObjectListing result = getObjectListingChunkV1(request); if (result != null) { return new S3AObjectListingChunkV1(request, result); } } else { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ListObjectsV2Result result = getObjectListingChunk(request); if (result != null) { return new S3AObjectListingChunk(request, result); } } return null; }
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) { LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName)); List<S3ObjectSummary> objects = new ArrayList<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(bucketName); listObjectsRequest.withPrefix(directory); ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); if(ol.isTruncated()) { do { objects.addAll(ol.getObjectSummaries()); listObjectsRequest.setMarker(ol.getNextMarker()); ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); } while (ol.isTruncated()); } else { objects.addAll(ol.getObjectSummaries()); } if (objects.isEmpty()) { return emptyList(); } return unmodifiableList(objects); }
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else {
private void registerDeleteObject(String key, String bucket) { if (policy.shouldDelay(key)) { // Record summary so we can add it back for some time post-deletion ListObjectsRequest request = new ListObjectsRequest() .withBucketName(bucket) .withPrefix(key); S3ObjectSummary summary = innerlistObjects(request).getObjectSummaries() .stream() .filter(result -> result.getKey().equals(key)) .findFirst() .orElse(null); delayedDeletes.put(key, new Delete(System.currentTimeMillis(), summary)); } }
private boolean fileExistsinS3(String bucketName, String path) { return s3Client.listObjects(new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(path.substring(1)) .withMaxKeys(1)).getObjectSummaries().isEmpty(); }
public long fetchLastCommittedOffset() { logger.debug("Getting max offset for " + topic + ":" + partition); String prefix = getKeyPrefix(); logger.debug("Listing keys for bucket/prefix " + bucket + "/" + prefix); List<S3ObjectSummary> objectSummaries = awsClient.listObjects(new ListObjectsRequest().withBucketName(bucket).withDelimiter("/").withPrefix(prefix)).getObjectSummaries(); logger.debug("Received result " + objectSummaries); long maxOffset = 0; for (S3ObjectSummary objectSummary : objectSummaries) { logger.debug(objectSummary.getKey()); String[] offsets = objectSummary.getKey().substring(prefix.length()).split("_"); long endOffset = Long.valueOf(offsets[1]); if (endOffset > maxOffset) maxOffset = endOffset; } return maxOffset; }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withDelimiter("/") .withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add); listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (final AmazonServiceException e) { log.error("Could not list common prefixes in S3", e); } return commonPrefixes; }
@Override protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException { List<String> directoryContents = new ArrayList<String>(); try { String prefix = getKey(directory); Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix)); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() // .withBucketName(this.bucketName) // .withPrefix(prefix) // .withDelimiter("/"); ObjectListing objectListing; objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); while (objectListing.isTruncated()) { objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); } return directoryContents; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e); } }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
@Override protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException { List<String> directoryContents = new ArrayList<>(); try { String prefix = getKey(directory); Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix)); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() // .withBucketName(this.bucketName) // .withPrefix(prefix) // .withDelimiter("/"); ObjectListing objectListing; objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); while (objectListing.isTruncated()) { objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); } return directoryContents; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e); } }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources, String prefix, String keyPattern) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix); ObjectListing objectListing = null; do { try { if (objectListing == null) { objectListing = this.amazonS3.listObjects(listObjectsRequest); } else { objectListing = this.amazonS3.listNextBatchOfObjects(objectListing); } Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries()); if (!newResources.isEmpty()) { resources.addAll(newResources); } } catch (AmazonS3Exception e) { if (301 != e.getStatusCode()) { throw e; } } } while (objectListing != null && objectListing.isTruncated()); }
public List<S3ObjectSummary> listFiles(String filePrefix) { ListObjectsRequest listObjectRequest = new ListObjectsRequest().withBucketName(getAmazonType().getS3Bucket()) .withPrefix(filePrefix); return getAmazonType().getAmazonS3().listObjects(listObjectRequest).getObjectSummaries(); }
private void copy(AmazonS3URI source, AmazonS3URI target) { ListObjectsRequest request = listObjectsRequestFactory .newInstance() .withBucketName(source.getBucket()) .withPrefix(source.getKey()); ObjectListing listing = srcClient.listObjects(request); submitCopyJobsFromListing(source, target, request, listing); while (listing.isTruncated()) { listing = srcClient.listNextBatchOfObjects(listing); submitCopyJobsFromListing(source, target, request, listing); } }
private void copy(AmazonS3URI source, AmazonS3URI target) { ListObjectsRequest request = listObjectsRequestFactory .newInstance() .withBucketName(source.getBucket()) .withPrefix(source.getKey()); ObjectListing listing = srcClient.listObjects(request); submitCopyJobsFromListing(source, target, request, listing); while (listing.isTruncated()) { listing = srcClient.listNextBatchOfObjects(listing); submitCopyJobsFromListing(source, target, request, listing); } }
public List<String> list(Run build, String bucket, String expandedFilter) { AmazonS3Client s3client = getClient(); String buildName = build.getDisplayName(); int buildID = build.getNumber(); Destination dest = new Destination(bucket, "jobs/" + buildName + "/" + buildID + "/" + name); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(dest.bucketName) .withPrefix(dest.objectName); List<String> files = Lists.newArrayList(); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { GetObjectRequest req = new GetObjectRequest(dest.bucketName, summary.getKey()); files.add(req.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return files; }