Refine search
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse); for ( S3ObjectSummary s : listObjectsResponse.getObjectSummaries() ) { log.debug("Skipping " + s.getKey() + " as it does not match filter."); continue; if ( leavesRoot(destinationDirectory, s.getKey()) ) { throw new RuntimeException("Cannot download key " + s.getKey() + ", its relative path resolves outside the parent directory."); if ( !s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER) ) { objectSummaries.add(s); totalSize += s.getSize(); commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes()); } while ( listObjectsResponse.isTruncated() ); } while ( !commonPrefixes.isEmpty() );
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Override public void setBucketName(String bucketName) { listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); }
/** * Constructs a new {@link ListObjectsRequest} object and * initializes all required and optional object fields. * * @param bucketName * The name of the bucket whose objects are to be listed. * @param prefix * The prefix restricting what keys will be listed. * @param marker * The key marker indicating where listing results should begin. * @param delimiter * The delimiter for condensing common prefixes in the returned * listing results. * @param maxKeys * The maximum number of results to return. * * @see ListObjectsRequest#ListObjectsRequest() */ public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys) { setBucketName(bucketName); setPrefix(prefix); setMarker(marker); setDelimiter(delimiter); setMaxKeys(maxKeys); }
private void initListing() { ListObjectsRequest listReq = new ListObjectsRequest(); listReq.setBucketName(bucket); listReq.setPrefix(prefix); if (StringUtils.isNotBlank(delimiter)) listReq.setDelimiter(delimiter); if (StringUtils.isNotBlank(marker)) listReq.setMarker(marker); objectListing = s3Client.listObjects(listReq); }
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) { LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName)); List<S3ObjectSummary> objects = new ArrayList<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(bucketName); listObjectsRequest.withPrefix(directory); ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); if(ol.isTruncated()) { do { objects.addAll(ol.getObjectSummaries()); listObjectsRequest.setMarker(ol.getNextMarker()); ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); } while (ol.isTruncated()); } else { objects.addAll(ol.getObjectSummaries()); } if (objects.isEmpty()) { return emptyList(); } return unmodifiableList(objects); }
ObjectListing bucketListing = amazonS3.listObjects( new ListObjectsRequest(bucket, prefix, null, null, 10000) ); List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries(); while (bucketListing.isTruncated()) { bucketListing = amazonS3.listNextBatchOfObjects(bucketListing); summaries.addAll(bucketListing.getObjectSummaries()); String itemName = summary.getKey(); int indexOfLastSlash = itemName.lastIndexOf("/"); Map<String, Object> objectMetadataMap = new HashMap<>(); long updatedTimestamp = summary.getLastModified().getTime();
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
@Override public String[] listNames(String path) throws IOException { String[] bucketPrefix = splitPathToBucketAndKey(path, false); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketPrefix[0]); if (bucketPrefix.length > 1) { listObjectsRequest.setPrefix(bucketPrefix[1]); } /* For listing objects, Amazon S3 returns up to 1,000 keys in the response. If you have more than 1,000 keys in your bucket, the response will be truncated. You should always check for if the response is truncated. */ ObjectListing objectListing; List<String> names = new ArrayList<>(); do { objectListing = this.amazonS3.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { names.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return names.toArray(new String[names.size()]); }
private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("m"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Override public List<DataRecord> getAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be null"); List<DataRecord> metadataList = new ArrayList<DataRecord>(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest); for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { metadataList.add(new S3DataRecord(this, s3service, bucket, new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true)); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } return metadataList; }
@Override protected void doStart() { AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3 s3Client = new AmazonS3Client(myCredentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ChannelProcessor channelProcessor = getChannelProcessor(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { String file = s3ObjectSummary.getKey(); LOGGER.info("Read the content of {}", file); GetObjectRequest objectRequest = new GetObjectRequest(bucket, file); S3Object objectPortion = s3Client.getObject(objectRequest); try { long startTime = System.currentTimeMillis(); processLines(channelProcessor, objectPortion.getObjectContent()); LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime); } catch (IOException e) { LOGGER.warn("Cannot process the {}, skipping", file, e); } } }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse("")); log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'"); final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream() .filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList()); log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket); AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent()) .build() .startScript();
ObjectListing objectListing = s3Client.listObjects(new ListObjectsRequest().withBucketName(<your bucket name>)); List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (S3ObjectSummary summary : objectSummaries) { String key = summary.getKey(); // do something with the key }
@Override public void list(String keyPrefix, FileListing callback) { checkArgument(keyPrefix != null, "keyPrefix is null"); String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix; ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(bucket); req.setPrefix(keyPrefix); ObjectListing listing; do { try { listing = getWithRetry(errorMessage, () -> client.listObjects(req)); } catch (StorageFileNotFoundException ex) { throw Throwables.propagate(ex.getCause()); } callback.accept(Lists.transform( listing.getObjectSummaries(), (summary) -> StorageObjectSummary.builder() .key(summary.getKey()) .contentLength(summary.getSize()) .lastModified(summary.getLastModified().toInstant()) .build() )); req.setMarker(listing.getNextMarker()); } while (listing.isTruncated()); }
ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix); ObjectListing storedAuxFilesList = null; try { storedAuxFilesList = s3.listObjects(req); } catch (SdkClientException sce) { throw new IOException ("S3 listAuxObjects: failed to get a listing for "+prefix); return ret; List<S3ObjectSummary> storedAuxFilesSummary = storedAuxFilesList.getObjectSummaries(); try { while (storedAuxFilesList.isTruncated()) { logger.fine("S3 listAuxObjects: going to next page of list"); storedAuxFilesList = s3.listNextBatchOfObjects(storedAuxFilesList); if (storedAuxFilesList != null) { storedAuxFilesSummary.addAll(storedAuxFilesList.getObjectSummaries()); String destinationKey = item.getKey(); String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1); logger.fine("S3 cached aux object fileName: " + fileName);