for ( S3ObjectSummary s : listObjectsResponse.getObjectSummaries() ) { log.debug("Skipping " + s.getKey() + " as it does not match filter."); continue; if ( leavesRoot(destinationDirectory, s.getKey()) ) { throw new RuntimeException("Cannot download key " + s.getKey() + ", its relative path resolves outside the parent directory."); if ( !s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER) ) { objectSummaries.add(s); totalSize += s.getSize(); } else { log.debug("Skipping download for object " + s.getKey() + " since it is also a virtual directory"); commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes()); } while ( listObjectsResponse.isTruncated() ); } while ( !commonPrefixes.isEmpty() ); File f = new File(destinationDirectory, summary.getKey()); File parentFile = f.getParentFile(); GetObjectRequest req = new GetObjectRequest(summary.getBucketName(), summary.getKey()) .<GetObjectRequest>withGeneralProgressListener( listener);
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
if (!objectListing.getObjectSummaries().isEmpty()) { nextMarker = objectListing.getObjectSummaries() .get(objectListing.getObjectSummaries().size() - 1) .getKey(); } else if (!objectListing.getCommonPrefixes().isEmpty()) { if (name.equals("Key")) { lastKey = getText(); currentObject.setKey(decodeIfSpecified (lastKey, shouldSDKDecodeResponse)); } else if (name.equals("LastModified")) { currentObject.setLastModified( ServiceUtils.parseIso8601Date(getText())); currentObject.setETag( ServiceUtils.removeQuotes(getText())); currentObject.setSize(parseLong(getText())); currentObject.setStorageClass(getText()); currentObject.setOwner(currentOwner); currentOwner = null; currentOwner.setId(getText()); currentOwner.setDisplayName(getText());
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> objects) { // NOTE: for encrypted objects, S3ObjectSummary.size() used below is NOT correct, // however, to get the correct size we'd need to make an additional request to get // user metadata, and in this case it doesn't matter. return objects.stream() .filter(object -> !object.getKey().endsWith(PATH_SEPARATOR)) .map(object -> new FileStatus( object.getSize(), false, 1, BLOCK_SIZE.toBytes(), object.getLastModified().getTime(), qualifiedPath(new Path(PATH_SEPARATOR + object.getKey())))) .map(this::createLocatedFileStatus) .iterator(); }
/** * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. * * <pre> * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}} * </pre> */ private static URI toUri(S3ObjectSummary object) { final String originalAuthority = object.getBucketName(); final String originalPath = object.getKey(); final String authority = originalAuthority.endsWith("/") ? originalAuthority.substring(0, originalAuthority.length() - 1) : originalAuthority; final String path = originalPath.startsWith("/") ? originalPath.substring(1) : originalPath; return URI.create(StringUtils.format("s3://%s/%s", authority, path)); } }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { logger.info("Listing objects"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/") .withBucketName(bucketName).withPrefix( prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); commonPrefixes.addAll(objectListing.getCommonPrefixes()); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException e) { logger.error("Error Message: " + e.getMessage()); } return commonPrefixes; }
ObjectListing bucketListing = amazonS3.listObjects( new ListObjectsRequest(bucket, prefix, null, null, 10000) ); List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries(); while (bucketListing.isTruncated()) { bucketListing = amazonS3.listNextBatchOfObjects(bucketListing); summaries.addAll(bucketListing.getObjectSummaries()); String itemName = summary.getKey(); int indexOfLastSlash = itemName.lastIndexOf("/"); Map<String, Object> objectMetadataMap = new HashMap<>(); long updatedTimestamp = summary.getLastModified().getTime();
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
@Override public String[] listNames(String path) throws IOException { String[] bucketPrefix = splitPathToBucketAndKey(path, false); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketPrefix[0]); if (bucketPrefix.length > 1) { listObjectsRequest.setPrefix(bucketPrefix[1]); } /* For listing objects, Amazon S3 returns up to 1,000 keys in the response. If you have more than 1,000 keys in your bucket, the response will be truncated. You should always check for if the response is truncated. */ ObjectListing objectListing; List<String> names = new ArrayList<>(); do { objectListing = this.amazonS3.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { names.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return names.toArray(new String[names.size()]); }
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("m"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated());
bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix; s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred); s3.setEndpoint(bucketEndpoint); if (!s3.doesBucketExist(bucketName)) { try { s3.createBucket(bucketName); for (S3ObjectSummary sum : list.getObjectSummaries()) { S3CheckpointData data = read(sum.getKey()); if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else
@Override public List<DataRecord> getAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be null"); List<DataRecord> metadataList = new ArrayList<DataRecord>(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest); for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { metadataList.add(new S3DataRecord(this, s3service, bucket, new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true)); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } return metadataList; }
@Override protected void doStart() { AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3 s3Client = new AmazonS3Client(myCredentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ChannelProcessor channelProcessor = getChannelProcessor(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { String file = s3ObjectSummary.getKey(); LOGGER.info("Read the content of {}", file); GetObjectRequest objectRequest = new GetObjectRequest(bucket, file); S3Object objectPortion = s3Client.getObject(objectRequest); try { long startTime = System.currentTimeMillis(); processLines(channelProcessor, objectPortion.getObjectContent()); LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime); } catch (IOException e) { LOGGER.warn("Cannot process the {}, skipping", file, e); } } }
ObjectListing object_listing = s3.listObjects(bucket_name); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary)iterator.next(); s3.deleteObject(bucket_name, summary.getKey()); if (object_listing.isTruncated()) { object_listing = s3.listNextBatchOfObjects(object_listing); } else { break;
String prefix = bend < path.length()?path.substring(bend+1):""; AmazonS3 s3 = PersistS3.getClient(); if( !s3.doesBucketExist(bucket) ) throw new IllegalArgumentException("S3 Bucket " + bucket + " not found!");; ArrayList<String> succ = new ArrayList<String>(); ArrayList<String> fail = new ArrayList<String>(); ObjectListing currentList = s3.listObjects(bucket, prefix); while(true){ for(S3ObjectSummary obj:currentList.getObjectSummaries()) try { succ.add(S3FileVec.make(obj,fs).toString()); } catch( Throwable e ) { fail.add(obj.getKey()); Log.err("Failed to loadfile from S3: path = " + obj.getKey() + ", error = " + e.getClass().getName() + ", msg = " + e.getMessage()); if(currentList.isTruncated()) currentList = s3.listNextBatchOfObjects(currentList); else break;
final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse("")); log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'"); final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream() .filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList()); log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket); AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent()) .build() .startScript();
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {s3client.copyObject(sourceBucketName, objectSummary.getKey(), destinationBucketName, objectSummary.getKey());}
@Bean public AmazonS3 amazonS3() { AmazonS3 amazonS3 = Mockito.mock(AmazonS3.class); willAnswer(invocation -> { ObjectListing objectListing = new ObjectListing(); List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (S3Object s3Object : S3_OBJECTS) { S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setBucketName(S3_BUCKET); s3ObjectSummary.setKey(s3Object.getKey()); Calendar calendar = Calendar.getInstance(); calendar.add(Calendar.DATE, 1); s3ObjectSummary.setLastModified(calendar.getTime()); objectSummaries.add(s3ObjectSummary); } return objectListing; }).given(amazonS3).listObjects(any(ListObjectsRequest.class)); for (final S3Object s3Object : S3_OBJECTS) { willAnswer(invocation -> s3Object).given(amazonS3).getObject(S3_BUCKET, s3Object.getKey()); } return amazonS3; }