@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SdkClientException, AmazonServiceException { listObjectsRequest = beforeClientExecution(listObjectsRequest); rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag is enabled * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)}, * otherwise, it will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null; Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListObjects"); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
@Mock public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException { ObjectListing listing = new ObjectListing(); listing.setBucketName(listObjectsRequest.getBucketName()); listing.setPrefix(listObjectsRequest.getPrefix()); return listing; }
@Test public void testCallsListObjects() throws Exception { s3Objects.iterator().hasNext(); ArgumentCaptor<ListObjectsRequest> listCaptor = ArgumentCaptor .forClass(ListObjectsRequest.class); verify(s3).listObjects(listCaptor.capture()); assertEquals(s3Objects.getBucketName(), listCaptor.getValue().getBucketName()); assertEquals(s3Objects.getPrefix(), listCaptor.getValue().getPrefix()); assertNull(listCaptor.getValue().getMaxKeys()); }
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { assertParameterNotNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag * is enabled only when the customers don't explicitly call * {@link ListObjectsRequest#setEncodingType(String)}, otherwise, it * will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = Constants.URL_ENCODING .equals(listObjectsRequest.getEncodingType()); // if the url is encoded, the prefix also has to be encoded. final Request<ListObjectsRequest> request = createRequest( listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "encoding-type", listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) { request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); } return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
@Override /** * Only bucketName and prefix is used from the ListObjectsRequest. */ public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException { return listObjects(listObjectsRequest.getBucketName(), listObjectsRequest.getPrefix()); }
@Override public String toString() { if (isV1()) { return String.format("List %s:/%s", v1Request.getBucketName(), v1Request.getPrefix()); } else { return String.format("List %s:/%s", v2Request.getBucketName(), v2Request.getPrefix()); } } }
/** * Run the list object call without any failure probability. * This stops a very aggressive failure rate from completely overloading * the retry logic. * @param listObjectsRequest request * @return listing * @throws AmazonClientException failure */ private ObjectListing innerlistObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { LOG.debug("prefix {}", listObjectsRequest.getPrefix()); ObjectListing listing = super.listObjects(listObjectsRequest); listing = filterListObjects(listing); listing = restoreListObjects(listObjectsRequest, listing); return listing; }
log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'");
/** * Simulate eventual consistency of delete for this list operation: Any * recently-deleted keys will be added. * @param request List request * @param rawListing listing returned from underlying S3 * @return listing with recently-deleted items restored */ private ObjectListing restoreListObjects(ListObjectsRequest request, ObjectListing rawListing) { List<S3ObjectSummary> outputList = rawListing.getObjectSummaries(); List<String> outputPrefixes = rawListing.getCommonPrefixes(); // recursive list has no delimiter, returns everything that matches a // prefix. boolean recursiveObjectList = !("/".equals(request.getDelimiter())); String prefix = request.getPrefix(); restoreDeleted(outputList, outputPrefixes, recursiveObjectList, prefix); return new CustomObjectListing(rawListing, outputList, outputPrefixes); }
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException { if (listObjectsRequest.getBucketName().equals("tajo-test") && listObjectsRequest.getPrefix().equals("test/")) { MockObjectListing objectListing = new MockObjectListing(); return objectListing; } else { throw new TajoInternalError(new UnsupportedException()); } }
private void submitCopyJobsFromListing( AmazonS3URI sourceS3Uri, final AmazonS3URI targetS3Uri, ListObjectsRequest request, ObjectListing listing) { LOG .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(), request.getPrefix()); List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries(); for (final S3ObjectSummary s3ObjectSummary : objectSummaries) { String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey()); final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName; LOG .info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); if (s3s3CopierOptions.getCannedAcl() != null) { copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl()); } applyObjectMetadata(copyObjectRequest); TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary, targetS3Uri, targetKey); Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener); totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer(); copyJobs.add(copy); } }
private void submitCopyJobsFromListing( AmazonS3URI sourceS3Uri, final AmazonS3URI targetS3Uri, ListObjectsRequest request, ObjectListing listing) { LOG .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(), request.getPrefix()); List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries(); for (final S3ObjectSummary s3ObjectSummary : objectSummaries) { String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey()); final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName; LOG .info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey); if (s3s3CopierOptions.getCannedAcl() != null) { copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl()); } applyObjectMetadata(copyObjectRequest); TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary, targetS3Uri, targetKey); Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener); totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer(); copyJobs.add(copy); } }
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { assertParameterNotNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag * is enabled only when the customers don't explicitly call * {@link ListObjectsRequest#setEncodingType(String)}, otherwise, it * will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = Constants.URL_ENCODING .equals(listObjectsRequest.getEncodingType()); // if the url is encoded, the prefix also has to be encoded. final Request<ListObjectsRequest> request = createRequest( listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "encoding-type", listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) { request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); } return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SdkClientException, AmazonServiceException { listObjectsRequest = beforeClientExecution(listObjectsRequest); rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag is enabled * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)}, * otherwise, it will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null; Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
assertEquals(s3KeyPrefix, listObjectsRequest.getPrefix());