private void getNextBatch() { if (listing == null) { listing = time(new Function<ListObjectsResult>() { @Override public ListObjectsResult call() { ListObjectsRequest request = new ListObjectsRequest(config.getBucketName()); request.setPrefix("".equals(prefix) ? null : prefix); if (config.isUrlEncodeKeys()) request.setEncodingType(EncodingType.url); return s3.listObjects(request); } }, OPERATION_LIST_OBJECTS); } else { log.info("getting next page of objects [prefix: {}, marker: {}, nextMarker: {}, encodingType: {}, maxKeys: {}]", listing.getPrefix(), listing.getMarker(), listing.getNextMarker(), listing.getEncodingType(), listing.getMaxKeys()); listing = time(new Function<ListObjectsResult>() { @Override public ListObjectsResult call() { return s3.listMoreObjects(listing); } }, OPERATION_LIST_OBJECTS); } objectIterator = listing.getObjects().iterator(); } }
@Override public ListObjectsResult listObjects(ListObjectsRequest request) { ListObjectsResult result = executeRequest(client, request, ListObjectsResult.class); if (result.isTruncated() && result.getNextMarker() == null) result.setNextMarker(result.getObjects().get(result.getObjects().size() - 1).getKey()); return result; }
@Override public ListObjectsResult listMoreObjects(ListObjectsResult lastResult) { return listObjects(new ListObjectsRequest(lastResult.getBucketName()) .withPrefix(lastResult.getPrefix()) .withDelimiter(lastResult.getDelimiter()) .withEncodingType(lastResult.getEncodingType()) .withMaxKeys(lastResult.getMaxKeys()) .withMarker(lastResult.getNextMarker())); }
private boolean doExists(String streamSegmentName) { try { ListObjectsResult result = client.listObjects(config.getBucket(), config.getRoot() + streamSegmentName); return !result.getObjects().isEmpty(); } catch (S3Exception e) { /* * TODO: This implementation is supporting both an empty list and a no such key * exception to indicate that the segment doesn't exist. It is trying to be safe, * but this is an indication that the behavior is not well understood. We need to * investigate the exact behavior we should expect out of this call and react * accordingly rather than guess. * * See https://github.com/pravega/pravega/issues/1559 */ if ( e.getErrorCode().equals("NoSuchKey")) { return false; } else { throw e; } } }
else listing = s3.listMoreObjects(listing); for (final S3Object summary : listing.getObjects()) { executor.blockingSubmit(new Runnable() { @Override } while (listing.isTruncated());
@Override protected ObjectSummary getNextObject() { if (listing == null || (!objectIterator.hasNext() && listing.isTruncated())) { getNextBatch(); } if (objectIterator.hasNext()) { S3Object object = objectIterator.next(); return new ObjectSummary(object.getKey(), false, object.getSize()); } // list is not truncated and iterators are finished; no more objects return null; }
TestContext() throws Exception { String bucketName = BUCKET_NAME_PREFIX + UUID.randomUUID().toString(); this.adapterConfig = ExtendedS3StorageConfig.builder() .with(ExtendedS3StorageConfig.BUCKET, bucketName) .with(ExtendedS3StorageConfig.ACCESS_KEY_ID, "x") .with(ExtendedS3StorageConfig.SECRET_KEY, "x") .with(ExtendedS3StorageConfig.ROOT, "test") .with(ExtendedS3StorageConfig.URI, endpoint) .build(); URI uri = URI.create(endpoint); s3Config = new S3Config(uri) .withIdentity(adapterConfig.getAccessKey()).withSecretKey(adapterConfig.getSecretKey()); s3Proxy = new S3ProxyImpl(endpoint, s3Config); s3Proxy.start(); client = new S3JerseyClientWrapper(s3Config, s3Proxy); client.createBucket(bucketName); List<ObjectKey> keys = client.listObjects(bucketName).getObjects().stream() .map(object -> new ObjectKey(object.getKey())) .collect(Collectors.toList()); if (!keys.isEmpty()) { client.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(keys)); } }
private SegmentHandle doCreate(String streamSegmentName) throws StreamSegmentExistsException { long traceId = LoggerHelpers.traceEnter(log, "create", streamSegmentName); if (!client.listObjects(config.getBucket(), config.getRoot() + streamSegmentName).getObjects().isEmpty()) { throw new StreamSegmentExistsException(streamSegmentName);