@Override public boolean isTruncated() { return (objectListing == null) ? false : objectListing.isTruncated(); } }
@Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } };
@Override @Nullable public ObjectListingChunk getNextChunk() throws IOException { if (mResult.isTruncated()) { ObjectListing nextResult = getObjectListingChunkV1(mRequest); if (nextResult != null) { return new S3AObjectListingChunkV1(mRequest, nextResult); } } return null; } }
@Override public boolean hasNext() { if (iterator.hasNext()) { return true; } else { while (objectListing.isTruncated() && !iterator.hasNext()) { objectListing = s3Client.listNextBatchOfObjects(objectListing); iterator = createIterator(); } } return iterator.hasNext(); }
ObjectListing listing = s3.listObjects( bucketName, prefix ); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3.listNextBatchOfObjects (listing); summaries.addAll (listing.getObjectSummaries()); }
Log.err("Failed to loadfile from S3: path = " + obj.getKey() + ", error = " + e.getClass().getName() + ", msg = " + e.getMessage()); if(currentList.isTruncated()) currentList = s3.listNextBatchOfObjects(currentList); else
int numberOfIteration = 0; while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries());
} while ( listing.isTruncated() ); return completeList;
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
if (object_listing.isTruncated()) { object_listing = s3.listNextBatchOfObjects(object_listing); } else {
} while ( listObjectsResponse.isTruncated() ); } while ( !commonPrefixes.isEmpty() );
if (objectListing.isTruncated() && objectListing.getNextMarker() == null) {
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Override public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws SdkClientException, AmazonServiceException { listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest); rejectNull(listNextBatchOfObjectsRequest, "The request object parameter must be specified when listing the next batch of objects in a bucket"); ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }
@Override protected Response serve() { String bucket = _bucket.value(); Log.info("ImportS3 processing (" + bucket + ")"); JsonObject json = new JsonObject(); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); AmazonS3 s3 = PersistS3.getClient(); ObjectListing currentList = s3.listObjects(bucket); processListing(currentList, succ, fail); while(currentList.isTruncated()){ currentList = s3.listNextBatchOfObjects(currentList); processListing(currentList, succ, fail); } json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); DKV.write_barrier(); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); return r; } }
if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else
@Test public void testGetsNextPageWithParamsFromFirstPage() throws Exception { when(objectListing.isTruncated()).thenReturn(true); ObjectListing secondPage = mock(ObjectListing.class); when(secondPage.isTruncated()).thenReturn(false); when(s3.listNextBatchOfObjects(objectListing)).thenReturn(secondPage); s3Objects.iterator().hasNext(); verify(s3).listNextBatchOfObjects(objectListing); }
@Before public void setUpSharedTests() throws Exception { objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn( new ArrayList<S3ObjectSummary>()); when(objectListing.isTruncated()).thenReturn(false); s3 = mock(AmazonS3.class); when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn( objectListing); firstSummary = mock(S3ObjectSummary.class); secondSummary = mock(S3ObjectSummary.class); }
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Override public ObjectListing listNextBatchOfObjects( ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) throws AmazonClientException, AmazonServiceException { final ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing(); if (!previousObjectListing.isTruncated()) { final ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setEncodingType(previousObjectListing.getEncodingType()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest()); }