@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { ListenableFuture<SplitBatch> batch = splitSource.getNextBatch(partitionHandle, lifespan, maxSize); return Futures.transform(batch, splitBatch -> new SplitBatch( splitBatch.getSplits().stream() .filter(input -> ThreadLocalRandom.current().nextDouble() < sampleRatio) .collect(toImmutableList()), splitBatch.isLastBatch()), directExecutor()); }
private ListenableFuture<?> fetchSplits() { if (splits.size() >= min) { return immediateFuture(null); } ListenableFuture<SplitBatch> future = splitSource.getNextBatch(partitionHandle, lifespan, max - splits.size()); return Futures.transformAsync(future, splitBatch -> { splits.addAll(splitBatch.getSplits()); if (splitBatch.isLastBatch()) { noMoreSplits = true; return immediateFuture(null); } return fetchSplits(); }, directExecutor()); } }
private static ListenableFuture<NextBatchResult> getNextBatch(SplitSource splitSource, int maxSize) { ListenableFuture<SplitBatch> future = splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), maxSize); return Futures.transform(future, NextBatchResult::new, directExecutor()); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
try (SplitSource splitSource = splitManager.getSplits(session, layouts.get(0).getLayout().getHandle(), UNGROUPED_SCHEDULING)) { while (!Thread.currentThread().isInterrupted()) { SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)); List<Split> splits = splitBatch.getSplits();
scheduleGroup.nextSplitBatchFuture = splitSource.getNextBatch(scheduleGroup.partitionHandle, lifespan, splitBatchSize - pendingSplits.size());
@Override public CompletableFuture<List<Split>> getNextBatch(int maxSize) { return splitSource.getNextBatch(maxSize) .thenApply(splits -> splits.stream() .filter(input -> ThreadLocalRandom.current().nextDouble() < sampleRatio) .collect(toImmutableList())); }
private Split getLocalQuerySplit(Session session, TableLayoutHandle handle) { SplitSource splitSource = splitManager.getSplits(session, handle); List<Split> splits = new ArrayList<>(); splits.addAll(getFutureValue(splitSource.getNextBatch(1000))); while (!splitSource.isFinished()) { splits.addAll(getFutureValue(splitSource.getNextBatch(1000))); } checkArgument(splits.size() == 1, "Expected only one split for a local query, but got %s splits", splits.size()); return splits.get(0); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
batchFuture = splitSource.getNextBatch(splitBatchSize); batchFuture.thenRun(() -> stage.recordGetSplitTime(start));
for (Split split : getFutureValue(splitSource.getNextBatch(1000))) { scheduledSplits.add(new ScheduledSplit(sequenceId++, split));