public LocalMergeSourceOperator(OperatorContext operatorContext, List<LocalExchangeSource> sources, List<Type> types, PageWithPositionComparator comparator) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.sources = requireNonNull(sources, "sources is null"); List<WorkProcessor<Page>> pageProducers = sources.stream() .map(LocalExchangeSource::pages) .collect(toImmutableList()); mergedPages = mergeSortedPages( pageProducers, requireNonNull(comparator, "comparator is null"), types, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); }
private WorkProcessor<Page> mergeFromDisk() { checkState(spiller.isPresent()); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList()), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, memoryLimitForMerge); }
private WorkProcessor<Page> mergeFromDiskAndMemory() { checkState(spiller.isPresent()); hashAggregationBuilder.setOutputPartial(); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), ImmutableList.<WorkProcessor<Page>>builder() .addAll(spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList())) .add(hashAggregationBuilder.buildHashSortedResult()) .build(), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, max(memoryLimitForMerge - memoryLimitForMergeWithMemory, 1L)); }
@Override public void noMoreSplits() { mergedPages = mergeSortedPages( pageProducers, comparator, outputChannels, outputTypes, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); blockedOnSplits.set(null); }
@Override public final void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); requireNonNull(page, "page is null"); checkState(mergingOutput.needsInput(), "Page buffer is full"); mergingOutput.addInput(processor.process( operatorContext.getSession().toConnectorSession(), operatorContext.getDriverContext().getYieldSignal(), pageProcessorMemoryContext, page)); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); }
OperationTimer operationTimer = createTimer(); driverContext.startProcessTimer(); driverContext.getYieldSignal().setWithDelay(maxRuntime, driverContext.getYieldExecutor()); try { long start = System.nanoTime(); driverContext.getYieldSignal().reset(); driverContext.recordProcessed(operationTimer);
private static void processRow(final Driver joinDriver, final TaskStateMachine taskStateMachine) { joinDriver.getDriverContext().getYieldSignal().setWithDelay(TimeUnit.SECONDS.toNanos(1), joinDriver.getDriverContext().getYieldExecutor()); joinDriver.process(); joinDriver.getDriverContext().getYieldSignal().reset(); checkErrors(taskStateMachine); }
private void processProbe(LookupSource lookupSource) { verify(probe != null); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (!yieldSignal.isSet()) { if (probe.getPosition() >= 0) { if (!joinCurrentPosition(lookupSource, yieldSignal)) { break; } if (!currentProbePositionProducedRow) { currentProbePositionProducedRow = true; if (!outerJoinCurrentPosition()) { break; } } } currentProbePositionProducedRow = false; if (!advanceProbePosition(lookupSource)) { break; } statisticsCounter.recordProbe(joinSourcePositions); joinSourcePositions = 0; } }
(leftPosition, leftPage, rightPosition, rightPage) -> { filterFunctionCalls.incrementAndGet(); driverContext.getYieldSignal().forceYieldForTesting(); return true; })); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); filterFunctionCalls.set(0); assertNull(operator.getOutput()); assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once"); driverContext.getYieldSignal().reset(); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); driverContext.getYieldSignal().reset();
(leftPosition, leftPage, rightPosition, rightPage) -> { filterFunctionCalls.incrementAndGet(); driverContext.getYieldSignal().forceYieldForTesting(); return true; })); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); assertNull(operator.getOutput()); assertEquals(filterFunctionCalls.get(), i + 1, "Expected join to stop processing (yield) after calling filter function once"); driverContext.getYieldSignal().reset(); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); Page output = operator.getOutput(); assertNotNull(output);
DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (probePosition < probe.getPositionCount()) { if (joinPositions == null) {
driverContext.getYieldSignal().forceYieldForTesting(); return value; }))); driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor()); assertNull(operator.getOutput()); driverContext.getYieldSignal().reset(); driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor()); Page output = operator.getOutput(); driverContext.getYieldSignal().reset(); assertNotNull(output); assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
for (int i = 0; i < totalColumns; i++) { functions.add(new GenericLongFunction("page_col" + i, value -> { driverContext.getYieldSignal().forceYieldForTesting(); return value; })); driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor()); Page page = operator.getOutput(); if (i == totalColumns) { assertNull(page); driverContext.getYieldSignal().reset();
private Page processPageSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && mergingOutput.needsInput() && !yieldSignal.isSet()) { Page page = pageSource.getNextPage(); finishing = pageSource.isFinished(); pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); if (page != null) { page = recordProcessedInput(page); // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; Iterator<Optional<Page>> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); mergingOutput.addInput(output); } if (finishing) { mergingOutput.finish(); } } Page result = mergingOutput.getOutput(); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); return result; }
private Page processColumnSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && !yieldSignal.isSet()) { CursorProcessorOutput output = cursorProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, cursor, pageBuilder); pageSourceMemoryContext.setBytes(cursor.getSystemMemoryUsage()); long bytesProcessed = cursor.getCompletedBytes() - completedBytes; long elapsedNanos = cursor.getReadTimeNanos() - readTimeNanos; operatorContext.recordRawInputWithTiming(bytesProcessed, elapsedNanos); // TODO: derive better values for cursors operatorContext.recordProcessedInput(bytesProcessed, output.getProcessedRows()); completedBytes = cursor.getCompletedBytes(); readTimeNanos = cursor.getReadTimeNanos(); if (output.isNoMoreRows()) { finishing = true; mergingOutput.finish(); } } // only return a page if buffer is full or we are finishing Page page = null; if (!pageBuilder.isEmpty() && (finishing || pageBuilder.isFull())) { page = pageBuilder.build(); pageBuilder.reset(); } outputMemoryContext.setBytes(pageBuilder.getRetainedSizeInBytes()); return page; }
(leftPosition, leftPage, rightPosition, rightPage) -> { filterFunctionCalls.incrementAndGet(); driverContext.getYieldSignal().forceYieldForTesting(); return true; })); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); assertNull(operator.getOutput()); assertEquals(filterFunctionCalls.get(), i + 1, "Expected join to stop processing (yield) after calling filter function once"); driverContext.getYieldSignal().reset(); driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor()); Page output = operator.getOutput(); assertNotNull(output);