@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(probe == null, "Current page has not been completely processed yet"); checkState(tryFetchLookupSourceProvider(), "Not ready to handle input yet"); SpillInfoSnapshot spillInfoSnapshot = lookupSourceProvider.withLease(SpillInfoSnapshot::from); addInput(page, spillInfoSnapshot); }
private void clearProbe() { // Before updating the probe flush the current page buildPage(); probe = null; } }
/** * @return whether there are more positions on probe side */ private boolean advanceProbePosition(LookupSource lookupSource) { if (!probe.advanceNextPosition()) { clearProbe(); return false; } // update join position joinPosition = probe.getCurrentJoinPosition(lookupSource); return true; }
private void processProbe(LookupSource lookupSource) { verify(probe != null); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (!yieldSignal.isSet()) { if (probe.getPosition() >= 0) { if (!joinCurrentPosition(lookupSource, yieldSignal)) { break; } if (!currentProbePositionProducedRow) { currentProbePositionProducedRow = true; if (!outerJoinCurrentPosition()) { break; } } } currentProbePositionProducedRow = false; if (!advanceProbePosition(lookupSource)) { break; } statisticsCounter.recordProbe(joinSourcePositions); joinSourcePositions = 0; } }
if (lookupSourceLease.spillEpoch() == inputPageSpillEpoch) { processProbe(lookupSourceLease.getLookupSource()); return Optional.empty(); boolean currentProbePositionProducedRow = this.currentProbePositionProducedRow; clearProbe(); addInput(currentPage, spillInfoSnapshot); int currentRowPartition = getPartitionGenerator().getPartition(currentPage, currentPosition); boolean currentRowSpilled = spillInfoSnapshot.getSpillMask().test(currentRowPartition); }); joinSourcePositions = 0; Page unprocessed = pageTail(currentPage, currentPosition + 1); addInput(unprocessed, spillInfoSnapshot); Page remaining = pageTail(currentPage, currentPosition); restoreProbe(remaining, currentJoinPosition, currentProbePositionProducedRow, joinSourcePositions, spillInfoSnapshot);
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LookupSourceFactory lookupSourceFactory = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupJoinOperator.class.getSimpleName()); lookupSourceFactory.setTaskContext(driverContext.getPipelineContext().getTaskContext()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new LookupJoinOperator( operatorContext, probeTypes, buildOutputTypes, joinType, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan()), totalOperatorsCount, probeHashGenerator, partitioningSpillerFactory); }
addInput(unspilledInputPages.next()); return; restoreProbe( savedRow.row, savedRow.joinPositionWithinPartition,
@Override public boolean isFinished() { boolean finished = this.finished && probe == null && pageBuilder.isEmpty() && outputPage == null; // if finished drop references so memory is freed early if (finished) { close(); } return finished; }
private void restoreProbe(Page probePage, long joinPosition, boolean currentProbePositionProducedRow, int joinSourcePositions, SpillInfoSnapshot spillInfoSnapshot) { verify(probe == null); addInput(probePage, spillInfoSnapshot); verify(probe.advanceNextPosition()); this.joinPosition = joinPosition; this.currentProbePositionProducedRow = currentProbePositionProducedRow; this.joinSourcePositions = joinSourcePositions; }
private Page spillAndMaskSpilledPositions(Page page, IntPredicate spillMask) { checkState(spillInProgress.isDone(), "Previous spill still in progress"); checkSuccess(spillInProgress, "spilling failed"); if (!spiller.isPresent()) { spiller = Optional.of(partitioningSpillerFactory.create( probeTypes, getPartitionGenerator(), operatorContext.getSpillContext().newLocalSpillContext(), operatorContext.newAggregateSystemMemoryContext())); } PartitioningSpillResult result = spiller.get().partitionAndSpill(page, spillMask); spillInProgress = result.getSpillingFuture(); return result.getRetained(); }
@Override public Page getOutput() { if (lookupSource == null) { return null; } // join probe page with the lookup source if (probe != null) { while (joinCurrentPosition()) { if (!advanceProbePosition()) { break; } if (!outerJoinCurrentPosition()) { break; } } } // only flush full pages unless we are done if (pageBuilder.isFull() || (finishing && !pageBuilder.isEmpty() && probe == null)) { Page page = pageBuilder.build(); pageBuilder.reset(); return page; } return null; }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupJoinOperator.class.getSimpleName()); return new LookupJoinOperator(operatorContext, lookupSourceSupplier, probeTypes, joinType, joinProbeFactory); }
private boolean tryBuildPage() { if (pageBuilder.isFull()) { buildPage(); return true; } return false; }