public synchronized Driver createDriver(DriverContext driverContext) { checkState(!closed, "DriverFactory is already closed"); requireNonNull(driverContext, "driverContext is null"); checkState(!closedLifespans.contains(driverContext.getLifespan()), "DriverFatory is already closed for driver group %s", driverContext.getLifespan()); encounteredLifespans.add(driverContext.getLifespan()); ImmutableList.Builder<Operator> operators = ImmutableList.builder(); for (OperatorFactory operatorFactory : operatorFactories) { Operator operator = operatorFactory.createOperator(driverContext); operators.add(operator); } return Driver.createDriver(driverContext, operators.build()); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); NestedLoopJoinBridge nestedLoopJoinBridge = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, NestedLoopJoinOperator.class.getSimpleName()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new NestedLoopJoinOperator( operatorContext, nestedLoopJoinBridge, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan())); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, NestedLoopBuildOperator.class.getSimpleName()); return new NestedLoopBuildOperator(operatorContext, nestedLoopJoinBridgeManager.getJoinBridge(driverContext.getLifespan())); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "LookupOuterOperatorFactory is closed"); Lifespan lifespan = driverContext.getLifespan(); if (createdLifespans.contains(lifespan)) { throw new IllegalStateException("Only one outer operator can be created per Lifespan"); } createdLifespans.add(lifespan); ListenableFuture<OuterPositionIterator> outerPositionsFuture = joinBridgeManager.getOuterPositionsFuture(lifespan); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupOuterOperator.class.getSimpleName()); joinBridgeManager.outerOperatorCreated(lifespan); return new LookupOuterOperator(operatorContext, outerPositionsFuture, probeOutputTypes, buildOutputTypes, () -> joinBridgeManager.outerOperatorClosed(lifespan)); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LocalExchange inMemoryExchange = localExchangeFactory.getLocalExchange(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LocalExchangeSourceOperator.class.getSimpleName()); return new LocalExchangeSourceOperator(operatorContext, inMemoryExchange.getNextSource()); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LocalExchange localExchange = localExchangeFactory.getLocalExchange(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LocalMergeSourceOperator.class.getSimpleName()); PageWithPositionComparator comparator = orderingCompiler.compilePageWithPositionComparator(types, sortChannels, orderings); List<LocalExchangeSource> sources = IntStream.range(0, localExchange.getBufferCount()) .boxed() .map(index -> localExchange.getNextSource()) .collect(toImmutableList()); return new LocalMergeSourceOperator(operatorContext, sources, types, comparator); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LookupSourceFactory lookupSourceFactory = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupJoinOperator.class.getSimpleName()); lookupSourceFactory.setTaskContext(driverContext.getPipelineContext().getTaskContext()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new LookupJoinOperator( operatorContext, probeTypes, buildOutputTypes, joinType, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan()), totalOperatorsCount, probeHashGenerator, partitioningSpillerFactory); }
@Override public SourceOperator createOperator(DriverContext driverContext) { checkState(!overallNoMoreOperators, "noMoreOperators() has been called"); checkState(!driverGroupsWithNoMoreOperators.contains(driverContext.getLifespan()), "noMoreOperators(lifespan) has been called"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, TestingScanOperator.class.getSimpleName()); return new TestingScanOperator(operatorContext, sourceId, driverContext.getLifespan()); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LocalExchangeSinkOperator.class.getSimpleName()); LocalExchangeSinkFactory localExchangeSinkFactory = localExchangeFactory.getLocalExchange(driverContext.getLifespan()).getSinkFactory(sinkFactoryId); return new LocalExchangeSinkOperator(operatorContext, localExchangeSinkFactory.createSink(), pagePreprocessor); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!overallNoMoreOperators, "noMoreOperators() has been called"); checkState(!driverGroupsWithNoMoreOperators.contains(driverContext.getLifespan()), "noMoreOperators(lifespan) has been called"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, TestingBuildOperator.class.getSimpleName()); buildStates.get(driverContext.getLifespan()).incrementPendingBuildCount(); return new TestingBuildOperator(operatorContext, driverContext.getLifespan()); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!overallNoMoreOperators, "noMoreOperators() has been called"); checkState(!driverGroupsWithNoMoreOperators.contains(driverContext.getLifespan()), "noMoreOperators(lifespan) has been called"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, TestingCrossJoinOperator.class.getSimpleName()); buildStates.get(driverContext.getLifespan()).incrementPendingLookupCount(); return new TestingCrossJoinOperator(operatorContext, driverContext.getLifespan()); }
@Override public HashBuilderOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName()); PartitionedLookupSourceFactory lookupSourceFactory = this.lookupSourceFactoryManager.getJoinBridge(driverContext.getLifespan()); int partitionIndex = getAndIncrementPartitionIndex(driverContext.getLifespan()); verify(partitionIndex < lookupSourceFactory.partitions()); return new HashBuilderOperator( operatorContext, lookupSourceFactory, partitionIndex, outputChannels, hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, expectedPositions, pagesIndexFactory, spillEnabled, singleStreamSpillerFactory); }
public Driver createDriver(DriverContext driverContext, @Nullable ScheduledSplit partitionedSplit) { Driver driver = driverFactory.createDriver(driverContext); // record driver so other threads add unpartitioned sources can see the driver // NOTE: this MUST be done before reading unpartitionedSources, so we see a consistent view of the unpartitioned sources drivers.add(new WeakReference<>(driver)); if (partitionedSplit != null) { // TableScanOperator requires partitioned split to be added before the first call to process driver.updateSource(new TaskSource(partitionedSplit.getPlanNodeId(), ImmutableSet.of(partitionedSplit), true)); } // add unpartitioned sources Optional<PlanNodeId> sourceId = driver.getSourceId(); if (sourceId.isPresent()) { TaskSource taskSource = unpartitionedSources.get(sourceId.get()); if (taskSource != null) { driver.updateSource(taskSource); } } status.decrementPendingCreation(pipelineContext.getPipelineId(), driverContext.getLifespan()); closeDriverFactoryIfFullyCreated(); return driver; }