@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext context = driverContext.addOperatorContext(operatorId, planNodeId, DeleteOperator.class.getSimpleName()); return new DeleteOperator(context, rowIdChannel); }
public Session getSession() { return driverContext.getSession(); }
try { inFlightException = closeAndDestroyOperators(activeOperators); if (driverContext.getMemoryUsage() > 0) { log.error("Driver still has memory reserved after freeing all operator memory."); if (driverContext.getSystemMemoryUsage() > 0) { log.error("Driver still has system memory reserved after freeing all operator memory."); if (driverContext.getRevocableMemoryUsage() > 0) { log.error("Driver still has revocable memory reserved after freeing all operator memory. Freeing it."); driverContext.finished(); t, "Error destroying driver for task %s", driverContext.getTaskId());
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, UnnestOperator.class.getSimpleName()); return new UnnestOperator(operatorContext, replicateChannels, replicateTypes, unnestChannels, unnestTypes, withOrdinality, SystemSessionProperties.isLegacyUnnest(driverContext.getSession())); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, NestedLoopBuildOperator.class.getSimpleName()); return new NestedLoopBuildOperator(operatorContext, nestedLoopJoinBridgeManager.getJoinBridge(driverContext.getLifespan())); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LookupSourceFactory lookupSourceFactory = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupJoinOperator.class.getSimpleName()); lookupSourceFactory.setTaskContext(driverContext.getPipelineContext().getTaskContext()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new LookupJoinOperator( operatorContext, probeTypes, buildOutputTypes, joinType, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan()), totalOperatorsCount, probeHashGenerator, partitioningSpillerFactory); }
private static void processRow(final Driver joinDriver, final TaskStateMachine taskStateMachine) { joinDriver.getDriverContext().getYieldSignal().setWithDelay(TimeUnit.SECONDS.toNanos(1), joinDriver.getDriverContext().getYieldExecutor()); joinDriver.process(); joinDriver.getDriverContext().getYieldSignal().reset(); checkErrors(taskStateMachine); }
driverContext.startProcessTimer(); driverContext.getYieldSignal().setWithDelay(maxRuntime, driverContext.getYieldExecutor()); try { long start = System.nanoTime(); driverContext.getYieldSignal().reset(); driverContext.recordProcessed(operationTimer);
for (int i = 0; i < activeOperators.size() - 1 && !driverContext.isDone(); i++) { Operator current = activeOperators.get(i); Operator next = activeOperators.get(i + 1); driverContext.recordBlocked(blocked); List<StackTraceElement> interrupterStack = exclusiveLock.getInterrupterStack(); if (interrupterStack == null) { driverContext.failed(t); throw t; PrestoException newException = new PrestoException(GENERIC_INTERNAL_ERROR, "Driver was interrupted", exception); newException.addSuppressed(t); driverContext.failed(newException); throw newException;
@Override public SourceOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, ExchangeOperator.class.getSimpleName()); if (exchangeClient == null) { exchangeClient = exchangeClientSupplier.get(driverContext.getPipelineContext().localSystemMemoryContext()); } return new ExchangeOperator( operatorContext, sourceId, serdeFactory.createPagesSerde(), exchangeClient); }
.addDriverContext(); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT, REAL, DOUBLE, VARCHAR) .row(100L, 4950L, 49.5, "399", 100L, 54950L, 44950.0f, 54950.0, "599") .build(); assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0);
assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); assertEquals(driverContext.getMemoryUsage(), 0); assertEquals(driverContext.getSystemMemoryUsage(), 0); assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0);
@Test(invocationCount = 1_000, timeOut = 10_000) public void testConcurrentClose() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"); ValuesOperator source = new ValuesOperator(operatorContext, rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); // let these threads race scheduledExecutor.submit(() -> driver.processFor(new Duration(1, TimeUnit.NANOSECONDS))); // don't want to call isFinishedInternal in processFor scheduledExecutor.submit(() -> driver.close()); while (!driverContext.isDone()) { Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS); } }
public static OperatorContext create(ScheduledExecutorService scheduledExecutor) { Executor executor = MoreExecutors.directExecutor(); TaskContext taskContext = TestingTaskContext.createTaskContext( executor, scheduledExecutor, TestingSession.testSessionBuilder().build()); MemoryTrackingContext pipelineMemoryContext = new MemoryTrackingContext(newSimpleAggregatedMemoryContext(), newSimpleAggregatedMemoryContext(), newSimpleAggregatedMemoryContext()); PipelineContext pipelineContext = new PipelineContext( 1, taskContext, executor, scheduledExecutor, pipelineMemoryContext, false, false, false); DriverContext driverContext = new DriverContext( pipelineContext, executor, scheduledExecutor, pipelineMemoryContext, Lifespan.taskWide()); OperatorContext operatorContext = driverContext.addOperatorContext( 1, new PlanNodeId("test"), "operator type"); return operatorContext; }
public LocalMergeSourceOperator(OperatorContext operatorContext, List<LocalExchangeSource> sources, List<Type> types, PageWithPositionComparator comparator) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.sources = requireNonNull(sources, "sources is null"); List<WorkProcessor<Page>> pageProducers = sources.stream() .map(LocalExchangeSource::pages) .collect(toImmutableList()); mergedPages = mergeSortedPages( pageProducers, requireNonNull(comparator, "comparator is null"), types, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); }
SourceOperator operator = testPreparer.newTableScanOperator(driverContext); assertEquals(driverContext.getSystemMemoryUsage(), 0); page.getBlock(1); if (memoryUsage == -1) { memoryUsage = driverContext.getSystemMemoryUsage(); assertBetweenInclusive(memoryUsage, 460000L, 469999L); assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); page.getBlock(1); if (memoryUsage == -1) { memoryUsage = driverContext.getSystemMemoryUsage(); assertBetweenInclusive(memoryUsage, 460000L, 469999L); assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); page.getBlock(1); if (memoryUsage == -1) { memoryUsage = driverContext.getSystemMemoryUsage(); assertBetweenInclusive(memoryUsage, 360000L, 369999L); assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); assertNull(operator.getOutput()); assertTrue(operator.isFinished()); assertEquals(driverContext.getSystemMemoryUsage(), 0);
public boolean isDone() { return driverContext.isDone(); }
/** * Runs Driver in another thread until it is finished */ private static void runDriverInThread(ExecutorService executor, Driver driver) { executor.execute(() -> { if (!driver.isFinished()) { try { driver.process(); } catch (PrestoException e) { driver.getDriverContext().failed(e); throw e; } runDriverInThread(executor, driver); } }); }
if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getSystemMemoryReservation().toBytes() > 0) { greaterThanZero = true; break;
DriverStats driverStats = driverContext.getDriverStats(); drivers.add(driverStats); if (driverContext.isPartitioned()) { queuedPartitionedDrivers++; if (driverContext.isPartitioned()) { runningPartitionedDrivers++; totalBlockedTime += driverStats.getTotalBlockedTime().roundTo(NANOSECONDS); List<OperatorStats> operators = ImmutableList.copyOf(transform(driverContext.getOperatorContexts(), OperatorContext::getOperatorStats)); for (OperatorStats operator : operators) { runningOperators.put(operator.getOperatorId(), operator);