private boolean loadNextPage() { Page nextPage = extractNonEmptyPage(pageBuffer); while (nextPage == null) { if (driver.isFinished()) { return false; } driver.process(); nextPage = extractNonEmptyPage(pageBuffer); } currentPage = nextPage; return true; }
private void assertDriversProgress(Predicate<OperatorContext> reason) { do { assertFalse(isOperatorBlocked(drivers, reason)); boolean progress = false; for (Driver driver : drivers) { ListenableFuture<?> blocked = driver.process(); progress = progress | blocked.isDone(); } // query should not block assertTrue(progress); } while (!drivers.stream().allMatch(Driver::isFinished)); }
private long runDriversUntilBlocked(Predicate<OperatorContext> reason) { long iterationsCount = 0; // run driver, until it blocks while (!isOperatorBlocked(drivers, reason)) { for (Driver driver : drivers) { driver.process(); } iterationsCount++; } // driver should be blocked waiting for memory for (Driver driver : drivers) { assertFalse(driver.isFinished()); } return iterationsCount; }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
driver.process(); processed = true;
/** * Runs Driver in another thread until it is finished */ private static void runDriverInThread(ExecutorService executor, Driver driver) { executor.execute(() -> { if (!driver.isFinished()) { try { driver.process(); } catch (PrestoException e) { driver.getDriverContext().failed(e); throw e; } runDriverInThread(executor, driver); } }); }
/** * Runs Driver in another thread until it is finished */ private static void runDriverInThread(ExecutorService executor, Driver driver) { executor.execute(() -> { if (!driver.isFinished()) { try { driver.process(); } catch (PrestoException e) { driver.getDriverContext().failed(e); throw e; } runDriverInThread(executor, driver); } }); }
driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(split), true)); while (!driver.isFinished()) { ListenableFuture<?> process = driver.process(); checkState(process.isDone(), "Driver should never block");
private void buildLookupSource(BuildSideSetup buildSideSetup) { requireNonNull(buildSideSetup, "buildSideSetup is null"); LookupSourceFactory lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager().getJoinBridge(Lifespan.taskWide()); Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider(); List<Driver> buildDrivers = buildSideSetup.getBuildDrivers(); while (!lookupSourceProvider.isDone()) { for (Driver buildDriver : buildDrivers) { buildDriver.process(); } } getFutureValue(lookupSourceProvider).close(); for (Driver buildDriver : buildDrivers) { runDriverInThread(executor, buildDriver); } }
private static void processRow(final Driver joinDriver, final TaskStateMachine taskStateMachine) { joinDriver.getDriverContext().getYieldSignal().setWithDelay(TimeUnit.SECONDS.toNanos(1), joinDriver.getDriverContext().getYieldExecutor()); joinDriver.process(); joinDriver.getDriverContext().getYieldSignal().reset(); checkErrors(taskStateMachine); }
sourceDriver.process();
@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*") public void testMemoryLimit(boolean hashEnabled) { DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName()); List<Type> buildTypes = ImmutableList.of(BIGINT); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes); Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder .addSequencePage(10000, 20) .build()); SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory( 1, new PlanNodeId("test"), buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig())); Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process(); } } }
driver.process();
driver.process();
driver.process();
driver.process();
for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); processed = true;
driver.process();
protected void execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); processed = true; } } done = !processed; } }
private boolean loadNextPage() { Page nextPage = extractNonEmptyPage(pageBuffer); while (nextPage == null) { if (driver.isFinished()) { return false; } driver.process(); nextPage = extractNonEmptyPage(pageBuffer); } currentPage = nextPage; return true; }