@Override public synchronized boolean isFinished() { if (closed) { return true; } return driver != null && driver.isFinished(); }
private boolean loadNextPage() { Page nextPage = extractNonEmptyPage(pageBuffer); while (nextPage == null) { if (driver.isFinished()) { return false; } driver.process(); nextPage = extractNonEmptyPage(pageBuffer); } currentPage = nextPage; return true; }
private long runDriversUntilBlocked(Predicate<OperatorContext> reason) { long iterationsCount = 0; // run driver, until it blocks while (!isOperatorBlocked(drivers, reason)) { for (Driver driver : drivers) { driver.process(); } iterationsCount++; } // driver should be blocked waiting for memory for (Driver driver : drivers) { assertFalse(driver.isFinished()); } return iterationsCount; }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
if (!driver.isFinished()) { driver.process(); processed = true;
/** * Runs Driver in another thread until it is finished */ private static void runDriverInThread(ExecutorService executor, Driver driver) { executor.execute(() -> { if (!driver.isFinished()) { try { driver.process(); } catch (PrestoException e) { driver.getDriverContext().failed(e); throw e; } runDriverInThread(executor, driver); } }); }
/** * Runs Driver in another thread until it is finished */ private static void runDriverInThread(ExecutorService executor, Driver driver) { executor.execute(() -> { if (!driver.isFinished()) { try { driver.process(); } catch (PrestoException e) { driver.getDriverContext().failed(e); throw e; } runDriverInThread(executor, driver); } }); }
ScheduledSplit split = new ScheduledSplit(0, sourcePlanNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(recordSetForLookupSource))); driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(split), true)); while (!driver.isFinished()) { ListenableFuture<?> process = driver.process(); checkState(process.isDone(), "Driver should never block");
@Test public void testNormalFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
assertFalse(driver.isFinished()); assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertTrue(driver.isFinished());
@Test public void testBrokenOperatorCloseWhileProcessing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), false); final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of())); assertSame(driver.getDriverContext(), driverContext); // block thread in operator processing Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() { @Override public Boolean call() { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } }); brokenOperator.waitForLocked(); driver.close(); assertTrue(driver.isFinished()); try { driverProcessFor.get(1, TimeUnit.SECONDS); fail("Expected InterruptedException"); } catch (ExecutionException e) { assertDriverInterrupted(e.getCause()); } }
@Test public void testAbruptFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); PageConsumerOperator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); // finish is only called in normal operations assertFalse(source.isFinished()); assertFalse(sink.isFinished()); // close is always called (values operator doesn't have a closed state) assertTrue(sink.isClosed()); }
@Test public void testBrokenOperatorProcessWhileClosing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true); final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of())); assertSame(driver.getDriverContext(), driverContext); // block thread in operator close Future<Boolean> driverClose = executor.submit(new Callable<Boolean>() { @Override public Boolean call() { driver.close(); return true; } }); brokenOperator.waitForLocked(); assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertTrue(driver.isFinished()); brokenOperator.unlock(); assertTrue(driverClose.get()); }
assertFalse(driver.isFinished()); assertFalse(driver.isFinished()); assertFalse(driver.isFinished()); assertFalse(driver.isFinished()); assertTrue(driver.isFinished());
@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*") public void testMemoryLimit(boolean hashEnabled) { DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName()); List<Type> buildTypes = ImmutableList.of(BIGINT); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes); Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder .addSequencePage(10000, 20) .build()); SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory( 1, new PlanNodeId("test"), buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig())); Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process(); } } }
Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); Split testSplit = new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new TestSplit()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
while (!driver.isFinished()) { driver.process();
while (!driver.isFinished()) { driver.process();
while (!driver.isFinished()) { driver.process();
boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); processed = true;