@Test public void testFinish() throws Exception { SourceOperator operator = createExchangeOperator(); operator.addSplit(newRemoteSplit(TASK_1_ID)); operator.addSplit(newRemoteSplit(TASK_2_ID)); operator.addSplit(newRemoteSplit(TASK_3_ID)); operator.noMoreSplits(); // add pages and leave buffers open taskBuffers.getUnchecked(TASK_1_ID).addPages(1, false); taskBuffers.getUnchecked(TASK_2_ID).addPages(1, false); taskBuffers.getUnchecked(TASK_3_ID).addPages(1, false); // read 3 pages waitForPages(operator, 3); // verify state assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); // finish without closing buffers operator.finish(); // wait for finished waitForFinished(operator); }
private Driver(DriverContext driverContext, List<Operator> operators) { this.driverContext = requireNonNull(driverContext, "driverContext is null"); this.allOperators = ImmutableList.copyOf(requireNonNull(operators, "operators is null")); checkArgument(allOperators.size() > 1, "At least two operators are required"); this.activeOperators = new ArrayList<>(operators); checkArgument(!operators.isEmpty(), "There must be at least one operator"); Optional<SourceOperator> sourceOperator = Optional.empty(); Optional<DeleteOperator> deleteOperator = Optional.empty(); for (Operator operator : operators) { if (operator instanceof SourceOperator) { checkArgument(!sourceOperator.isPresent(), "There must be at most one SourceOperator"); sourceOperator = Optional.of((SourceOperator) operator); } else if (operator instanceof DeleteOperator) { checkArgument(!deleteOperator.isPresent(), "There must be at most one DeleteOperator"); deleteOperator = Optional.of((DeleteOperator) operator); } } this.sourceOperator = sourceOperator; this.deleteOperator = deleteOperator; currentTaskSource = sourceOperator.map(operator -> new TaskSource(operator.getSourceId(), ImmutableSet.of(), false)).orElse(null); // initially the driverBlockedFuture is not blocked (it is completed) SettableFuture<?> future = SettableFuture.create(); future.set(null); driverBlockedFuture.set(future); }
Split split = newSplit.getSplit(); Supplier<Optional<UpdatablePageSource>> pageSource = sourceOperator.addSplit(split); deleteOperator.ifPresent(deleteOperator -> deleteOperator.setPageSource(pageSource)); sourceOperator.noMoreSplits();
int totalRows = 0; while (totalRows < 20000) { assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); assertNull(operator.getOutput()); assertTrue(operator.isFinished()); assertEquals(driverContext.getSystemMemoryUsage(), 0);
operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); operator.noMoreSplits(); Page page = operator.getOutput(); if (i == totalColumns) { assertNotNull(page);
public SourceOperator newTableScanOperator(DriverContext driverContext) { ConnectorPageSource pageSource = newPageSource(); SourceOperatorFactory sourceOperatorFactory = new TableScanOperatorFactory( 0, new PlanNodeId("0"), (session, split, columnHandles) -> pageSource, columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList())); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); return operator; }
private SourceOperator createExchangeOperator() { ExchangeOperatorFactory operatorFactory = new ExchangeOperatorFactory(0, new PlanNodeId("test"), exchangeClientSupplier, SERDE_FACTORY); DriverContext driverContext = createTaskContext(scheduler, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) .addDriverContext(); SourceOperator operator = operatorFactory.createOperator(driverContext); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); return operator; }
private Object selectSingleValue(SourceOperatorFactory operatorFactory, Type type, Split split, Session session) { SourceOperator operator = operatorFactory.createOperator(createDriverContext(session)); operator.addSplit(split); operator.noMoreSplits(); return selectSingleValue(operator, type); }
operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); operator.noMoreSplits(); assertNull(operator.getOutput()); driverContext.getYieldSignal().reset(); Page output = operator.getOutput(); driverContext.getYieldSignal().reset(); assertNotNull(output);
assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); assertNull(operator.getOutput()); assertTrue(operator.isFinished()); assertEquals(driverContext.getSystemMemoryUsage(), 0);
public SourceOperator newScanFilterAndProjectOperator(DriverContext driverContext) { ConnectorPageSource pageSource = newPageSource(); ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); for (int i = 0; i < types.size(); i++) { projectionsBuilder.add(field(i, types.get(i))); } Supplier<CursorProcessor> cursorProcessor = EXPRESSION_COMPILER.compileCursorProcessor(Optional.empty(), projectionsBuilder.build(), "key"); Supplier<PageProcessor> pageProcessor = EXPRESSION_COMPILER.compilePageProcessor(Optional.empty(), projectionsBuilder.build()); SourceOperatorFactory sourceOperatorFactory = new ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, columnHandles) -> pageSource, cursorProcessor, pageProcessor, columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList()), types, new DataSize(0, BYTE), 0); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); return operator; }
private SourceOperator createExchangeOperator() { ExchangeOperatorFactory operatorFactory = new ExchangeOperatorFactory(0, new PlanNodeId("test"), exchangeClientSupplier, TYPES); DriverContext driverContext = createTaskContext(executor, TEST_SESSION) .addPipelineContext(true, true) .addDriverContext(); SourceOperator operator = operatorFactory.createOperator(driverContext); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); return operator; }
@Test public void testWaitForNoMoreSplits() throws Exception { SourceOperator operator = createExchangeOperator(); // add a buffer location containing one page and close the buffer operator.addSplit(newRemoteSplit(TASK_1_ID)); // add pages and leave buffers open taskBuffers.getUnchecked(TASK_1_ID).addPages(1, true); // read page waitForPages(operator, 1); // verify state assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); // add a buffer location operator.addSplit(newRemoteSplit(TASK_2_ID)); // set no more splits (buffer locations) operator.noMoreSplits(); // add two pages and close the last buffer taskBuffers.getUnchecked(TASK_2_ID).addPages(2, true); // read all pages waitForPages(operator, 2); // wait for finished waitForFinished(operator); }
private static boolean executeFilter(SourceOperatorFactory operatorFactory, Split split, Session session) { SourceOperator operator = operatorFactory.createOperator(createDriverContext(session)); operator.addSplit(split); operator.noMoreSplits(); return executeFilter(operator); }
assertFalse(operator.isFinished()); operator.getOutput(); assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 550000L, 639999L); assertFalse(operator.isFinished()); operator.getOutput(); assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 450000L, 539999L); assertFalse(operator.isFinished()); assertNull(operator.getOutput()); assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 110000L, 119999L); assertFalse(operator.isFinished()); Page lastPage = operator.getOutput(); assertNotNull(lastPage); assertTrue(operator.isFinished());
public SourceOperator newTableScanOperator(DriverContext driverContext) { ConnectorPageSource pageSource = newPageSource(); SourceOperatorFactory sourceOperatorFactory = new TableScanOperatorFactory( 0, new PlanNodeId("0"), (session, split, columnHandles) -> pageSource, types, columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList()) ); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); operator.addSplit(new Split("test", TestingTransactionHandle.create("test"), TestingSplit.createLocalSplit())); return operator; }
public void updateSource(TaskSource sourceUpdate) { checkLockNotHeld("Can not update sources while holding the driver lock"); checkArgument( sourceOperator.isPresent() && sourceOperator.get().getSourceId().equals(sourceUpdate.getPlanNodeId()), "sourceUpdate is for a plan node that is different from this Driver's source node"); // stage the new updates pendingTaskSourceUpdates.updateAndGet(current -> current == null ? sourceUpdate : current.update(sourceUpdate)); // attempt to get the lock and process the updates we staged above // updates will be processed in close if and only if we got the lock tryWithLock(() -> TRUE); }
@Test public void testWaitForClose() throws Exception { SourceOperator operator = createExchangeOperator(); operator.addSplit(newRemoteSplit(TASK_1_ID)); operator.addSplit(newRemoteSplit(TASK_2_ID)); operator.addSplit(newRemoteSplit(TASK_3_ID)); operator.noMoreSplits(); // add pages and leave buffers open taskBuffers.getUnchecked(TASK_1_ID).addPages(1, false); taskBuffers.getUnchecked(TASK_2_ID).addPages(1, false); taskBuffers.getUnchecked(TASK_3_ID).addPages(1, false); // read 3 pages waitForPages(operator, 3); // verify state assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); // add more pages and close the buffers taskBuffers.getUnchecked(TASK_1_ID).addPages(2, true); taskBuffers.getUnchecked(TASK_2_ID).addPages(2, true); taskBuffers.getUnchecked(TASK_3_ID).addPages(2, true); // read all pages waitForPages(operator, 6); // wait for finished waitForFinished(operator); }
@Test public void testSimple() throws Exception { SourceOperator operator = createExchangeOperator(); operator.addSplit(newRemoteSplit(TASK_1_ID)); operator.addSplit(newRemoteSplit(TASK_2_ID)); operator.addSplit(newRemoteSplit(TASK_3_ID)); operator.noMoreSplits(); // add pages and close the buffers taskBuffers.getUnchecked(TASK_1_ID).addPages(10, true); taskBuffers.getUnchecked(TASK_2_ID).addPages(10, true); taskBuffers.getUnchecked(TASK_3_ID).addPages(10, true); // read the pages waitForPages(operator, 30); // wait for finished waitForFinished(operator); }
int totalRows = 0; while (totalRows < 20000) { assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); Page page = operator.getOutput(); assertNotNull(page); page.getBlock(1); assertFalse(operator.isFinished()); assertNull(operator.getOutput()); assertTrue(operator.isFinished()); assertEquals(driverContext.getSystemMemoryUsage(), 0);