How to use

createDriver

method

in

com.facebook.presto.operator.Driver

@VisibleForTesting
public static Driver createDriver(DriverContext driverContext, Operator firstOperator, Operator... otherOperators)
{
  requireNonNull(driverContext, "driverContext is null");
  requireNonNull(firstOperator, "firstOperator is null");
  requireNonNull(otherOperators, "otherOperators is null");
  ImmutableList<Operator> operators = ImmutableList.<Operator>builder()
      .add(firstOperator)
      .add(otherOperators)
      .build();
  return createDriver(driverContext, operators);
}

public synchronized Driver createDriver(DriverContext driverContext)
{
  checkState(!closed, "DriverFactory is already closed");
  requireNonNull(driverContext, "driverContext is null");
  checkState(!closedLifespans.contains(driverContext.getLifespan()), "DriverFatory is already closed for driver group %s", driverContext.getLifespan());
  encounteredLifespans.add(driverContext.getLifespan());
  ImmutableList.Builder<Operator> operators = ImmutableList.builder();
  for (OperatorFactory operatorFactory : operatorFactories) {
    Operator operator = operatorFactory.createOperator(driverContext);
    operators.add(operator);
  }
  return Driver.createDriver(driverContext, operators.build());
}

private RevocableMemoryOperator setupConsumeRevocableMemory(DataSize reservedPerPage, long numberOfPages)
{
  AtomicReference<RevocableMemoryOperator> createOperator = new AtomicReference<>();
  setUp(() -> {
    DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
    OperatorContext revokableOperatorContext = driverContext.addOperatorContext(
        Integer.MAX_VALUE,
        new PlanNodeId("revokable_operator"),
        TableScanOperator.class.getSimpleName());
    OutputFactory outputFactory = new PageConsumerOutputFactory(types -> (page -> {}));
    Operator outputOperator = outputFactory.createOutputOperator(2, new PlanNodeId("output"), ImmutableList.of(), Function.identity(), new TestingPagesSerdeFactory()).createOperator(driverContext);
    RevocableMemoryOperator revocableMemoryOperator = new RevocableMemoryOperator(revokableOperatorContext, reservedPerPage, numberOfPages);
    createOperator.set(revocableMemoryOperator);
    Driver driver = Driver.createDriver(driverContext, revocableMemoryOperator, outputOperator);
    return ImmutableList.of(driver);
  });
  return createOperator.get();
}

private void instantiateBuildDrivers(BuildSideSetup buildSideSetup, TaskContext taskContext)
{
  PipelineContext buildPipeline = taskContext.addPipelineContext(1, true, true, false);
  List<Driver> buildDrivers = new ArrayList<>();
  List<HashBuilderOperator> buildOperators = new ArrayList<>();
  for (int i = 0; i < buildSideSetup.getPartitionCount(); i++) {
    DriverContext buildDriverContext = buildPipeline.addDriverContext();
    HashBuilderOperator buildOperator = buildSideSetup.getBuildOperatorFactory().createOperator(buildDriverContext);
    Driver driver = Driver.createDriver(
        buildDriverContext,
        buildSideSetup.getBuildSideSourceOperatorFactory().createOperator(buildDriverContext),
        buildOperator);
    buildDrivers.add(driver);
    buildOperators.add(buildOperator);
  }
  buildSideSetup.setDriversAndOperators(buildDrivers, buildOperators);
}

@Test
public void testBrokenOperatorCloseWhileProcessing()
    throws Exception
{
  BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), false);
  final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of()));
  assertSame(driver.getDriverContext(), driverContext);
  // block thread in operator processing
  Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>()
  {
    @Override
    public Boolean call()
    {
      return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
    }
  });
  brokenOperator.waitForLocked();
  driver.close();
  assertTrue(driver.isFinished());
  try {
    driverProcessFor.get(1, TimeUnit.SECONDS);
    fail("Expected InterruptedException");
  }
  catch (ExecutionException e) {
    assertDriverInterrupted(e.getCause());
  }
}

@Test(invocationCount = 1_000, timeOut = 10_000)
public void testConcurrentClose()
{
  List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
  OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "values");
  ValuesOperator source = new ValuesOperator(operatorContext, rowPagesBuilder(types)
      .addSequencePage(10, 20, 30, 40)
      .build());
  Operator sink = createSinkOperator(types);
  Driver driver = Driver.createDriver(driverContext, source, sink);
  // let these threads race
  scheduledExecutor.submit(() -> driver.processFor(new Duration(1, TimeUnit.NANOSECONDS))); // don't want to call isFinishedInternal in processFor
  scheduledExecutor.submit(() -> driver.close());
  while (!driverContext.isDone()) {
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
  }
}

@Test
public void testBrokenOperatorProcessWhileClosing()
    throws Exception
{
  BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true);
  final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of()));
  assertSame(driver.getDriverContext(), driverContext);
  // block thread in operator close
  Future<Boolean> driverClose = executor.submit(new Callable<Boolean>()
  {
    @Override
    public Boolean call()
    {
      driver.close();
      return true;
    }
  });
  brokenOperator.waitForLocked();
  assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
  assertTrue(driver.isFinished());
  brokenOperator.unlock();
  assertTrue(driverClose.get());
}

@Test
public void testAbruptFinish()
{
  List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
  ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types)
      .addSequencePage(10, 20, 30, 40)
      .build());
  PageConsumerOperator sink = createSinkOperator(types);
  Driver driver = Driver.createDriver(driverContext, source, sink);
  assertSame(driver.getDriverContext(), driverContext);
  assertFalse(driver.isFinished());
  driver.close();
  assertTrue(driver.isFinished());
  // finish is only called in normal operations
  assertFalse(source.isFinished());
  assertFalse(sink.isFinished());
  // close is always called (values operator doesn't have a closed state)
  assertTrue(sink.isClosed());
}

@Test
public void testNormalFinish()
{
  List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
  ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types)
      .addSequencePage(10, 20, 30, 40)
      .build());
  Operator sink = createSinkOperator(types);
  Driver driver = Driver.createDriver(driverContext, source, sink);
  assertSame(driver.getDriverContext(), driverContext);
  assertFalse(driver.isFinished());
  ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
  assertTrue(blocked.isDone());
  assertTrue(driver.isFinished());
  assertTrue(sink.isFinished());
  assertTrue(source.isFinished());
}

@Test
public void testMemoryRevocationRace()
{
  List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
  TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"),
      new PlanNodeId("source"),
      new PageSourceProvider()
      {
        @Override
        public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns)
        {
          return new FixedPageSource(rowPagesBuilder(types)
              .addSequencePage(10, 20, 30, 40)
              .build());
        }
      },
      ImmutableList.of());
  Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types));
  // the table scan operator will request memory revocation with requestMemoryRevoking()
  // while the driver is still not done with the processFor() method and before it moves to
  // updateDriverBlockedFuture() method.
  assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone());
}

Driver driver = Driver.createDriver(driverContext, source, sink);

final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);

ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build());
LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity());
Driver sourceDriver = Driver.createDriver(collectDriverContext,
    valuesOperatorFactory.createOperator(collectDriverContext),
    sinkOperatorFactory.createOperator(collectDriverContext));

  @Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*")
  public void testMemoryLimit(boolean hashEnabled)
  {
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE))
        .addPipelineContext(0, true, true, false)
        .addDriverContext();

    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
    List<Type> buildTypes = ImmutableList.of(BIGINT);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes);
    Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder
        .addSequencePage(10000, 20)
        .build());
    SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(
        1,
        new PlanNodeId("test"),
        buildTypes.get(0),
        0,
        rowPagesBuilder.getHashChannel(),
        10,
        new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()));
    Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);

    Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
    while (!driver.isFinished()) {
      driver.process();
    }
  }
}

    ImmutableList.of());
PageConsumerOperator sink = createSinkOperator(types);
Driver driver = Driver.createDriver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());

Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
  driver.process();

Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
  driver.process();

Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
  driver.process();

Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
while (!driver.isFinished()) {
  driver.process();

Driver driver = Driver.createDriver(driverContext,
    valuesOperator,
    nestedLoopBuildOperator);