Refine search
private static Page getAtMostOnePage(Operator operator, Page sourcePage) { // add our input page if needed if (operator.needsInput()) { operator.addInput(sourcePage); } // try to get the output page Page result = operator.getOutput(); // tell operator to finish operator.finish(); // try to get output until the operator is finished while (!operator.isFinished()) { // operator should never block assertTrue(operator.isBlocked().isDone()); Page output = operator.getOutput(); if (output != null) { assertNull(result); result = output; } } return result; }
private static void waitForFinished(Operator operator) throws InterruptedException { // wait for finished or until 10 seconds has passed long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() - endTime < 0) { assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); if (operator.isFinished()) { break; } Thread.sleep(10); } // verify final state assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); } }
for (Operator operator : operators) { try { operator.close(); t, "Error closing operator %s for task %s", operator.getOperatorContext().getOperatorId(), driverContext.getTaskId()); operator.getOperatorContext().destroy(); t, "Error freeing all allocated memory for operator %s for task %s", operator.getOperatorContext().getOperatorId(), driverContext.getTaskId());
@GuardedBy("exclusiveLock") private void checkOperatorFinishedRevoking(Operator operator) { ListenableFuture<?> future = revokingOperators.get(operator); if (future.isDone()) { getFutureValue(future); // propagate exception if there was some revokingOperators.remove(operator); operator.finishMemoryRevoke(); operator.getOperatorContext().resetMemoryRevokingRequested(); } }
private Optional<ListenableFuture<?>> getBlockedFuture(Operator operator) { ListenableFuture<?> blocked = revokingOperators.get(operator); if (blocked != null) { // We mark operator as blocked regardless of blocked.isDone(), because finishMemoryRevoke has not been called yet. return Optional.of(blocked); } blocked = operator.isBlocked(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForRevocableMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } return Optional.empty(); }
@GuardedBy("exclusiveLock") private void handleMemoryRevoke() { for (int i = 0; i < activeOperators.size() && !driverContext.isDone(); i++) { Operator operator = activeOperators.get(i); if (revokingOperators.containsKey(operator)) { checkOperatorFinishedRevoking(operator); } else if (operator.getOperatorContext().isMemoryRevokingRequested()) { ListenableFuture<?> future = operator.startMemoryRevoke(); revokingOperators.put(operator, future); checkOperatorFinishedRevoking(operator); } } }
@Test public void testRowNumberUnpartitionedLimit() throws Exception 0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), Ints.asList(), ImmutableList.<Type>of(), Optional.of(3), Optional.empty(), assertEquals(rowNumberColumn.getPositionCount(), 3); MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), ImmutableList.<Type>of(DOUBLE, BIGINT), pages); assertEquals(actual.getMaterializedRows().size(), 3); ImmutableSet<?> actualSet = ImmutableSet.copyOf(actual.getMaterializedRows()); ImmutableSet<?> expectedRowsSet = ImmutableSet.copyOf(expectedRows.getMaterializedRows()); assertEquals(Sets.intersection(expectedRowsSet, actualSet).size(), 3);
@Test public void testYield() })); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .pageBreak() RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); for (int i = 0; i < 10; i++) { probePages.row(stPoint(1 + 0.1 * i, 1 + 0.1 * i), "x" + i); assertTrue(operator.needsInput()); operator.addInput(probeInput.get(0)); operator.finish(); assertNull(operator.getOutput()); assertEquals(filterFunctionCalls.get(), i + 1, "Expected join to stop processing (yield) after calling filter function once"); driverContext.getYieldSignal().reset(); Page output = operator.getOutput(); assertNotNull(output);
@Test(dataProvider = "hashJoinTestValues") public void testLookupOuterJoinWithEmptyLookupSource(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) List<Type> buildTypes = ImmutableList.of(VARCHAR); RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes); BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY); List<Type> probeTypes = ImmutableList.of(VARCHAR); RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes); OperatorFactory joinOperatorFactory = new LookupJoinOperators().lookupOuterJoin( operator.addInput(pages.get(0)); Page outputPage = operator.getOutput(); assertNull(outputPage);
assertTrue(operator.isBlocked().isDone()); assertFalse(operator.isFinished()); assertTrue(operator.needsInput()); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); assertFalse(operator.isBlocked().isDone()); assertFalse(operator.isFinished()); assertFalse(operator.needsInput()); assertNull(operator.getOutput()); assertTrue(operator.isBlocked().isDone()); assertFalse(operator.isFinished()); assertTrue(operator.needsInput()); operator.addInput(rowPagesBuilder(BIGINT).row(44).build().get(0)); assertFalse(operator.isBlocked().isDone()); assertFalse(operator.isFinished()); assertFalse(operator.needsInput()); operator.finish(); assertFalse(operator.isBlocked().isDone()); assertFalse(operator.isFinished()); assertFalse(operator.needsInput()); List<Type> expectedTypes = ImmutableList.of(BIGINT, VARBINARY); assertPageEquals(expectedTypes,
assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); page = operator.getOutput(); if (page != null) { break; assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); operator.finish(); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertNotNull(operator.getOutput()); assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null);
@Test public void testNormalFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testLimitZero() throws Exception { List<Page> input = rowPagesBuilder(BIGINT).row(1).build(); TopNOperatorFactory factory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT), 0, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST), false); Operator operator = factory.createOperator(driverContext); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).build(); // assertOperatorEquals assumes operators do not start in finished state assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); List<Page> pages = OperatorAssertion.toPages(operator, input.iterator()); MaterializedResult actual = OperatorAssertion.toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); assertEquals(actual, expected); }
@Test public void testExceedMemoryLimit() throws Exception { List<Page> input = rowPagesBuilder(BIGINT) .row(1L) .build(); DriverContext smallDiverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(1, BYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); TopNOperatorFactory operatorFactory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT), 100, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_LAST)); try (Operator operator = operatorFactory.createOperator(smallDiverContext)) { operator.addInput(input.get(0)); fail("must fail because of exceeding local memory limit"); } catch (ExceededMemoryLimitException ignore) { } } }
new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), useSystemMemory); assertTrue(operator.needsInput()); operator.addInput(input); assertEquals(driverContext.getMemoryUsage(), 0); assertEquals(driverContext.getSystemMemoryUsage(), 0); assertThat(driverContext.getMemoryUsage()).isGreaterThan(0);
@Test public void testLimitZero() throws Exception { List<Page> input = rowPagesBuilder(BIGINT).row(1L).build(); TopNOperatorFactory factory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT), 0, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST)); try (Operator operator = factory.createOperator(driverContext)) { assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); } }
public static void assertOperatorEqualsIgnoreOrder(Operator operator, List<Page> input, MaterializedResult expected, boolean hashEnabled, Optional<Integer> hashChannel) { List<Page> pages = toPages(operator, input); MaterializedResult actual; if (hashEnabled && hashChannel.isPresent()) { // Drop the hashChannel for all pages List<Page> actualPages = dropChannel(pages, ImmutableList.of(hashChannel.get())); List<Type> expectedTypes = without(operator.getTypes(), ImmutableList.of(hashChannel.get())); actual = toMaterializedResult(operator.getOperatorContext().getSession(), expectedTypes, actualPages); } else { actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); } assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); }
private static MaterializedResult getProperColumns(Operator joinOperator, List<Type> types, RowPagesBuilder probePages, List<Page> actualPages) { if (probePages.getHashChannel().isPresent()) { List<Integer> hashChannels = ImmutableList.of(probePages.getHashChannel().get()); actualPages = dropChannel(actualPages, hashChannels); types = without(types, hashChannels); } return OperatorAssertion.toMaterializedResult(joinOperator.getOperatorContext().getSession(), types, actualPages); }
public static List<Page> toPages(Operator operator) { // operator does not have input so should never require input assertEquals(operator.needsInput(), false); ImmutableList.Builder<Page> outputPages = ImmutableList.builder(); addRemainingOutputPages(operator, outputPages); return outputPages.build(); }
assertEquals(operator.isBlocked().isDone(), true); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), true); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); assertEquals(operator.isBlocked().isDone(), false); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); assertEquals(operator.isBlocked().isDone(), true); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), true); operator.addInput(rowPagesBuilder(BIGINT).row(44).build().get(0)); assertEquals(operator.isBlocked().isDone(), false); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); operator.finish(); assertEquals(operator.isBlocked().isDone(), false); assertEquals(operator.isFinished(), false); assertEquals(operator.needsInput(), false); operator.getOutput(), rowPagesBuilder(TableWriterOperator.TYPES).row(2, null).build().get(0));