@GuardedBy("exclusiveLock") private void checkOperatorFinishedRevoking(Operator operator) { ListenableFuture<?> future = revokingOperators.get(operator); if (future.isDone()) { getFutureValue(future); // propagate exception if there was some revokingOperators.remove(operator); operator.finishMemoryRevoke(); operator.getOperatorContext().resetMemoryRevokingRequested(); } }
private Optional<ListenableFuture<?>> getBlockedFuture(Operator operator) { ListenableFuture<?> blocked = revokingOperators.get(operator); if (blocked != null) { // We mark operator as blocked regardless of blocked.isDone(), because finishMemoryRevoke has not been called yet. return Optional.of(blocked); } blocked = operator.isBlocked(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForRevocableMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } return Optional.empty(); }
@GuardedBy("exclusiveLock") private void handleMemoryRevoke() { for (int i = 0; i < activeOperators.size() && !driverContext.isDone(); i++) { Operator operator = activeOperators.get(i); if (revokingOperators.containsKey(operator)) { checkOperatorFinishedRevoking(operator); } else if (operator.getOperatorContext().isMemoryRevokingRequested()) { ListenableFuture<?> future = operator.startMemoryRevoke(); revokingOperators.put(operator, future); checkOperatorFinishedRevoking(operator); } } }
t, "Error closing operator %s for task %s", operator.getOperatorContext().getOperatorId(), driverContext.getTaskId()); operator.getOperatorContext().destroy(); t, "Error freeing all allocated memory for operator %s for task %s", operator.getOperatorContext().getOperatorId(), driverContext.getTaskId());
Operator rootOperator = activeOperators.get(0); rootOperator.finish(); rootOperator.getOperatorContext().recordFinish(operationTimer); current.getOperatorContext().recordGetOutput(operationTimer, page); next.getOperatorContext().recordAddInput(operationTimer, page); movedPage = true; next.getOperatorContext().recordFinish(operationTimer); Operator newRootOperator = activeOperators.get(0); newRootOperator.finish(); newRootOperator.getOperatorContext().recordFinish(operationTimer); operator.getOperatorContext().recordBlocked(blocked);
private static MaterializedResult getProperColumns(Operator joinOperator, List<Type> types, RowPagesBuilder probePages, List<Page> actualPages) { if (probePages.getHashChannel().isPresent()) { List<Integer> hashChannels = ImmutableList.of(probePages.getHashChannel().get()); actualPages = dropChannel(actualPages, hashChannels); types = without(types, hashChannels); } return OperatorAssertion.toMaterializedResult(joinOperator.getOperatorContext().getSession(), types, actualPages); }
private static void waitForFinished(Operator operator) throws InterruptedException { // wait for finished or until 10 seconds has passed long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() - endTime < 0) { assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); if (operator.isFinished()) { break; } Thread.sleep(10); } // verify final state assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); } }
private static void handleMemoryRevoking(Operator operator) { if (operator.getOperatorContext().getReservedRevocableBytes() > 0) { getFutureValue(operator.startMemoryRevoke()); operator.finishMemoryRevoke(); } }
if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getSystemMemoryReservation().toBytes() > 0) { greaterThanZero = true; break; assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
memoryPool.reserve(queryId, "test", reservedMemoryInBytes); long oldMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); int oldCapacity = getHashCapacity.apply(operator); long newMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); assertTrue(operator.getOperatorContext().isWaitingForMemory().isDone()); assertFalse(operator.getOperatorContext().isWaitingForMemory().isDone()); long rehashedMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); assertBetweenInclusive(rehashedMemoryUsage * 1.0 / newMemoryUsage, 0.99, 1.01);
private static ListenableFuture<?> isBlocked(Operator operator) { ListenableFuture<?> blocked = operator.isBlocked(); if (blocked.isDone()) { blocked = operator.getOperatorContext().isWaitingForMemory(); } return blocked; }
public static void assertOperatorEquals(Operator operator, MaterializedResult expected) { List<Page> pages = toPages(operator); MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); assertEquals(actual, expected); }
private void waitForFinished(Operator operator) throws InterruptedException { // wait for finished or until 10 seconds has passed long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() < endTime) { assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); if (operator.isFinished()) { break; } Thread.sleep(10); } // verify final state assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); }
public static void assertOperatorEqualsIgnoreOrder(Operator operator, List<Page> input, MaterializedResult expected, boolean hashEnabled, Optional<Integer> hashChannel) { List<Page> pages = toPages(operator, input); MaterializedResult actual; if (hashEnabled && hashChannel.isPresent()) { // Drop the hashChannel for all pages List<Page> actualPages = dropChannel(pages, ImmutableList.of(hashChannel.get())); List<Type> expectedTypes = without(operator.getTypes(), ImmutableList.of(hashChannel.get())); actual = toMaterializedResult(operator.getOperatorContext().getSession(), expectedTypes, actualPages); } else { actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); } assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); }
public static void assertOperatorEquals(Operator operator, List<Page> input, MaterializedResult expected, boolean hashEnabled, List<Integer> hashChannels) { List<Page> pages = toPages(operator, input); MaterializedResult actual; if (hashEnabled && !hashChannels.isEmpty()) { // Drop the hashChannel for all pages List<Page> actualPages = dropChannel(pages, hashChannels); List<Type> expectedTypes = without(operator.getTypes(), hashChannels); actual = toMaterializedResult(operator.getOperatorContext().getSession(), expectedTypes, actualPages); } else { actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); } assertEquals(actual, expected); }
if (operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes() > 0) { greaterThanZero = true; break; assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
@Test public void testLimitZero() throws Exception { List<Page> input = rowPagesBuilder(BIGINT).row(1).build(); TopNOperatorFactory factory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT), 0, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST), false); Operator operator = factory.createOperator(driverContext); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).build(); // assertOperatorEquals assumes operators do not start in finished state assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertEquals(operator.getOutput(), null); List<Page> pages = OperatorAssertion.toPages(operator, input.iterator()); MaterializedResult actual = OperatorAssertion.toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); assertEquals(actual, expected); }
MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages);
MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), ImmutableList.<Type>of(DOUBLE, BIGINT), pages); assertEquals(actual.getMaterializedRows().size(), 3); ImmutableSet<?> actualSet = ImmutableSet.copyOf(actual.getMaterializedRows());
MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), ImmutableList.<Type>of(DOUBLE, BIGINT), pages); assertEqualsIgnoreOrder(actual.getMaterializedRows(), expectedResult.getMaterializedRows());