private static OptionalInt getHashChannelAsInt(RowPagesBuilder probePages) { return probePages.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()); }
private static List<Integer> getHashChannels(RowPagesBuilder probe, RowPagesBuilder build) { ImmutableList.Builder<Integer> hashChannels = ImmutableList.builder(); if (probe.getHashChannel().isPresent()) { hashChannels.add(probe.getHashChannel().get()); } if (build.getHashChannel().isPresent()) { hashChannels.add(probe.getTypes().size() + build.getHashChannel().get()); } return hashChannels.build(); }
private static MaterializedResult getProperColumns(Operator joinOperator, List<Type> types, RowPagesBuilder probePages, List<Page> actualPages) { if (probePages.getHashChannel().isPresent()) { List<Integer> hashChannels = ImmutableList.of(probePages.getHashChannel().get()); actualPages = dropChannel(actualPages, hashChannels); types = without(types, hashChannels); } return OperatorAssertion.toMaterializedResult(joinOperator.getOperatorContext().getSession(), types, actualPages); }
protected void initializeBuildPages() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(buildHashEnabled, hashChannels, ImmutableList.of(VARCHAR, BIGINT, BIGINT)); int maxValue = BUILD_ROWS_NUMBER / buildRowsRepetition + 40; int rows = 0; while (rows < BUILD_ROWS_NUMBER) { int newRows = Math.min(BUILD_ROWS_NUMBER - rows, ROWS_PER_PAGE); buildPagesBuilder.addSequencePage(newRows, (rows + 20) % maxValue, (rows + 30) % maxValue, (rows + 40) % maxValue); buildPagesBuilder.pageBreak(); rows += newRows; } types = buildPagesBuilder.getTypes(); buildPages = buildPagesBuilder.build(); hashChannel = buildPagesBuilder.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()); } }
@Setup public void setup() { executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); int groupsPerPage = ROWS_PER_PAGE / rowsPerGroup; boolean hashAggregation = operatorType.equalsIgnoreCase("hash"); RowPagesBuilder pagesBuilder = RowPagesBuilder.rowPagesBuilder(hashAggregation, ImmutableList.of(0), VARCHAR, BIGINT); for (int i = 0; i < TOTAL_PAGES; i++) { BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, ROWS_PER_PAGE); for (int j = 0; j < groupsPerPage; j++) { String groupKey = format("%s", i * groupsPerPage + j); repeatToStringBlock(groupKey, rowsPerGroup, blockBuilder); } pagesBuilder.addBlocksPage(blockBuilder.build(), createLongSequenceBlock(0, ROWS_PER_PAGE)); } pages = pagesBuilder.build(); if (hashAggregation) { operatorFactory = createHashAggregationOperatorFactory(pagesBuilder.getHashChannel()); } else { operatorFactory = createStreamingAggregationOperatorFactory(); } }
false, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 1,
@Test(dataProvider = "hashEnabledValues") public void testMarkDistinct(boolean hashEnabled) { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(100, 0) .addSequencePage(100, 0) .build(); OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), ImmutableList.of(0), rowPagesBuilder.getHashChannel(), joinCompiler); MaterializedResult.Builder expected = resultBuilder(driverContext.getSession(), BIGINT, BOOLEAN); for (long i = 0; i < 100; i++) { expected.row(i, true); expected.row(i, false); } OperatorAssertion.assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected.build(), hashEnabled, Optional.of(1)); }
buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), UNGROUPED_EXECUTION, new DataSize(32, DataSize.Unit.MEGABYTE)); rangeList(buildPages.getTypes().size()), hashChannels, buildPages.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()), filterFunctionFactory,
@Test(dataProvider = "hashEnabledValues") public void testDistinctLimitWithPageAlignment(boolean hashEnabled) { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(3, 1) .addSequencePage(3, 2) .build(); OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), Ints.asList(0), 3, rowPagesBuilder.getHashChannel(), joinCompiler); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT) .row(1L) .row(2L) .row(3L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected, hashEnabled, ImmutableList.of(1)); }
@Test(dataProvider = "hashEnabledValues") public void testDistinctLimitValuesLessThanLimit(boolean hashEnabled) { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(3, 1) .addSequencePage(3, 2) .build(); OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), Ints.asList(0), 5, rowPagesBuilder.getHashChannel(), joinCompiler); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT) .row(1L) .row(2L) .row(3L) .row(4L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected, hashEnabled, ImmutableList.of(1)); }
@Test(dataProvider = "hashEnabledValues") public void testDistinctLimit(boolean hashEnabled) { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(3, 1) .addSequencePage(5, 2) .build(); OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), Ints.asList(0), 5, rowPagesBuilder.getHashChannel(), joinCompiler); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT) .row(1L) .row(2L) .row(3L) .row(4L) .row(5L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected, hashEnabled, ImmutableList.of(1)); }
@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*") public void testMemoryLimit(boolean hashEnabled) { DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName()); List<Type> buildTypes = ImmutableList.of(BIGINT); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes); Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder .addSequencePage(10000, 20) .build()); SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory( 1, new PlanNodeId("test"), buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig())); Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process(); } } }
ImmutableList.of(BIGINT), Optional.of(10), rowPagesBuilder.getHashChannel(), 10, joinCompiler);
buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()));
buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()));
buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()));
rowPagesBuilder.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()));
private static List<Integer> getHashChannels(RowPagesBuilder probe, RowPagesBuilder build) { ImmutableList.Builder<Integer> hashChannels = ImmutableList.builder(); if (probe.getHashChannel().isPresent()) { hashChannels.add(probe.getHashChannel().get()); } if (build.getHashChannel().isPresent()) { hashChannels.add(probe.getTypes().size() + build.getHashChannel().get()); } return hashChannels.build(); }
@Test(dataProvider = "hashEnabledValues") public void testMarkDistinct(boolean hashEnabled) throws Exception { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(100, 0) .addSequencePage(100, 0) .build(); OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), ImmutableList.of(0), rowPagesBuilder.getHashChannel()); Operator operator = operatorFactory.createOperator(driverContext); MaterializedResult.Builder expected = resultBuilder(driverContext.getSession(), BIGINT, BOOLEAN); for (int i = 0; i < 100; i++) { expected.row(i, true); expected.row(i, false); } OperatorAssertion.assertOperatorEqualsIgnoreOrder(operator, input, expected.build(), hashEnabled, Optional.of(1)); } }
@Test(dataProvider = "hashEnabledValues") public void testDistinctLimitWithPageAlignment(boolean hashEnabled) throws Exception { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT); List<Page> input = rowPagesBuilder .addSequencePage(3, 1) .addSequencePage(3, 2) .build(); OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), Ints.asList(0), 3, rowPagesBuilder.getHashChannel()); Operator operator = operatorFactory.createOperator(driverContext); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT) .row(1) .row(2) .row(3) .build(); OperatorAssertion.assertOperatorEquals(operator, input, expected); }