protected void initializeBuildPages() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(buildHashEnabled, hashChannels, ImmutableList.of(VARCHAR, BIGINT, BIGINT)); int maxValue = BUILD_ROWS_NUMBER / buildRowsRepetition + 40; int rows = 0; while (rows < BUILD_ROWS_NUMBER) { int newRows = Math.min(BUILD_ROWS_NUMBER - rows, ROWS_PER_PAGE); buildPagesBuilder.addSequencePage(newRows, (rows + 20) % maxValue, (rows + 30) % maxValue, (rows + 40) % maxValue); buildPagesBuilder.pageBreak(); rows += newRows; } types = buildPagesBuilder.getTypes(); buildPages = buildPagesBuilder.build(); hashChannel = buildPagesBuilder.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()); } }
protected void generateRandomInputPage() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(types); for (int i = 0; i < PAGE_SIZE; i++) { Object[] values = types.stream() .map(this::generateRandomValue) .collect(toList()).toArray(); buildPagesBuilder.row(values); } inputPage = getOnlyElement(buildPagesBuilder.build()); }
public static RowPagesBuilder rowPagesBuilder(Iterable<Type> types) { return new RowPagesBuilder(types); }
protected void generateInputPage(int... initialValues) { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(types); buildPagesBuilder.addSequencePage(PAGE_SIZE, initialValues); inputPage = getOnlyElement(buildPagesBuilder.build()); }
private void testSimpleSpiller(Spiller spiller) throws ExecutionException, InterruptedException { RowPagesBuilder builder = RowPagesBuilder.rowPagesBuilder(TYPES); builder.addSequencePage(10, 0, 5, 10, 15); builder.pageBreak(); builder.addSequencePage(10, 0, -5, -10, -15); List<Page> firstSpill = builder.build(); builder = RowPagesBuilder.rowPagesBuilder(TYPES); builder.addSequencePage(10, 10, 15, 20, 25); builder.pageBreak(); builder.addSequencePage(10, -10, -15, -20, -25); List<Page> secondSpill = builder.build(); testSpiller(TYPES, spiller, firstSpill, secondSpill); }
public List<Page> build() { pageBreak(); List<Page> resultPages = pages.build(); if (hashEnabled) { return pagesWithHash(resultPages); } return resultPages; }
@Test public void testEmptyProbe() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .row(null, "null") .pageBreak() .row(POLYGON_B, "B"); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test public void testMergeOutput() List<Page> input = rowPagesBuilder(VARCHAR, BIGINT) .addSequencePage(100, 0, 0) .addSequencePage(100, 0, 0) .addSequencePage(100, 0, 0) .addSequencePage(100, 0, 0) .build(); 2); List<Page> expected = rowPagesBuilder(BIGINT) .row(10L) .row(10L) .row(10L) .row(10L) .build();
@Test public void testPartiallyPreGroupedPartitionWithEmptyInput() { List<Page> input = rowPagesBuilder(BIGINT, VARCHAR, BIGINT, VARCHAR) .pageBreak() .pageBreak() .build(); WindowOperatorFactory operatorFactory = createFactoryUnbounded( ImmutableList.of(BIGINT, VARCHAR, BIGINT, VARCHAR), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0, 1), Ints.asList(1), Ints.asList(3), ImmutableList.of(SortOrder.ASC_NULLS_LAST), 0); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, VARCHAR, BIGINT, VARCHAR, BIGINT) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Setup public void setup() { executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); int groupsPerPage = ROWS_PER_PAGE / rowsPerGroup; boolean hashAggregation = operatorType.equalsIgnoreCase("hash"); RowPagesBuilder pagesBuilder = RowPagesBuilder.rowPagesBuilder(hashAggregation, ImmutableList.of(0), VARCHAR, BIGINT); for (int i = 0; i < TOTAL_PAGES; i++) { BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, ROWS_PER_PAGE); for (int j = 0; j < groupsPerPage; j++) { String groupKey = format("%s", i * groupsPerPage + j); repeatToStringBlock(groupKey, rowsPerGroup, blockBuilder); } pagesBuilder.addBlocksPage(blockBuilder.build(), createLongSequenceBlock(0, ROWS_PER_PAGE)); } pages = pagesBuilder.build(); if (hashAggregation) { operatorFactory = createHashAggregationOperatorFactory(pagesBuilder.getHashChannel()); } else { operatorFactory = createStreamingAggregationOperatorFactory(); } }
RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)) .addSequencePage(10, 20, 30, 40); BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY); JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager(); RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)); List<Page> probeInput = probePages .addSequencePage(1000, 0, 1000, 2000) .build(); OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactory, probePages, PARTITIONING_SPILLER_FACTORY); MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probePages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())) .row("20", 1020L, 2020L, "20", 30L, 40L) .row("21", 1021L, 2021L, "21", 31L, 41L)
FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), UNGROUPED_EXECUTION, new DataSize(32, DataSize.Unit.MEGABYTE)); ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build()); LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity()); Driver sourceDriver = Driver.createDriver(collectDriverContext, buildPages.getTypes(), rangeList(buildPages.getTypes().size()).stream() .map(buildPages.getTypes()::get) .collect(toImmutableList()), hashChannels.stream() .map(buildPages.getTypes()::get) .collect(toImmutableList()), partitionCount, new PlanNodeId("build"), lookupSourceFactoryManager, rangeList(buildPages.getTypes().size()), hashChannels, buildPages.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()), filterFunctionFactory,
private void assertSpatialJoin(TaskContext taskContext, Type joinType, RowPagesBuilder buildPages, RowPagesBuilder probePages, MaterializedResult expected) { DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.empty(), buildPages); OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), joinType, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory); assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected); }
private static List<Integer> getHashChannels(RowPagesBuilder probe, RowPagesBuilder build) { ImmutableList.Builder<Integer> hashChannels = ImmutableList.builder(); if (probe.getHashChannel().isPresent()) { hashChannels.add(probe.getHashChannel().get()); } if (build.getHashChannel().isPresent()) { hashChannels.add(probe.getTypes().size() + build.getHashChannel().get()); } return hashChannels.build(); }
@Test public void testEmptyInput() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder.build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT).build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*", dataProvider = "testMemoryLimitProvider") public void testMemoryLimit(boolean parallelBuild, boolean buildHashEnabled) { TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)); RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)) .addSequencePage(10, 20, 30, 40); BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY); instantiateBuildDrivers(buildSideSetup, taskContext); buildLookupSource(buildSideSetup); }
private OperatorFactory innerJoinOperatorFactory(JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager, RowPagesBuilder probePages, PartitioningSpillerFactory partitioningSpillerFactory) { return LOOKUP_JOIN_OPERATORS.innerJoin( 0, new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), partitioningSpillerFactory); }
public static RowPagesBuilder rowPagesBuilder(Type... types) { return rowPagesBuilder(ImmutableList.copyOf(types)); }
public RowPagesBuilder rows(Object[]... rows) { for (Object[] row : rows) { row(row); } return this; }
public RowPagesBuilder addSequencePage(int length, int... initialValues) { checkArgument(length > 0, "length must be at least 1"); requireNonNull(initialValues, "initialValues is null"); checkArgument(initialValues.length == types.size(), "Expected %s initialValues, but got %s", types.size(), initialValues.length); pageBreak(); Page page = SequencePageBuilder.createSequencePage(types, length, initialValues); pages.add(page); return this; }