protected void generateRandomInputPage() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(types); for (int i = 0; i < PAGE_SIZE; i++) { Object[] values = types.stream() .map(this::generateRandomValue) .collect(toList()).toArray(); buildPagesBuilder.row(values); } inputPage = getOnlyElement(buildPagesBuilder.build()); }
@Test public void testPageSorterSorted() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
@Test public void testPageSorterMultipleChannels() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0, 1, 2); List<SortOrder> sortOrders = Collections.nCopies(sortChannels.size(), ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "d") .row(1L, 2.2, "c") .pageBreak() .row(1L, 2.2, "b") .row(1L, 2.2, "a") .pageBreak() .row(1L, 2.2, null) .row(1L, null, "z") .row(1L, null, null) .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(1L, null, null) .row(1L, null, "z") .row(1L, 2.2, null) .row(1L, 2.2, "a") .row(1L, 2.2, "b") .row(1L, 2.2, "c") .row(2L, 1.1, "d") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
@Test public void testEmptyProbe() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .row(null, "null") .pageBreak() .row(POLYGON_B, "B"); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test public void testPageSorter() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "d") .row(1L, 2.2, "c") .pageBreak() .row(-2L, 2.2, "b") .row(-12L, 2.2, "a") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "c") .row(2L, 1.1, "d") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
@Test public void testPageSorterForceExpansion() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "c") .row(1L, 2.2, "d") .pageBreak() .row(-2L, 2.2, "b") .row(-12L, 2.2, "a") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 2); }
@Test public void testSingleGroupingValue() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder .row(true, "a", 1) .row(false, "a", 2) .row(true, "a", 3) .row(false, "a", 4) .row(true, "a", 5) .pageBreak() .row(false, "a", 6) .row(true, "a", 7) .row(false, "a", 8) .pageBreak() .pageBreak() .row(true, "a", 9) .row(false, "a", 10) .build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT) .row("a", 10L, 55L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); } }
@Test public void testEmptyBuild() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POINT_X, "x") .row(null, "null") .row(POINT_Y, "y") .pageBreak() .row(POINT_Z, "z") .pageBreak() .row(POINT_W, "w"); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test public void testSinglePage() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder .row(false, "a", 5) .build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT) .row("a", 1L, 5L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test(expectedExceptions = IllegalStateException.class) public void addInputFailsOnBlockedOperator() { Operator operator = createTableWriterOperator(new BlockingPageSink()); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); assertFalse(operator.isBlocked().isDone()); assertFalse(operator.needsInput()); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); }
@Test public void testSpatialJoin() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .row(null, "null") .pageBreak() .row(POLYGON_B, "B"); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POINT_X, "x") .row(null, "null") .row(POINT_Y, "y") .pageBreak() .row(POINT_Z, "z") .pageBreak() .row(POINT_W, "w"); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)) .row("x", "A") .row("y", "A") .row("y", "B") .row("z", "B") .build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test public void testSingleFieldKey() { List<Page> input = rowPagesBuilder(BIGINT, DOUBLE) .row(1L, 0.1) .row(2L, 0.2) .pageBreak() .row(-1L, -0.1) .row(4L, 0.4) .pageBreak() .row(5L, 0.5) .row(4L, 0.41) .row(6L, 0.6) .pageBreak() .build(); TopNOperatorFactory operatorFactory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), 2, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST)); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, DOUBLE) .row(6L, 0.6) .row(5L, 0.5) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test public void testSingleStream() throws Exception { List<Type> types = ImmutableList.of(INTEGER, INTEGER); MaterializedResult actual = mergeSortedPages( types, ImmutableList.of(0, 1), ImmutableList.of(ASC_NULLS_FIRST, DESC_NULLS_FIRST), ImmutableList.of( rowPagesBuilder(types) .row(1, 4) .row(2, 3) .pageBreak() .row(3, 2) .row(4, 1) .build())); MaterializedResult expected = resultBuilder(TEST_SESSION, types) .row(1, 4) .row(2, 3) .row(3, 2) .row(4, 1) .build(); assertEquals(actual, expected); }
@Test public void testReverseOrder() { List<Page> input = rowPagesBuilder(BIGINT, DOUBLE) .row(1L, 0.1) .row(2L, 0.2) .pageBreak() .row(-1L, -0.1) .row(4L, 0.4) .pageBreak() .row(5L, 0.5) .row(4L, 0.41) .row(6L, 0.6) .pageBreak() .build(); TopNOperatorFactory operatorFactory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), 2, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_LAST)); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, DOUBLE) .row(-1L, -0.1) .row(1L, 0.1) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test public void testMultiFieldKey() { List<Page> input = rowPagesBuilder(VARCHAR, BIGINT) .row("a", 1L) .row("b", 2L) .pageBreak() .row("f", 3L) .row("a", 4L) .pageBreak() .row("d", 5L) .row("d", 7L) .row("e", 6L) .build(); TopNOperatorFactory operatorFactory = new TopNOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), 3, ImmutableList.of(0, 1), ImmutableList.of(DESC_NULLS_LAST, DESC_NULLS_LAST)); MaterializedResult expected = MaterializedResult.resultBuilder(driverContext.getSession(), VARCHAR, BIGINT) .row("f", 3L) .row("e", 6L) .row("d", 7L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test public void testSimpleTwoStreams() throws Exception List<Type> types = ImmutableList.of(INTEGER); MaterializedResult actual = mergeSortedPages( types, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_FIRST), ImmutableList.of( rowPagesBuilder(types) .row(1) .row(3) .pageBreak() .row(5) .row(7) .build(), rowPagesBuilder(types) .row(2) .row(4) .pageBreak() .row(6) .row(8) .build())); MaterializedResult expected = resultBuilder(TEST_SESSION, types) .row(1) .row(2)
@Test public void testSpatialLeftJoin() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .row(null, "null") .pageBreak() .row(POLYGON_B, "B"); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POINT_X, "x") .row(null, "null") .row(POINT_Y, "y") .pageBreak() .row(POINT_Z, "z") .pageBreak() .row(POINT_W, "w"); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)) .row("x", "A") .row("null", null) .row("y", "A") .row("y", "B") .row("z", "B") .row("w", null) .build(); assertSpatialJoin(taskContext, LEFT, buildPages, probePages, expected); }
@Test public void testEmptyProbePage() { TaskContext taskContext = createTaskContext(); // build List<Type> buildTypes = ImmutableList.of(VARCHAR); RowPagesBuilder buildPages = rowPagesBuilder(buildTypes) .row("A") .row("B") .pageBreak() .row("C"); // probe List<Type> probeTypes = ImmutableList.of(VARCHAR); RowPagesBuilder probePages = rowPagesBuilder(probeTypes); List<Page> probeInput = probePages .pageBreak() .build(); NestedLoopJoinOperatorFactory joinOperatorFactory = newJoinOperatorFactoryWithCompletedBuild(taskContext, buildPages); // expected MaterializedResult expected = resultBuilder(taskContext.getSession(), concat(probeTypes, buildPages.getTypes())) .build(); assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected); }
@Test public void testEmptyBuildLeftJoin() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POINT_X, "x") .row(null, "null") .row(POINT_Y, "y") .pageBreak() .row(POINT_Z, "z") .pageBreak() .row(POINT_W, "w"); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)) .row("x", null) .row("null", null) .row("y", null) .row("z", null) .row("w", null) .build(); assertSpatialJoin(taskContext, LEFT, buildPages, probePages, expected); }
@Test public void testDistributedSpatialJoin() { TaskContext taskContext = createTaskContext(); DriverContext driverContext = taskContext.addPipelineContext(0, true, true, true).addDriverContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, INTEGER)) .row(POLYGON_A, "A", 1) .row(POLYGON_A, "A", 2) .row(null, "null", null) .pageBreak() .row(POLYGON_B, "B", 0) .row(POLYGON_B, "B", 2); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, INTEGER)) .row(POINT_X, "x", 2) .row(null, "null", null) .row(POINT_Y, "y", 2) .pageBreak() .row(POINT_Z, "z", 0); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)) .row("x", "A") .row("y", "A") .row("y", "B") .row("z", "B") .build(); PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.of(2), Optional.of(KDB_TREE_JSON), Optional.empty(), buildPages); OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.of(2), pagesSpatialIndexFactory); assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected); }