private static Page createPage(List<? extends Type> types, boolean dictionary) { if (dictionary) { return SequencePageBuilder.createSequencePageWithDictionaryBlocks(types, POSITIONS); } else { return SequencePageBuilder.createSequencePage(types, POSITIONS); } }
private static Page createPage(int i) { return SequencePageBuilder.createSequencePage(TYPES, 100, i); }
public static Page createSequencePage(List<? extends Type> types, int length) { return createSequencePage(types, length, new int[types.size()]); }
private static Page createPage(List<? extends Type> types, boolean dictionary) { return dictionary ? createSequencePageWithDictionaryBlocks(types, POSITIONS) : createSequencePage(types, POSITIONS); }
private void createPages(int totalChannels, int pagesCount) { int positionCount = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES / (totalChannels * 8); pages = new ArrayList<>(pagesCount); for (int numPage = 0; numPage < pagesCount; numPage++) { pages.add(createSequencePage(types, positionCount)); } totalPositions = positionCount * pagesCount; }
private static Page createPage(List<? extends Type> types, boolean dictionary) { if (dictionary) { return SequencePageBuilder.createSequencePageWithDictionaryBlocks(types, POSITIONS); } else { return SequencePageBuilder.createSequencePage(types, POSITIONS); } }
private static Page somePage(List<Type> types) { int[] initialValues = new int[types.size()]; Arrays.setAll(initialValues, i -> 100 * i); return createSequencePage(types, 7, initialValues); } }
private static Page createPage(List<? extends Type> types, boolean dictionary) { return dictionary ? createSequencePageWithDictionaryBlocks(types, POSITIONS) : createSequencePage(types, POSITIONS); } }
public RowPagesBuilder addSequencePage(int length, int... initialValues) { checkArgument(length > 0, "length must be at least 1"); requireNonNull(initialValues, "initialValues is null"); checkArgument(initialValues.length == types.size(), "Expected %s initialValues, but got %s", types.size(), initialValues.length); pageBreak(); Page page = SequencePageBuilder.createSequencePage(types, length, initialValues); pages.add(page); return this; }
@Test public void testSplitPage() { int positionCount = 10; int maxPageSizeInBytes = 100; List<Type> types = ImmutableList.of(BIGINT, BIGINT, BIGINT); Page largePage = createSequencePage(types, positionCount, 0, 1, 1); List<Page> pages = splitPage(largePage, maxPageSizeInBytes); assertGreaterThan(pages.size(), 1); assertPageSize(pages, maxPageSizeInBytes); assertPositionCount(pages, positionCount); MaterializedResult actual = toMaterializedResult(TEST_SESSION, types, pages); MaterializedResult expected = toMaterializedResult(TEST_SESSION, types, ImmutableList.of(largePage)); assertEquals(actual, expected); }
@Test public void testMinRowCountThreshold() { Page page = createSequencePage(TYPES, 10); MergingPageOutput output = new MergingPageOutput(TYPES, 1024 * 1024, page.getPositionCount(), Integer.MAX_VALUE); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(page)); assertFalse(output.needsInput()); assertSame(output.getOutput(), page); }
@Test public void testMinPageSizeThreshold() { Page page = createSequencePage(TYPES, 10); MergingPageOutput output = new MergingPageOutput(TYPES, page.getSizeInBytes(), Integer.MAX_VALUE, Integer.MAX_VALUE); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(page)); assertFalse(output.needsInput()); assertSame(output.getOutput(), page); }
@Test public void testFlushOnBigPage() { Page smallPage = createSequencePage(TYPES, 10); Page bigPage = createSequencePage(TYPES, 100); MergingPageOutput output = new MergingPageOutput(TYPES, bigPage.getSizeInBytes(), bigPage.getPositionCount(), Integer.MAX_VALUE); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(smallPage)); assertFalse(output.needsInput()); assertNull(output.getOutput()); assertTrue(output.needsInput()); output.addInput(createPagesIterator(bigPage)); assertFalse(output.needsInput()); assertPageEquals(TYPES, output.getOutput(), smallPage); assertFalse(output.needsInput()); assertSame(output.getOutput(), bigPage); }
@Test public void testLimitWithPageAlignment() { List<Page> input = rowPagesBuilder(BIGINT) .addSequencePage(3, 1) .addSequencePage(2, 4) .addSequencePage(2, 6) .build(); OperatorFactory operatorFactory = new LimitOperatorFactory(0, new PlanNodeId("test"), 5); MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT) .page(createSequencePage(ImmutableList.of(BIGINT), 3, 1)) .page(createSequencePage(ImmutableList.of(BIGINT), 2, 4)) .build(); OperatorAssertion.assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Test public void testFlushOnFullPage() { int singlePageRowCount = 10; List<Type> types = ImmutableList.of(BIGINT); Page page = createSequencePage(types, singlePageRowCount * 2); List<Page> splits = splitPage(page, page.getSizeInBytes() / 2); MergingPageOutput output = new MergingPageOutput(types, page.getSizeInBytes() / 2 + 1, page.getPositionCount() / 2 + 1, toIntExact(page.getSizeInBytes())); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(splits.get(0))); assertFalse(output.needsInput()); assertNull(output.getOutput()); assertTrue(output.needsInput()); output.addInput(createPagesIterator(splits.get(1))); assertFalse(output.needsInput()); assertPageEquals(types, output.getOutput(), page); output.addInput(createPagesIterator(splits.get(0), splits.get(1))); assertFalse(output.needsInput()); assertPageEquals(types, output.getOutput(), page); }
@Test public void testBufferSmallPages() { int singlePageRowCount = 10; Page page = createSequencePage(TYPES, singlePageRowCount * 2); List<Page> splits = splitPage(page, page.getSizeInBytes() / 2); MergingPageOutput output = new MergingPageOutput(TYPES, page.getSizeInBytes() + 1, page.getPositionCount() + 1, Integer.MAX_VALUE); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(splits.get(0))); assertFalse(output.needsInput()); assertNull(output.getOutput()); assertTrue(output.needsInput()); output.addInput(createPagesIterator(splits.get(1))); assertFalse(output.needsInput()); assertNull(output.getOutput()); output.finish(); assertFalse(output.needsInput()); assertPageEquals(TYPES, output.getOutput(), page); }
@Test public void testCloseDuringReading() throws Exception { Iterator<Page> readingInProgress; try (PartitioningSpiller spiller = factory.create( TYPES, new ModuloPartitionFunction(0, 4), mockSpillContext(), mockMemoryContext(scheduledExecutor))) { Page page = SequencePageBuilder.createSequencePage(TYPES, 10, FIRST_PARTITION_START, 5, 10, 15); PartitioningSpillResult spillResult = spiller.partitionAndSpill(page, partition -> true); assertEquals(spillResult.getRetained().getPositionCount(), 0); getFutureValue(spillResult.getSpillingFuture()); // We get the iterator but we do not exhaust it, so that close happens during reading readingInProgress = spiller.getSpilledPages(0); } try { readingInProgress.hasNext(); fail("Iterator.hasNext() should fail since underlying resources are closed"); } catch (UncheckedIOException ignored) { // expected } }
Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1); DriverContext driverContext = newDriverContext();
Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0); DriverContext driverContext = newDriverContext();
public RowPagesBuilder addSequencePage(int length, int... initialValues) { checkArgument(length > 0, "length must be at least 1"); requireNonNull(initialValues, "initialValues is null"); checkArgument(initialValues.length == types.size(), "Expected %s initialValues, but got %s", types.size(), initialValues.length); pageBreak(); Page page = SequencePageBuilder.createSequencePage(types, length, initialValues); pages.add(page); return this; }