private WorkProcessor<Page> buildResult(IntIterator groupIds) { final PageBuilder pageBuilder = new PageBuilder(buildTypes()); return WorkProcessor.create(() -> { if (!groupIds.hasNext()) { return ProcessState.finished(); } pageBuilder.reset(); List<Type> types = groupByHash.getTypes(); while (!pageBuilder.isFull() && groupIds.hasNext()) { int groupId = groupIds.nextInt(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); pageBuilder.declarePosition(); for (int i = 0; i < aggregators.size(); i++) { Aggregator aggregator = aggregators.get(i); BlockBuilder output = pageBuilder.getBlockBuilder(types.size() + i); aggregator.evaluate(groupId, output); } } return ProcessState.ofResult(pageBuilder.build()); }); }
@Test public void testAppendToMultipleTuplesPerGroup() { List<Long> values = new ArrayList<>(); for (long i = 0; i < 100; i++) { values.add(i % 50); } Block valuesBlock = BlockAssertions.createLongsBlock(values); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)).process(); assertEquals(groupByHash.getGroupCount(), 50); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page outputPage = pageBuilder.build(); assertEquals(outputPage.getPositionCount(), 50); BlockAssertions.assertBlockEquals(BIGINT, outputPage.getBlock(0), BlockAssertions.createLongSequenceBlock(0, 50)); }
@Test public void testAppendTo() { Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); Work<GroupByIdBlock> work = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)); work.process(); GroupByIdBlock groupIds = work.getResult(); for (int i = 0; i < groupIds.getPositionCount(); i++) { assertEquals(groupIds.getGroupId(i), i); } assertEquals(groupByHash.getGroupCount(), 100); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page page = pageBuilder.build(); // Ensure that all blocks have the same positionCount for (int i = 0; i < groupByHash.getTypes().size(); i++) { assertEquals(page.getBlock(i).getPositionCount(), 100); } assertEquals(page.getPositionCount(), 100); BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock); BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object bigintGroupByHash(SingleChannelBenchmarkData data) { GroupByHash groupByHash = new BigintGroupByHash(0, data.getHashEnabled(), EXPECTED_SIZE, NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object groupByHashPreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.getGroupIds(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object addPagePreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Override protected Page computeNext() { if (groupId >= groupCount) { return endOfData(); } pageBuilder.reset(); List<Type> types = groupByHash.getTypes(); while (!pageBuilder.isFull() && groupId < groupCount) { groupByHash.appendValuesTo(groupId, pageBuilder, 0); pageBuilder.declarePosition(); for (int i = 0; i < aggregators.size(); i++) { Aggregator aggregator = aggregators.get(i); BlockBuilder output = pageBuilder.getBlockBuilder(types.size() + i); aggregator.evaluate(groupId, output); } groupId++; } return pageBuilder.build(); } };
@Test public void testAppendToMultipleTuplesPerGroup() throws Exception { List<Long> values = new ArrayList<>(); for (long i = 0; i < 100; i++) { values.add(i % 50); } Block valuesBlock = BlockAssertions.createLongsBlock(values); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0 }, Optional.<Integer>empty(), Optional.of(1), 100); groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)); assertEquals(groupByHash.getGroupCount(), 50); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page outputPage = pageBuilder.build(); assertEquals(outputPage.getPositionCount(), 50); BlockAssertions.assertBlockEquals(BIGINT, outputPage.getBlock(1), BlockAssertions.createLongSequenceBlock(0, 50)); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object bigintGroupByHash(SingleChannelBenchmarkData data) { GroupByHash groupByHash = new BigintGroupByHash(0, Optional.empty(), data.getHashEnabled(), EXPECTED_SIZE); data.getPages().forEach(groupByHash::addPage); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Test public void testAppendTo() throws Exception { Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.<Integer>empty(), Optional.of(1), 100); GroupByIdBlock groupIds = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)); for (int i = 0; i < groupIds.getPositionCount(); i++) { assertEquals(groupIds.getGroupId(i), i); } assertEquals(groupByHash.getGroupCount(), 100); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page page = pageBuilder.build(); // Ensure that all blocks have the same positionCount for (int i = 0; i < groupByHash.getTypes().size(); i++) { assertEquals(page.getBlock(i).getPositionCount(), 100); } assertEquals(page.getPositionCount(), 100); BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock); BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object addPagePreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), Optional.empty(), data.getHashChannel(), EXPECTED_SIZE, false); data.getPages().forEach(groupByHash::addPage); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object groupByHashPreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), Optional.empty(), data.getHashChannel(), EXPECTED_SIZE, false); data.getPages().forEach(groupByHash::getGroupIds); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }