public void processPage(GroupByIdBlock groupIds, Page page) { if (step.isInputRaw()) { aggregation.addInput(groupIds, page); } else { aggregation.addIntermediate(groupIds, page.getBlock(intermediateChannel)); } }
@Benchmark public GroupedAccumulator testSharedGroupWithLargeBlocksRunner(Data data) { GroupedAccumulator groupedAccumulator = data.groupedAccumulator; for (int i = 0; i < data.numGroups; i++) { GroupByIdBlock groupByIdBlock = data.groupByIdBlocks[i]; Page page = data.pages[i]; groupedAccumulator.addInput(groupByIdBlock, page); } return groupedAccumulator; }
@Override public void prepareFinal() { pagesIndex.sort(orderByChannels, orderings); Iterator<Page> pagesIterator = pagesIndex.getSortedPages(); pagesIterator.forEachRemaining(page -> { // The last channel of the page is the group id GroupByIdBlock groupIds = new GroupByIdBlock(groupCount, page.getBlock(page.getChannelCount() - 1)); // We pass group id together with the other input channels to accumulator. Accumulator knows which input channels // to use. Since we did not change the order of original input channels, passing the group id is safe. accumulator.addInput(groupIds, page); }); } }
public GroupedAccumulator runPagesOnAccumulator(long groupId, Supplier<GroupedAccumulator> accumulatorSupplier) { GroupedAccumulator accumulator = accumulatorSupplier.get(); for (Page page : getPages()) { accumulator.addInput(getGroupIdBlock(groupId, page), page); } return accumulator; }
public void runPagesOnAccumulatorWithAssertion(long groupId, GroupedAccumulator groupedAccumulator, AggregationTestOutput expectedValue) { GroupedAccumulator accumulator = Suppliers.ofInstance(groupedAccumulator).get(); for (Page page : getPages()) { accumulator.addInput(getGroupIdBlock(groupId, page), page); } expectedValue.validateAccumulator(accumulator, groupId); }
@Override public void addInput(GroupByIdBlock groupIdsBlock, Page page) { Page withGroup = page.prependColumn(groupIdsBlock); // 1. filter out positions based on mask, if present Page filtered = maskChannel .map(channel -> filter(withGroup, withGroup.getBlock(channel + 1))) // offset by one because of group id in column 0 .orElse(withGroup); // 2. compute a mask for the distinct rows (including the group id) Work<Block> work = hash.markDistinctRows(filtered); checkState(work.process()); Block distinctMask = work.getResult(); // 3. feed a Page with a new mask to the underlying aggregation GroupByIdBlock groupIds = new GroupByIdBlock(groupIdsBlock.getGroupCount(), filtered.getBlock(0)); // drop the group id column and prepend the distinct mask column Block[] columns = new Block[filtered.getChannelCount()]; columns[0] = distinctMask; for (int i = 1; i < filtered.getChannelCount(); i++) { columns[i] = filtered.getBlock(i); } accumulator.addInput(groupIds, new Page(filtered.getPositionCount(), columns)); }
public static Object groupedAggregation(InternalAggregationFunction function, int[] args, Page... pages) { GroupedAccumulator groupedAggregation = function.bind(Ints.asList(args), Optional.empty()).createGroupedAccumulator(); for (Page page : pages) { groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); } Object groupValue = getGroupValue(groupedAggregation, 0); for (Page page : pages) { groupedAggregation.addInput(createGroupByIdBlock(4000, page.getPositionCount()), page); } Object largeGroupValue = getGroupValue(groupedAggregation, 4000); assertEquals(largeGroupValue, groupValue, "Inconsistent results with large group id"); return groupValue; }
@Test(dataProvider = "partitionCount") public void test(int partitionCount) { InternalAggregationFunction function = getFunction(); List<OGCGeometry> geometries = makeGeometries(); Block geometryBlock = makeGeometryBlock(geometries); Block partitionCountBlock = BlockAssertions.createRLEBlock(partitionCount, geometries.size()); Rectangle expectedExtent = new Rectangle(-10, -10, Math.nextUp(10.0), Math.nextUp(10.0)); String expectedValue = getSpatialPartitioning(expectedExtent, geometries, partitionCount); AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); Page page = new Page(geometryBlock, partitionCountBlock); Accumulator accumulator = accumulatorFactory.createAccumulator(); accumulator.addInput(page); String aggregation = (String) BlockAssertions.getOnlyValue(accumulator.getFinalType(), getFinalBlock(accumulator)); assertEquals(aggregation, expectedValue); GroupedAccumulator groupedAggregation = accumulatorFactory.createGroupedAccumulator(); groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); String groupValue = (String) getGroupValue(groupedAggregation, 0); assertEquals(groupValue, expectedValue); }
public static Object groupedPartialAggregation(InternalAggregationFunction function, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty()); GroupedAccumulator finalAggregation = factory.createGroupedIntermediateAccumulator(); // Add an empty block to test the handling of empty intermediates GroupedAccumulator emptyAggregation = factory.createGroupedAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(createGroupByIdBlock(0, emptyBlock.getPositionCount()), emptyBlock); for (Page page : pages) { GroupedAccumulator partialAggregation = factory.createGroupedAccumulator(); partialAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(createGroupByIdBlock(0, partialBlock.getPositionCount()), partialBlock); } finalAggregation.addIntermediate(createGroupByIdBlock(0, emptyBlock.getPositionCount()), emptyBlock); return getGroupValue(finalAggregation, 0); }
public void processPage(GroupByIdBlock groupIds, Page page) { if (step.isInputRaw()) { aggregation.addInput(groupIds, page); } else { aggregation.addIntermediate(groupIds, page.getBlock(intermediateChannel)); } }
public static boolean groupedApproximateAggregationWithinErrorBound(InternalAggregationFunction function, int sampleWeightChannel, double confidence, Double expectedValue, Page... pages) { GroupedAccumulator groupedAggregation = function.bind(ImmutableList.of(0), Optional.empty(), Optional.of(sampleWeightChannel), confidence).createGroupedAccumulator(); for (Page page : pages) { groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); } Object groupValue = getGroupValue(groupedAggregation, 0); if (expectedValue == null) { return groupValue == null; } return withinErrorBound(groupValue.toString(), expectedValue); }
public static Object groupedAggregation(InternalAggregationFunction function, double confidence, int[] args, Page... pages) { GroupedAccumulator groupedAggregation = function.bind(Ints.asList(args), Optional.empty(), Optional.empty(), confidence).createGroupedAccumulator(); for (Page page : pages) { groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); } Object groupValue = getGroupValue(groupedAggregation, 0); for (Page page : pages) { groupedAggregation.addInput(createGroupByIdBlock(4000, page.getPositionCount()), page); } Object largeGroupValue = getGroupValue(groupedAggregation, 4000); assertEquals(largeGroupValue, groupValue, "Inconsistent results with large group id"); return groupValue; }
@Test(dataProvider = "partitionCount") public void test(int partitionCount) { InternalAggregationFunction function = getFunction(); List<OGCGeometry> geometries = makeGeometries(); Block geometryBlock = makeGeometryBlock(geometries); Block partitionCountBlock = BlockAssertions.createRLEBlock(partitionCount, geometries.size()); Rectangle expectedExtent = new Rectangle(-10, -10, Math.nextUp(10.0), Math.nextUp(10.0)); String expectedValue = getSpatialPartitioning(expectedExtent, geometries, partitionCount); AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); Page page = new Page(geometryBlock, partitionCountBlock); Accumulator accumulator = accumulatorFactory.createAccumulator(); accumulator.addInput(page); String aggregation = (String) BlockAssertions.getOnlyValue(accumulator.getFinalType(), getFinalBlock(accumulator)); assertEquals(aggregation, expectedValue); GroupedAccumulator groupedAggregation = accumulatorFactory.createGroupedAccumulator(); groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); String groupValue = (String) getGroupValue(groupedAggregation, 0); assertEquals(groupValue, expectedValue); }
public static Object groupedPartialAggregation(InternalAggregationFunction function, double confidence, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty(), Optional.empty(), confidence); GroupedAccumulator finalAggregation = factory.createGroupedIntermediateAccumulator(); // Add an empty block to test the handling of empty intermediates GroupedAccumulator emptyAggregation = factory.createGroupedAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(createGroupByIdBlock(0, emptyBlock.getPositionCount()), emptyBlock); for (Page page : pages) { GroupedAccumulator partialAggregation = factory.createGroupedAccumulator(); partialAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(createGroupByIdBlock(0, partialBlock.getPositionCount()), partialBlock); } finalAggregation.addIntermediate(createGroupByIdBlock(0, emptyBlock.getPositionCount()), emptyBlock); return getGroupValue(finalAggregation, 0); }