public void processPage(Page page) { if (step.isInputRaw()) { aggregation.addInput(page); } else { aggregation.addIntermediate(page.getBlock(intermediateChannel)); } }
public static Block getFinalBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getFinalType().createBlockBuilder(null, 1000); accumulator.evaluateFinal(blockBuilder); return blockBuilder.build(); }
public void evaluate(BlockBuilder blockBuilder) { if (step.isOutputPartial()) { aggregation.evaluateIntermediate(blockBuilder); } else { aggregation.evaluateFinal(blockBuilder); } }
public Type getType() { if (step.isOutputPartial()) { return aggregation.getIntermediateType(); } else { return aggregation.getFinalType(); } }
private static Object aggregation(InternalAggregationFunction function, int[] args, Optional<Integer> maskChannel, Page... pages) { Accumulator aggregation = function.bind(Ints.asList(args), maskChannel).createAccumulator(); for (Page page : pages) { if (page.getPositionCount() > 0) { aggregation.addInput(page); } } Block block = getFinalBlock(aggregation); return BlockAssertions.getOnlyValue(aggregation.getFinalType(), block); }
private static void assertLearnClassifer(Accumulator accumulator) { accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); Slice slice = accumulator.getFinalType().getSlice(block, 0); Model deserialized = ModelUtils.deserialize(slice); assertNotNull(deserialized, "deserialization failed"); assertTrue(deserialized instanceof Classifier, "deserialized model is not a classifier"); }
public static Object partialAggregation(InternalAggregationFunction function, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty()); Accumulator finalAggregation = factory.createIntermediateAccumulator(); // Test handling of empty intermediate blocks Accumulator emptyAggregation = factory.createAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(emptyBlock); for (Page page : pages) { Accumulator partialAggregation = factory.createAccumulator(); if (page.getPositionCount() > 0) { partialAggregation.addInput(page); } Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(partialBlock); } finalAggregation.addIntermediate(emptyBlock); Block finalBlock = getFinalBlock(finalAggregation); return BlockAssertions.getOnlyValue(finalAggregation.getFinalType(), finalBlock); }
private void accumulate(int start, int end) { accumulator.addInput(windowIndex, argumentChannels, start, end); }
public static Block getIntermediateBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getIntermediateType().createBlockBuilder(null, 1000); accumulator.evaluateIntermediate(blockBuilder); return blockBuilder.build(); }
@Override public void evaluateFinal(BlockBuilder blockBuilder) { accumulator.evaluateFinal(blockBuilder); } }
@Override public Type getFinalType() { return accumulator.getFinalType(); }
public long getEstimatedSize() { return aggregation.getEstimatedSize(); } }
@Test(dataProvider = "partitionCount") public void test(int partitionCount) { InternalAggregationFunction function = getFunction(); List<OGCGeometry> geometries = makeGeometries(); Block geometryBlock = makeGeometryBlock(geometries); Block partitionCountBlock = BlockAssertions.createRLEBlock(partitionCount, geometries.size()); Rectangle expectedExtent = new Rectangle(-10, -10, Math.nextUp(10.0), Math.nextUp(10.0)); String expectedValue = getSpatialPartitioning(expectedExtent, geometries, partitionCount); AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); Page page = new Page(geometryBlock, partitionCountBlock); Accumulator accumulator = accumulatorFactory.createAccumulator(); accumulator.addInput(page); String aggregation = (String) BlockAssertions.getOnlyValue(accumulator.getFinalType(), getFinalBlock(accumulator)); assertEquals(aggregation, expectedValue); GroupedAccumulator groupedAggregation = accumulatorFactory.createGroupedAccumulator(); groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); String groupValue = (String) getGroupValue(groupedAggregation, 0); assertEquals(groupValue, expectedValue); }
@Test public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
public static Object partialAggregation(InternalAggregationFunction function, double confidence, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty(), Optional.empty(), confidence); Accumulator finalAggregation = factory.createIntermediateAccumulator(); // Test handling of empty intermediate blocks Accumulator emptyAggregation = factory.createAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(emptyBlock); for (Page page : pages) { Accumulator partialAggregation = factory.createAccumulator(); if (page.getPositionCount() > 0) { partialAggregation.addInput(page); } Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(partialBlock); } finalAggregation.addIntermediate(emptyBlock); Block finalBlock = getFinalBlock(finalAggregation); return BlockAssertions.getOnlyValue(finalAggregation.getFinalType(), finalBlock); }
@Override public void addInput(Page page) { // 1. filter out positions based on mask, if present Page filtered = maskChannel .map(channel -> filter(page, page.getBlock(channel))) .orElse(page); if (filtered.getPositionCount() == 0) { return; } // 2. compute a distinct mask Work<Block> work = hash.markDistinctRows(filtered); checkState(work.process()); Block distinctMask = work.getResult(); // 3. feed a Page with a new mask to the underlying aggregation accumulator.addInput(filtered.prependColumn(distinctMask)); }
public Type getType() { if (step == Step.PARTIAL) { return aggregation.getIntermediateType(); } else { return aggregation.getFinalType(); } }
public static Block getIntermediateBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getIntermediateType().createBlockBuilder(new BlockBuilderStatus(), 1000); accumulator.evaluateIntermediate(blockBuilder); return blockBuilder.build(); }
@Override public void evaluateFinal(BlockBuilder blockBuilder) { pagesIndex.sort(orderByChannels, orderings); Iterator<Page> pagesIterator = pagesIndex.getSortedPages(); pagesIterator.forEachRemaining(accumulator::addInput); accumulator.evaluateFinal(blockBuilder); } }
@Override public Type getFinalType() { return accumulator.getFinalType(); }