@Test public void testMinVarcharArray() { InternalAggregationFunction function = METADATA.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("min_by", AGGREGATE, parseTypeSignature("array(array(bigint))"), parseTypeSignature("array(bigint)"), parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT))); assertAggregation( function, ImmutableList.of(ImmutableList.of(2L, 3L), ImmutableList.of(4L, 5L)), createArrayBigintBlock(ImmutableList.of(ImmutableList.of(1L, 2L), ImmutableList.of(2L, 3L), ImmutableList.of(3L, 4L), ImmutableList.of(4L, 5L))), createStringsBlock("z", "a", "x", "b"), createRLEBlock(2L, 4)); }
public static Block createSlicesBlock(Slice... values) { requireNonNull(values, "varargs 'values' is null"); return createSlicesBlock(Arrays.asList(values)); }
blocks[i] = BlockAssertions.createLongSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createSequenceBlockOfReal(initialValue, initialValue + length); blocks[i] = BlockAssertions.createDoubleSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createStringSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createBooleanSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createDateSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createTimestampSequenceBlock(initialValue, initialValue + length); blocks[i] = BlockAssertions.createShortDecimalSequenceBlock(initialValue, initialValue + length, (DecimalType) type); blocks[i] = BlockAssertions.createLongDecimalSequenceBlock(initialValue, initialValue + length, (DecimalType) type);
@Override public Block[] getSequenceBlocks(int start, int length) { return new Block[] {createDoubleSequenceBlock(start, start + length), createLongRepeatBlock(2, length)}; }
private void testCustomAggregation(Double[] values, int n) { PriorityQueue<Double> heap = new PriorityQueue<>(n, (x, y) -> -Double.compare(x, y)); Arrays.stream(values).filter(x -> x != null).forEach(heap::add); Double[] expected = new Double[heap.size()]; for (int i = heap.size() - 1; i >= 0; i--) { expected[i] = heap.remove(); } testAggregation(Arrays.asList(expected), createDoublesBlock(values), createLongRepeatBlock(n, values.length)); } }
private static List<Page> createPages(int pageCount, int channelCount, Type type) { int positionCount = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES / (channelCount * 8); List<Page> pages = new ArrayList<>(pageCount); for (int numPage = 0; numPage < pageCount; numPage++) { Block[] blocks = new Block[channelCount]; for (int numChannel = 0; numChannel < channelCount; numChannel++) { if (type.equals(BIGINT)) { blocks[numChannel] = BlockAssertions.createLongSequenceBlock(0, positionCount); } else if (type.equals(VARCHAR)) { blocks[numChannel] = BlockAssertions.createStringSequenceBlock(0, positionCount); } else if (type.equals(DOUBLE)) { blocks[numChannel] = BlockAssertions.createDoubleSequenceBlock(0, positionCount); } else if (type.equals(BOOLEAN)) { blocks[numChannel] = BlockAssertions.createBooleanSequenceBlock(0, positionCount); } else { throw new IllegalArgumentException("Unsupported type: " + type); } } pages.add(new Page(blocks)); } return pages; }
@Test public void testMaxDoubleVarchar() { InternalAggregationFunction function = METADATA.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("max_by", AGGREGATE, parseTypeSignature("array(varchar)"), parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.DOUBLE), parseTypeSignature(StandardTypes.BIGINT))); assertAggregation( function, ImmutableList.of("a", "z"), createStringsBlock("z", "a", null), createDoublesBlock(1.0, 2.0, null), createRLEBlock(2L, 3)); assertAggregation( function, ImmutableList.of("bb", "hi"), createStringsBlock("zz", "hi", "bb", "a"), createDoublesBlock(0.0, 1.0, 2.0, -1.0), createRLEBlock(2L, 4)); assertAggregation( function, ImmutableList.of("hi", "zz"), createStringsBlock("zz", "hi", null, "a"), createDoublesBlock(0.0, 1.0, null, -1.0), createRLEBlock(2L, 4)); }
public static Page createSequencePage(List<? extends Type> types, int length, int... initialValues) { Block[] blocks = new Block[initialValues.length]; for (int i = 0; i < blocks.length; i++) { Type type = types.get(i); int initialValue = initialValues[i]; if (type.equals(BIGINT)) { blocks[i] = BlockAssertions.createLongSequenceBlock(initialValue, initialValue + length); } else if (type.equals(DOUBLE)) { blocks[i] = BlockAssertions.createDoubleSequenceBlock(initialValue, initialValue + length); } else if (type.equals(VARCHAR)) { blocks[i] = BlockAssertions.createStringSequenceBlock(initialValue, initialValue + length); } else if (type.equals(BOOLEAN)) { blocks[i] = BlockAssertions.createBooleanSequenceBlock(initialValue, initialValue + length); } else if (type.equals(DATE)) { blocks[i] = BlockAssertions.createDateSequenceBlock(initialValue, initialValue + length); } else if (type.equals(TIMESTAMP)) { blocks[i] = BlockAssertions.createTimestampSequenceBlock(initialValue, initialValue + length); } else { throw new IllegalStateException("Unsupported type " + type); } } return new Page(blocks); }
@Test public void testContainsMultipleColumns() { Block valuesBlock = BlockAssertions.createDoubleSequenceBlock(0, 10); Block stringValuesBlock = BlockAssertions.createStringSequenceBlock(0, 10); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(DOUBLE, VARCHAR), valuesBlock, stringValuesBlock); int[] hashChannels = {0, 1}; GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(DOUBLE, VARCHAR), hashChannels, Optional.of(2), 100, JOIN_COMPILER); groupByHash.getGroupIds(new Page(valuesBlock, stringValuesBlock, hashBlock)).process(); Block testValuesBlock = BlockAssertions.createDoublesBlock((double) 3); Block testStringValuesBlock = BlockAssertions.createStringsBlock("3"); Block testHashBlock = TypeUtils.getHashBlock(ImmutableList.of(DOUBLE, VARCHAR), testValuesBlock, testStringValuesBlock); assertTrue(groupByHash.contains(0, new Page(testValuesBlock, testStringValuesBlock, testHashBlock), hashChannels)); }
DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, null, createDoublesBlock(null, null), createRLEBlock(0.5, 2)); DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, createDoublesBlock(null, 1.0), createRLEBlock(0.5, 2)); DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 2.0, createDoublesBlock(null, 1.0, 2.0, 3.0), createRLEBlock(0.5, 4)); DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 2.0, createDoublesBlock(1.0, 2.0, 3.0), createRLEBlock(0.5, 3)); DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 3.0, createDoublesBlock(1.0, null, 2.0, 2.0, null, 2.0, 2.0, null, 2.0, 2.0, null, 3.0, 3.0, null, 3.0, null, 3.0, 4.0, 5.0, 6.0, 7.0), createRLEBlock(0.5, 21)); DOUBLE_APPROXIMATE_PERCENTILE_ARRAY_AGGREGATION, null, createDoublesBlock(null, null), createRLEBlock(ImmutableList.of(0.5), 2));
public static Object getGroupValue(GroupedAccumulator groupedAggregation, int groupId) { BlockBuilder out = groupedAggregation.getFinalType().createBlockBuilder(null, 1); groupedAggregation.evaluateFinal(groupId, out); return BlockAssertions.getOnlyValue(groupedAggregation.getFinalType(), out.build()); }
@Test(dataProvider = "partitionCount") public void test(int partitionCount) { InternalAggregationFunction function = getFunction(); List<OGCGeometry> geometries = makeGeometries(); Block geometryBlock = makeGeometryBlock(geometries); Block partitionCountBlock = BlockAssertions.createRLEBlock(partitionCount, geometries.size()); Rectangle expectedExtent = new Rectangle(-10, -10, Math.nextUp(10.0), Math.nextUp(10.0)); String expectedValue = getSpatialPartitioning(expectedExtent, geometries, partitionCount); AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); Page page = new Page(geometryBlock, partitionCountBlock); Accumulator accumulator = accumulatorFactory.createAccumulator(); accumulator.addInput(page); String aggregation = (String) BlockAssertions.getOnlyValue(accumulator.getFinalType(), getFinalBlock(accumulator)); assertEquals(aggregation, expectedValue); GroupedAccumulator groupedAggregation = accumulatorFactory.createGroupedAccumulator(); groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); String groupValue = (String) getGroupValue(groupedAggregation, 0); assertEquals(groupValue, expectedValue); }
@Test public void testAppendToMultipleTuplesPerGroup() { List<Long> values = new ArrayList<>(); for (long i = 0; i < 100; i++) { values.add(i % 50); } Block valuesBlock = BlockAssertions.createLongsBlock(values); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)).process(); assertEquals(groupByHash.getGroupCount(), 50); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page outputPage = pageBuilder.build(); assertEquals(outputPage.getPositionCount(), 50); BlockAssertions.assertBlockEquals(BIGINT, outputPage.getBlock(0), BlockAssertions.createLongSequenceBlock(0, 50)); }
createDoublesBlock(1.0, null, 2.0, null, 3.0, null, 4.0, null, 5.0, null), createRLEBlock(1, 10), 0.01, 1.0, 2.0, 3.0, 4.0, 5.0); testAggregationDouble( createDoublesBlock(null, null, null, null, null), createRLEBlock(1, 5), NaN); testAggregationDouble( createDoublesBlock(-1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0), createRLEBlock(1, 10), 0.01, -1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0); testAggregationDouble( createDoublesBlock(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0), createRLEBlock(1, 10), 0.01, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0); testAggregationDouble( createDoublesBlock(), createRLEBlock(1, 0), NaN); testAggregationDouble( createDoublesBlock(1.0), createRLEBlock(1, 1), 0.01, 1.0); testAggregationDouble( createDoubleSequenceBlock(-1000, 1000), createRLEBlock(1, 2000), 0.01, LongStream.range(-1000, 1000).asDoubleStream().toArray());
@Test public void testAppendTo() { Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock); GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); Work<GroupByIdBlock> work = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)); work.process(); GroupByIdBlock groupIds = work.getResult(); for (int i = 0; i < groupIds.getPositionCount(); i++) { assertEquals(groupIds.getGroupId(i), i); } assertEquals(groupByHash.getGroupCount(), 100); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(i, pageBuilder, 0); } Page page = pageBuilder.build(); // Ensure that all blocks have the same positionCount for (int i = 0; i < groupByHash.getTypes().size(); i++) { assertEquals(page.getBlock(i).getPositionCount(), 100); } assertEquals(page.getPositionCount(), 100); BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock); BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock); }
@Test public void testMaxLongArraySlice() { InternalAggregationFunction function = METADATA.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("max_by", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature("array(bigint)"))); assertAggregation( function, "a", createStringsBlock("a", "b", "c"), createArrayBigintBlock(asList(asList(3L, 4L), null, asList(2L, 2L)))); }
@Test public void testSanityFilterOnRLE() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression filter = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L); }
public static Block createDoublesBlock(Double... values) { requireNonNull(values, "varargs 'values' is null"); return createDoublesBlock(Arrays.asList(values)); }
@Test public void testMaxBooleanLongArray() { InternalAggregationFunction function = METADATA.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("max_by", AGGREGATE, parseTypeSignature("array(bigint)"), parseTypeSignature("array(bigint)"), parseTypeSignature(StandardTypes.BOOLEAN))); assertAggregation( function, asList(2L, 2L), createArrayBigintBlock(asList(asList(3L, 4L), null, asList(2L, 2L))), createBooleansBlock(false, false, true)); }
@Test public void testMinLongArrayLongArray() { InternalAggregationFunction function = METADATA.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("min_by", AGGREGATE, parseTypeSignature("array(bigint)"), parseTypeSignature("array(bigint)"), parseTypeSignature("array(bigint)"))); assertAggregation( function, asList(1L, 2L), createArrayBigintBlock(asList(asList(3L, 3L), null, asList(1L, 2L))), createArrayBigintBlock(asList(asList(3L, 4L), null, asList(2L, 2L)))); }