@Override public Block toBlock(Type desiredType) { checkArgument(desiredType.getTypeParameters().size() == 1 && BIGINT.equals(desiredType.getTypeParameters().get(0)), "type doesn't match: %s", desiredType); int numberOfRecords = numberOfRecords(); return ArrayBlock.fromElementBlock( numberOfRecords, Optional.of(nulls == null ? new boolean[numberOfRecords] : nulls), calculateOffsets(sizes, nulls, numberOfRecords), values != null ? values.toBlock(BIGINT) : new LongArrayBlock(0, Optional.empty(), new long[] {})); }
@Test public void testBigintSerializedSize() { BlockBuilder builder = BIGINT.createBlockBuilder(null, 5); // empty page Page page = new Page(builder.build()); int pageSize = serializedSize(ImmutableList.of(BIGINT), page); assertEquals(pageSize, 48); // page overhead ideally 35 but since a 0 sized block will be a RLEBlock we have an overhead of 13 // page with one value BIGINT.writeLong(builder, 123); pageSize = 35; // Now we have moved to the normal block implementation so the page size overhead is 35 page = new Page(builder.build()); int firstValueSize = serializedSize(ImmutableList.of(BIGINT), page) - pageSize; assertEquals(firstValueSize, 9); // value size + value overhead // page with two values BIGINT.writeLong(builder, 456); page = new Page(builder.build()); int secondValueSize = serializedSize(ImmutableList.of(BIGINT), page) - (pageSize + firstValueSize); assertEquals(secondValueSize, 8); // value size (value overhead is shared with previous value) }
public long getCurrentJoinPosition(LookupSource lookupSource) { if (currentRowContainsNull()) { return -1; } if (probeHashBlock.isPresent()) { long rawHash = BIGINT.getLong(probeHashBlock.get(), position); return lookupSource.getJoinPosition(position, probePage, page, rawHash); } return lookupSource.getJoinPosition(position, probePage, page); }
private Block createRowNumberBlock() { BlockBuilder rowNumberBlock = BIGINT.createFixedSizeBlockBuilder(inputPage.getPositionCount()); for (int currentPosition = 0; currentPosition < inputPage.getPositionCount(); currentPosition++) { long partitionId = getPartitionId(currentPosition); long nextRowCount = partitionRowCount.get(partitionId) + 1; BIGINT.writeLong(rowNumberBlock, nextRowCount); partitionRowCount.set(partitionId, nextRowCount); } return rowNumberBlock.build(); }
@Override public Block[] getSequenceBlocks(int start, int length) { BlockBuilder blockBuilder = BIGINT.createBlockBuilder(null, length); for (int i = start; i < start + length; i++) { BIGINT.writeLong(blockBuilder, i); } return new Block[] {blockBuilder.build()}; }
public static Block createLongSequenceBlock(int start, int end) { BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(end - start); for (int i = start; i < end; i++) { BIGINT.writeLong(builder, i); } return builder.build(); }
@Test public void testBackwardsCompatible() { ClientTypeSignature signature = new ClientTypeSignature(StandardTypes.ARRAY, ImmutableList.of(new ClientTypeSignatureParameter(TypeSignatureParameter.of(BIGINT.getTypeSignature())))); ClientTypeSignature legacy = CLIENT_TYPE_SIGNATURE_CODEC.fromJson("{\"rawType\":\"array\",\"literalArguments\":[],\"typeArguments\":[{\"rawType\":\"bigint\",\"literalArguments\":[],\"typeArguments\":[]}]}"); assertEquals(legacy, signature); }
@Test public void testSanityFilterOnRLE() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression filter = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L); }
private static AggregationNode.Aggregation countWithFilter(Expression condition) { FunctionCall countCall = new FunctionCall( QualifiedName.of("count"), Optional.empty(), Optional.of(condition), Optional.empty(), false, ImmutableList.<Expression>of()); /* arguments */ return new AggregationNode.Aggregation( countCall, new Signature("count", FunctionKind.AGGREGATE, BIGINT.getTypeSignature()), Optional.<Symbol>empty()); /* mask */ }
@Override protected List<? extends OperatorFactory> createOperatorFactories() { OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey"); InternalAggregationFunction countFunction = localQueryRunner.getMetadata().getFunctionRegistry().getAggregateFunctionImplementation( new Signature("count", AGGREGATE, BIGINT.getTypeSignature())); AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(countFunction.bind(ImmutableList.of(0), Optional.empty())), false); return ImmutableList.of(tableScanOperator, aggregationOperator); }
@Test public void testExpressionProfiler() internalOperator(ADD, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature())), BIGINT, field(0, BIGINT), Supplier<PageProjection> projectionSupplier = functionCompiler.compileProjection(add10Expression, Optional.empty()); PageProjection projection = projectionSupplier.get(); Page page = new Page(createLongSequenceBlock(1, 11)); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); for (int i = 0; i < 100; i++) { profiler.start(); Work<Block> work = projection.project(SESSION, new DriverYieldSignal(), page, SelectedPositions.positionsRange(0, page.getPositionCount())); if (i < 10) { profiler.stop(page.getPositionCount()); assertTrue(profiler.isExpressionExpensive()); profiler.stop(page.getPositionCount()); assertFalse(profiler.isExpressionExpensive());
@Test public void testReverseSortedPositionLinks() { JoinFilterFunction filterFunction = (leftAddress, rightPosition, rightPage) -> BIGINT.getLong(TEST_PAGE.getBlock(0), leftAddress) < 4; PositionLinks.FactoryBuilder factoryBuilder = buildSortedPositionLinks(); PositionLinks positionLinks = factoryBuilder.build().create(ImmutableList.of(filterFunction)); assertEquals(positionLinks.start(0, 0, TEST_PAGE), 0); assertEquals(positionLinks.next(0, 0, TEST_PAGE), 1); assertEquals(positionLinks.next(1, 0, TEST_PAGE), 2); assertEquals(positionLinks.next(2, 0, TEST_PAGE), 3); assertEquals(positionLinks.next(3, 0, TEST_PAGE), -1); assertEquals(positionLinks.start(10, 0, TEST_PAGE), -1); }
@Setup public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); metadata.addFunctions(new FunctionListBuilder().function(EXACT_ARRAY_FILTER_FUNCTION).getFunctions()); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); Signature signature = new Signature(name, FunctionKind.SCALAR, arrayType.getTypeSignature(), arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)")); Signature greaterThan = new Signature("$operator$" + GREATER_THAN.name(), FunctionKind.SCALAR, BOOLEAN.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()); projectionsBuilder.add(new CallExpression(signature, arrayType, ImmutableList.of( field(0, arrayType), new LambdaDefinitionExpression( ImmutableList.of(BIGINT), ImmutableList.of("x"), new CallExpression(greaterThan, BOOLEAN, ImmutableList.of(new VariableReferenceExpression("x", BIGINT), constant(0L, BIGINT))))))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } ImmutableList<RowExpression> projections = projectionsBuilder.build(); pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); page = new Page(blocks); }
@Test public void testBinaryMergeIteratorOverEmptyPageAndNonEmptyPage() { Page emptyPage = new Page(0, BIGINT.createFixedSizeBlockBuilder(0).build()); Page page = rowPagesBuilder(BIGINT).row(42).build().get(0); WorkProcessor<Page> mergedPage = new MergeHashSort(newSimpleAggregatedMemoryContext()).merge( ImmutableList.of(BIGINT), ImmutableList.of(BIGINT), ImmutableList.of(ImmutableList.of(emptyPage, page).iterator()).stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList()), new DriverYieldSignal()); assertTrue(mergedPage.process()); Page actualPage = mergedPage.getResult(); assertEquals(actualPage.getPositionCount(), 1); assertEquals(actualPage.getChannelCount(), 1); assertEquals(actualPage.getBlock(0).getLong(0, 0), 42); assertFinishes(mergedPage); }
@Test public void testJsonRoundTrip() { TypeSignature bigint = BIGINT.getTypeSignature(); assertJsonRoundTrip(new ClientTypeSignature(bigint)); assertJsonRoundTrip(new ClientTypeSignature( "array", ImmutableList.of(new ClientTypeSignatureParameter(TypeSignatureParameter.of(bigint))))); assertJsonRoundTrip(new ClientTypeSignature( "foo", ImmutableList.of(new ClientTypeSignatureParameter(TypeSignatureParameter.of(42))))); assertJsonRoundTrip(new ClientTypeSignature( "row", ImmutableList.of( new ClientTypeSignatureParameter(TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("foo", false)), bigint))), new ClientTypeSignatureParameter(TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("bar", false)), bigint)))))); }
@Test public void testFixedTypeParameterParse() { Signature expectedSignature = new Signature( "fixed_type_parameter_scalar_function", FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()), false); List<SqlScalarFunction> functions = ScalarFromAnnotationsParser.parseFunctionDefinition(FixedTypeParameterScalarFunction.class); assertEquals(functions.size(), 1); ParametricScalar scalar = (ParametricScalar) functions.get(0); assertImplementationCount(scalar, 1, 0, 0); assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); assertFalse(scalar.isHidden()); assertEquals(scalar.getDescription(), "Parametric scalar that uses TypeParameter with fixed type"); }
@Override public Block toBlock(Type desiredType) { checkArgument(BIGINT.equals(desiredType), "type doesn't match: %s", desiredType); int numberOfRecords = numberOfRecords(); return new LongArrayBlock( numberOfRecords, Optional.ofNullable(nulls), longs == null ? new long[numberOfRecords] : longs); }
@Test public void testBinaryMergeIteratorOverEmptyPage() { Page emptyPage = new Page(0, BIGINT.createFixedSizeBlockBuilder(0).build()); WorkProcessor<Page> mergedPage = new MergeHashSort(newSimpleAggregatedMemoryContext()).merge( ImmutableList.of(BIGINT), ImmutableList.of(BIGINT), ImmutableList.of(ImmutableList.of(emptyPage).iterator()).stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList()), new DriverYieldSignal()); assertFinishes(mergedPage); }
public static Signature dereferenceSignature(Type returnType, RowType rowType) { return internalScalarFunction(DEREFERENCE, returnType.getTypeSignature(), ImmutableList.of(rowType.getTypeSignature(), BigintType.BIGINT.getTypeSignature())); } }
@Test public void testFileVarbinarySpiller() throws Exception { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARBINARY); BlockBuilder col1 = BIGINT.createBlockBuilder(null, 1); BlockBuilder col2 = DOUBLE.createBlockBuilder(null, 1); BlockBuilder col3 = VARBINARY.createBlockBuilder(null, 1); col1.writeLong(42).closeEntry(); col2.writeLong(doubleToLongBits(43.0)).closeEntry(); col3.writeLong(doubleToLongBits(43.0)).writeLong(1).closeEntry(); Page page = new Page(col1.build(), col2.build(), col3.build()); try (Spiller spiller = factory.create(TYPES, bytes -> {}, memoryContext)) { testSpiller(types, spiller, ImmutableList.of(page)); } }