@Test public void parseSignatureWithLiterals() { TypeSignature result = parseTypeSignature("decimal(X,42)", ImmutableSet.of("X")); assertEquals(result.getParameters().size(), 2); assertEquals(result.getParameters().get(0).isVariable(), true); assertEquals(result.getParameters().get(1).isLongLiteral(), true); }
static void checkTypeParameters(TypeSignature typeSignature, Set<String> typeParameterNames, AnnotatedElement element) { // Check recursively if `typeSignature` contains something like `T<bigint>` if (typeParameterNames.contains(typeSignature.getBase())) { checkArgument(typeSignature.getParameters().isEmpty(), "Expected type parameter not to take parameters, but got %s on method [%s]", typeSignature.getBase(), element); return; } for (TypeSignatureParameter parameter : typeSignature.getParameters()) { Optional<TypeSignature> childTypeSignature = parameter.getTypeSignatureOrNamedTypeSignature(); if (childTypeSignature.isPresent()) { checkTypeParameters(childTypeSignature.get(), typeParameterNames, element); } } }
private static TypeSignature makeSignature(List<Field> fields) { int size = fields.size(); if (size == 0) { throw new IllegalArgumentException("Row type must have at least 1 field"); } List<TypeSignatureParameter> parameters = fields.stream() .map(field -> TypeSignatureParameter.of(new NamedTypeSignature(field.getName().map(name -> new RowFieldName(name, false)), field.getType().getTypeSignature()))) .collect(Collectors.toList()); return new TypeSignature(ROW, parameters); }
public ClientTypeSignature(TypeSignature typeSignature) { this( typeSignature.getBase(), Lists.transform(typeSignature.getParameters(), ClientTypeSignatureParameter::new)); }
public static TypeSignature applyBoundVariables(TypeSignature typeSignature, BoundVariables boundVariables) { String baseType = typeSignature.getBase(); if (boundVariables.containsTypeVariable(baseType)) { checkState(typeSignature.getParameters().isEmpty(), "Type parameters cannot have parameters"); return boundVariables.getTypeVariable(baseType).getTypeSignature(); } List<TypeSignatureParameter> parameters = typeSignature.getParameters().stream() .map(typeSignatureParameter -> applyBoundVariables(typeSignatureParameter, boundVariables)) .collect(toList()); return new TypeSignature(baseType, parameters); }
@Test public void testParquetTupleDomainStruct() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_struct", HiveType.valueOf("struct<a:int,b:int>"), parseTypeSignature(StandardTypes.ROW), 0, REGULAR, Optional.empty()); RowType.Field rowField = new RowType.Field(Optional.of("my_struct"), INTEGER); RowType rowType = RowType.from(ImmutableList.of(rowField)); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(rowType))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_struct", new PrimitiveType(OPTIONAL, INT32, "a"), new PrimitiveType(OPTIONAL, INT32, "b"))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@Test public void testBindLiteralForDecimal() { TypeSignature leftType = parseTypeSignature("decimal(p1,s1)", ImmutableSet.of("p1", "s1")); TypeSignature rightType = parseTypeSignature("decimal(p2,s2)", ImmutableSet.of("p2", "s2")); Signature function = functionSignature() .returnType(parseTypeSignature(StandardTypes.BOOLEAN)) .argumentTypes(leftType, rightType) .build(); assertThat(function) .boundTo("decimal(2,1)", "decimal(1,0)") .produces(new BoundVariables( ImmutableMap.of(), ImmutableMap.of( "p1", 2L, "s1", 1L, "p2", 1L, "s2", 0L))); }
@Test public void testDateColumn() { IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(new TypeRegistry()); List<HiveColumnHandle> columns = ImmutableList.of( new HiveColumnHandle("t1", HIVE_TIMESTAMP, parseTypeSignature(TIMESTAMP), 0, REGULAR, Optional.empty()), new HiveColumnHandle("t2", HIVE_DATE, parseTypeSignature(StandardTypes.DATE), 1, REGULAR, Optional.empty())); TupleDomain<HiveColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of( columns.get(1), Domain.create(SortedRangeSet.copyOf(DATE, ImmutableList.of(Range.equal(DATE, (long) DateTimeUtils.parseDate("2001-08-22")))), false))); assertEquals("SELECT s._1, s._2 FROM S3Object s WHERE (case s._2 when '' then null else CAST(s._2 AS TIMESTAMP) end = `2001-08-22`)", queryBuilder.buildSql(columns, tupleDomain)); }
@Test public void testParquetTupleDomainPrimitive() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_primitive", HiveType.valueOf("bigint"), parseTypeSignature(StandardTypes.BIGINT), 0, REGULAR, Optional.empty()); Domain singleValueDomain = Domain.singleValue(BIGINT, 123L); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, singleValueDomain)); MessageType fileSchema = new MessageType("hive_schema", new PrimitiveType(OPTIONAL, INT64, "my_primitive")); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertEquals(tupleDomain.getDomains().get().size(), 1); ColumnDescriptor descriptor = tupleDomain.getDomains().get().keySet().iterator().next(); assertEquals(descriptor.getPath().length, 1); assertEquals(descriptor.getPath()[0], "my_primitive"); Domain predicateDomain = Iterables.getOnlyElement(tupleDomain.getDomains().get().values()); assertEquals(predicateDomain, singleValueDomain); }
@Test public void testParquetTupleDomainPrimitiveArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array", new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
if (!lambdaArgumentTypes.isPresent()) { return SolverReturnStatus.UNCHANGED_NOT_SATISFIED; if (!typeSignatureProvider.hasDependency()) { actualLambdaTypeSignature = typeSignatureProvider.getTypeSignature(); if (!FunctionType.NAME.equals(actualLambdaTypeSignature.getBase()) || !getLambdaArgumentTypeSignatures(actualLambdaTypeSignature).equals(toTypeSignatures(lambdaArgumentTypes.get()))) { return SolverReturnStatus.UNSOLVABLE; actualLambdaTypeSignature = typeSignatureProvider.getTypeSignature(lambdaArgumentTypes.get()); if (!FunctionType.NAME.equals(actualLambdaTypeSignature.getBase())) { return SolverReturnStatus.UNSOLVABLE; Type actualReturnType = ((FunctionType) actualLambdaType).getReturnType(); ImmutableList.Builder<TypeConstraintSolver> constraintsBuilder = ImmutableList.builder(); if (!appendTypeRelationshipConstraintSolver(constraintsBuilder, formalLambdaReturnTypeSignature, new TypeSignatureProvider(actualReturnType.getTypeSignature()), false)) { return SolverReturnStatus.UNSOLVABLE; if (!appendConstraintSolvers(constraintsBuilder, formalLambdaReturnTypeSignature, new TypeSignatureProvider(actualReturnType.getTypeSignature()), allowCoercion)) { return SolverReturnStatus.UNSOLVABLE;
private static Block serializeStruct(Type type, BlockBuilder builder, Object object) { if (object == null) { requireNonNull(builder, "builder is null"); builder.appendNull(); return builder.build(); } if (builder == null) { builder = type.createBlockBuilder(null, 1); } BlockBuilder currentBuilder = builder.beginBlockEntry(); List<Type> typeParameters = type.getTypeParameters(); for (int i = 0; i < typeParameters.size(); i++) { Optional<String> fieldName = type.getTypeSignature().getParameters().get(i).getNamedTypeSignature().getName(); String name = ""; if (fieldName.isPresent()) { name = fieldName.get().toLowerCase(ENGLISH); } Object value = ((Map) object).get(name); serializeObject(typeParameters.get(i), currentBuilder, value); } builder.closeEntry(); return (Block) type.getObject(builder, 0); }
@Test public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); Type elementType; Object increment; mapType.getTypeSignature(), mapType.getTypeSignature(), parseTypeSignature(format("function(%s, %s, %s)", type, type, type))); Signature add = new Signature("$operator$" + ADD.name(), FunctionKind.SCALAR, elementType.getTypeSignature(), elementType.getTypeSignature(), elementType.getTypeSignature()); projectionsBuilder.add(call(signature, mapType, ImmutableList.of( field(0, mapType), new LambdaDefinitionExpression( ImmutableList.of(elementType, elementType), ImmutableList.of("x", "y"), call(add, elementType, ImmutableList.of( pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); page = new Page(block);
@Test public void testLearn() { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(TypeSignatureParameter.of(parseTypeSignature(StandardTypes.BIGINT)), TypeSignatureParameter.of(parseTypeSignature(StandardTypes.DOUBLE)))); InternalAggregationFunction aggregation = generateInternalAggregationFunction(LearnClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), mapType.getTypeSignature()), typeManager); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator()); }
@Test public void testNonDeterministicProject() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression random = new CallExpression( new Signature("random", SCALAR, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); InputReferenceExpression col0 = field(0, BIGINT); CallExpression lessThanRandomExpression = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(col0, random)); PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(lessThanRandomExpression), MAX_BATCH_SIZE).get(); assertFalse(new DeterminismEvaluator(metadataManager.getFunctionRegistry()).isDeterministic(lessThanRandomExpression)); Page page = new Page(createLongDictionaryBlock(1, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertFalse(outputPage.getBlock(0) instanceof DictionaryBlock); }
public TestRealHistogramAggregation() { TypeRegistry typeRegistry = new TypeRegistry(); FunctionRegistry functionRegistry = new FunctionRegistry(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); InternalAggregationFunction function = functionRegistry.getAggregateFunctionImplementation( new Signature("numeric_histogram", AGGREGATE, parseTypeSignature("map(real, real)"), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.REAL), parseTypeSignature(StandardTypes.DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); }
private AggregationNode aggregation(String id, PlanNode source) { AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), new Signature("count", AGGREGATE, parseTypeSignature(StandardTypes.BIGINT)), Optional.empty()); return new AggregationNode( new PlanNodeId(id), source, ImmutableMap.of(new Symbol("count"), aggregation), singleGroupingSet(source.getOutputSymbols()), ImmutableList.of(), AggregationNode.Step.FINAL, Optional.empty(), Optional.empty()); }
break; return primitiveType.getTypeSignature(); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; TypeSignature keyType = getTypeSignature(mapTypeInfo.getMapKeyTypeInfo()); TypeSignature valueType = getTypeSignature(mapTypeInfo.getMapValueTypeInfo()); return new TypeSignature( StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType), TypeSignatureParameter.of(valueType))); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; TypeSignature elementType = getTypeSignature(listTypeInfo.getListElementTypeInfo()); return new TypeSignature( StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType))); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, format("Invalid Hive struct type: %s", typeInfo)); ImmutableList.Builder<TypeSignatureParameter> typeSignatureBuilder = ImmutableList.builder(); for (int i = 0; i < structFieldTypeInfos.size(); i++) { TypeSignature typeSignature = getTypeSignature(structFieldTypeInfos.get(i)); typeSignatureBuilder.add(TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName(rowFieldName, false)), typeSignature))); return new TypeSignature(StandardTypes.ROW, typeSignatureBuilder.build());