private static List<ParameterMetadata> createInputParameterMetadata(List<Type> valueTypes) { return ImmutableList.<ParameterMetadata>builder() .add(new ParameterMetadata(STATE)) .addAll(valueTypes.stream().map(valueType -> new ParameterMetadata(INPUT_CHANNEL, valueType)).collect(Collectors.toList())) .build(); }
private static List<MemoryPool> getMemoryPools(LocalMemoryManager localMemoryManager) { requireNonNull(localMemoryManager, "localMemoryManager can not be null"); ImmutableList.Builder<MemoryPool> builder = new ImmutableList.Builder<>(); builder.add(localMemoryManager.getGeneralPool()); localMemoryManager.getReservedPool().ifPresent(builder::add); return builder.build(); }
@Override public ActualProperties visitTableScan(TableScanNode node, List<ActualProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); ActualProperties.Builder properties = ActualProperties.builder(); // Globally constant assignments Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) .forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue())); Map<Symbol, NullableValue> symbolConstants = globalConstants.entrySet().stream() .filter(entry -> assignments.containsKey(entry.getKey())) .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue)); properties.constants(symbolConstants); // Partitioning properties properties.global(deriveGlobalProperties(layout, assignments, globalConstants)); // Append the global constants onto the local properties to maximize their translation potential List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder() .addAll(globalConstants.keySet().stream().map(ConstantProperty::new).iterator()) .addAll(layout.getLocalProperties()) .build(); properties.local(LocalProperties.translate(constantAppendedLocalProperties, column -> Optional.ofNullable(assignments.get(column)))); return properties.build(); }
public static List<String> listAllDataPaths(SemiTransactionalHiveMetastore metastore, String schemaName, String tableName) { ImmutableList.Builder<String> locations = ImmutableList.builder(); Table table = metastore.getTable(schemaName, tableName).get(); if (table.getStorage().getLocation() != null) { // For partitioned table, there should be nothing directly under this directory. // But including this location in the set makes the directory content assert more // extensive, which is desirable. locations.add(table.getStorage().getLocation()); } Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName); if (partitionNames.isPresent()) { metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).values().stream() .map(Optional::get) .map(partition -> partition.getStorage().getLocation()) .filter(location -> !location.startsWith(table.getStorage().getLocation())) .forEach(locations::add); } return locations.build(); }
public List<File> files() { checkState(location.exists(), "location %s doesn't exist", location); if (!pattern.isPresent()) { return ImmutableList.of(location); } checkState(location.isDirectory(), "location %s is not a directory", location); try (DirectoryStream<Path> paths = newDirectoryStream(location.toPath(), pattern.get())) { ImmutableList.Builder<File> builder = ImmutableList.builder(); for (Path path : paths) { builder.add(path.toFile()); } List<File> files = builder.build(); if (files.isEmpty()) { throw new PrestoException(LOCAL_FILE_NO_FILES, "No matching files found in directory: " + location); } return files.stream() .sorted((o1, o2) -> Long.compare(o2.lastModified(), o1.lastModified())) .collect(Collectors.toList()); } catch (IOException e) { throw new PrestoException(LOCAL_FILE_FILESYSTEM_ERROR, "Error listing files in directory: " + location, e); } } }
private WorkProcessor<Page> mergeFromDiskAndMemory() { checkState(spiller.isPresent()); hashAggregationBuilder.setOutputPartial(); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), ImmutableList.<WorkProcessor<Page>>builder() .addAll(spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList())) .add(hashAggregationBuilder.buildHashSortedResult()) .build(), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, max(memoryLimitForMerge - memoryLimitForMergeWithMemory, 1L)); }
private static List<ColumnStatistics> toFileStats(List<List<ColumnStatistics>> stripes) { if (stripes.isEmpty()) { return ImmutableList.of(); } int columnCount = stripes.get(0).size(); checkArgument(stripes.stream().allMatch(stripe -> columnCount == stripe.size())); ImmutableList.Builder<ColumnStatistics> fileStats = ImmutableList.builder(); for (int i = 0; i < columnCount; i++) { int column = i; fileStats.add(ColumnStatistics.mergeColumnStatistics(stripes.stream() .map(stripe -> stripe.get(column)) .collect(toList()))); } return fileStats.build(); }
public static Optional<HiveBucketHandle> getHiveBucketHandle(Table table) { Optional<HiveBucketProperty> hiveBucketProperty = table.getStorage().getBucketProperty(); if (!hiveBucketProperty.isPresent()) { return Optional.empty(); } Map<String, HiveColumnHandle> map = getRegularColumnHandles(table).stream() .collect(Collectors.toMap(HiveColumnHandle::getName, identity())); ImmutableList.Builder<HiveColumnHandle> bucketColumns = ImmutableList.builder(); for (String bucketColumnName : hiveBucketProperty.get().getBucketedBy()) { HiveColumnHandle bucketColumnHandle = map.get(bucketColumnName); if (bucketColumnHandle == null) { throw new PrestoException( HIVE_INVALID_METADATA, format("Table '%s.%s' is bucketed on non-existent column '%s'", table.getDatabaseName(), table.getTableName(), bucketColumnName)); } bucketColumns.add(bucketColumnHandle); } int bucketCount = hiveBucketProperty.get().getBucketCount(); return Optional.of(new HiveBucketHandle(bucketColumns.build(), bucketCount, bucketCount)); }
@Setup public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; Type returnType = new ArrayType(BOOLEAN); for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); Signature signature = new Signature("transform", FunctionKind.SCALAR, returnType.getTypeSignature(), arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)")); Signature greaterThan = new Signature("$operator$" + GREATER_THAN.name(), FunctionKind.SCALAR, BOOLEAN.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()); projectionsBuilder.add(new CallExpression(signature, returnType, ImmutableList.of( new InputReferenceExpression(0, arrayType), new LambdaDefinitionExpression( ImmutableList.of(BIGINT), ImmutableList.of("x"), new CallExpression(greaterThan, BOOLEAN, ImmutableList.of(new VariableReferenceExpression("x", BIGINT), new ConstantExpression(0L, BIGINT))))))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } ImmutableList<RowExpression> projections = projectionsBuilder.build(); pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); pageBuilder = new PageBuilder(projections.stream().map(RowExpression::getType).collect(Collectors.toList())); page = new Page(blocks); }
@Override public Optional<Symbol> getAssignedSymbol(PlanNode node, Session session, Metadata metadata, SymbolAliases symbolAliases) { Optional<Symbol> result = Optional.empty(); ImmutableList.Builder<Expression> matchesBuilder = ImmutableList.builder(); Map<Symbol, Expression> assignments = getAssignments(node); if (assignments == null) { return result; } ExpressionVerifier verifier = new ExpressionVerifier(symbolAliases); for (Map.Entry<Symbol, Expression> assignment : assignments.entrySet()) { if (verifier.process(assignment.getValue(), expression)) { result = Optional.of(assignment.getKey()); matchesBuilder.add(assignment.getValue()); } } List<Expression> matches = matchesBuilder.build(); checkState(matches.size() < 2, "Ambiguous expression %s matches multiple assignments", expression, (matches.stream().map(Expression::toString).collect(Collectors.joining(", ")))); return result; }
private static List<Symbol> flipOutputSymbols(List<Symbol> outputSymbols, PlanNode left, PlanNode right) { List<Symbol> leftSymbols = outputSymbols.stream() .filter(symbol -> left.getOutputSymbols().contains(symbol)) .collect(Collectors.toList()); List<Symbol> rightSymbols = outputSymbols.stream() .filter(symbol -> right.getOutputSymbols().contains(symbol)) .collect(Collectors.toList()); return ImmutableList.<Symbol>builder() .addAll(rightSymbols) .addAll(leftSymbols) .build(); }
private static Expression extractCommonPredicates(LogicalBinaryExpression node) { List<List<Expression>> subPredicates = getSubPredicates(node); Set<Expression> commonPredicates = ImmutableSet.copyOf(subPredicates.stream() .map(Visitor::filterDeterministicPredicates) .reduce(Sets::intersection) .orElse(emptySet())); List<List<Expression>> uncorrelatedSubPredicates = subPredicates.stream() .map(predicateList -> removeAll(predicateList, commonPredicates)) .collect(toImmutableList()); LogicalBinaryExpression.Operator flippedOperator = node.getOperator().flip(); List<Expression> uncorrelatedPredicates = uncorrelatedSubPredicates.stream() .map(predicate -> combinePredicates(flippedOperator, predicate)) .collect(toImmutableList()); Expression combinedUncorrelatedPredicates = combinePredicates(node.getOperator(), uncorrelatedPredicates); return combinePredicates(flippedOperator, ImmutableList.<Expression>builder() .addAll(commonPredicates) .add(combinedUncorrelatedPredicates) .build()); }
@Override public List<GrantInfo> listTablePrivileges(ConnectorSession session, SchemaTablePrefix schemaTablePrefix) { ImmutableList.Builder<GrantInfo> grantInfos = ImmutableList.builder(); for (SchemaTableName tableName : listTables(session, schemaTablePrefix)) { Set<PrivilegeInfo> privileges = metastore.getTablePrivileges(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).stream() .map(HivePrivilegeInfo::toPrivilegeInfo) .flatMap(Set::stream) .collect(toImmutableSet()); grantInfos.add( new GrantInfo( privileges, session.getIdentity(), tableName, Optional.empty(), // Can't access grantor Optional.empty())); // Can't access withHierarchy } return grantInfos.build(); }
private static List<OptionalDouble> getActualValues(List<Metric> metrics, String query, QueryRunner runner) { String statsQuery = "SELECT " + metrics.stream().map(Metric::getComputingAggregationSql).collect(joining(",")) + " FROM (" + query + ")"; try { MaterializedRow actualValuesRow = getOnlyElement(runner.execute(statsQuery).getMaterializedRows()); ImmutableList.Builder<OptionalDouble> actualValues = ImmutableList.builder(); for (int i = 0; i < metrics.size(); ++i) { actualValues.add(metrics.get(i).getValueFromAggregationQueryResult(actualValuesRow.getField(i))); } return actualValues.build(); } catch (Exception e) { throw new RuntimeException(format("Failed to execute query to compute actual values: %s", statsQuery), e); } }
private PhysicalOperation( OperatorFactory operatorFactory, Map<Symbol, Integer> layout, LocalExecutionPlanContext context, Optional<PhysicalOperation> source, PipelineExecutionStrategy pipelineExecutionStrategy) { requireNonNull(operatorFactory, "operatorFactory is null"); requireNonNull(layout, "layout is null"); requireNonNull(context, "context is null"); requireNonNull(source, "source is null"); requireNonNull(pipelineExecutionStrategy, "pipelineExecutionStrategy is null"); this.operatorFactories = ImmutableList.<OperatorFactory>builder() .addAll(source.map(PhysicalOperation::getOperatorFactories).orElse(ImmutableList.of())) .add(operatorFactory) .build(); this.layout = ImmutableMap.copyOf(layout); this.types = toTypes(layout, context); this.pipelineExecutionStrategy = pipelineExecutionStrategy; }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, ValuesNode valuesNode, Set<Symbol> referencedOutputs) { List<Symbol> newOutputs = filteredCopy(valuesNode.getOutputSymbols(), referencedOutputs::contains); // for each output of project, the corresponding column in the values node int[] mapping = new int[newOutputs.size()]; for (int i = 0; i < mapping.length; i++) { mapping[i] = valuesNode.getOutputSymbols().indexOf(newOutputs.get(i)); } ImmutableList.Builder<List<Expression>> rowsBuilder = ImmutableList.builder(); for (List<Expression> row : valuesNode.getRows()) { rowsBuilder.add(Arrays.stream(mapping) .mapToObj(row::get) .collect(Collectors.toList())); } return Optional.of(new ValuesNode(valuesNode.getId(), newOutputs, rowsBuilder.build())); } }
private boolean hasAggregates(QuerySpecification node) { ImmutableList.Builder<Node> toExtractBuilder = ImmutableList.builder(); toExtractBuilder.addAll(node.getSelect().getSelectItems().stream() .filter(SingleColumn.class::isInstance) .collect(toImmutableList())); toExtractBuilder.addAll(getSortItemsFromOrderBy(node.getOrderBy())); node.getHaving().ifPresent(toExtractBuilder::add); List<FunctionCall> aggregates = extractAggregateFunctions(toExtractBuilder.build(), metadata.getFunctionRegistry()); return !aggregates.isEmpty(); }
private static List<TypeSignatureParameter> typeParameters(List<Type> argumentTypes, Type returnType) { requireNonNull(returnType, "returnType is null"); requireNonNull(argumentTypes, "argumentTypes is null"); ImmutableList.Builder<TypeSignatureParameter> builder = ImmutableList.builder(); argumentTypes.stream() .map(Type::getTypeSignature) .map(TypeSignatureParameter::of) .forEach(builder::add); builder.add(TypeSignatureParameter.of(returnType.getTypeSignature())); return builder.build(); }