@Override public Void visitAggregation(AggregationNode node, Void context) { StringBuilder builder = new StringBuilder(); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { if (entry.getValue().getMask().isPresent()) { builder.append(format("%s := %s (mask = %s)\\n", entry.getKey(), entry.getValue().getCall(), entry.getValue().getMask().get())); } else { builder.append(format("%s := %s\\n", entry.getKey(), entry.getValue().getCall())); } } printNode(node, format("Aggregate[%s]", node.getStep()), builder.toString(), NODE_COLORS.get(NodeType.AGGREGATE)); return node.getSource().accept(this, context); }
private PhysicalOperation planGlobalAggregation(AggregationNode node, PhysicalOperation source, LocalExecutionPlanContext context) { ImmutableMap.Builder<Symbol, Integer> outputMappings = ImmutableMap.builder(); AggregationOperatorFactory operatorFactory = createAggregationOperatorFactory( node.getId(), node.getAggregations(), node.getStep(), 0, outputMappings, source, context, node.getStep().isOutputPartial()); return new PhysicalOperation(operatorFactory, outputMappings.build(), context, source); }
private static boolean isSupportedAggregationNode(AggregationNode aggregationNode) { // Don't split streaming aggregations if (aggregationNode.isStreamable()) { return false; } if (aggregationNode.getHashSymbol().isPresent()) { // TODO: add support for hash symbol in aggregation node return false; } return aggregationNode.getStep() == PARTIAL && aggregationNode.getGroupingSetCount() == 1; }
/** * Recurse through a series of preceding ExchangeNodes and ProjectNodes to find the preceding PARTIAL aggregation */ private Optional<PlanNode> recurseToPartial(PlanNode node, Lookup lookup, PlanNodeIdAllocator idAllocator) { if (node instanceof AggregationNode && ((AggregationNode) node).getStep() == AggregationNode.Step.PARTIAL) { return Optional.of(addGatheringIntermediate((AggregationNode) node, idAllocator)); } if (!(node instanceof ExchangeNode) && !(node instanceof ProjectNode)) { return Optional.empty(); } ImmutableList.Builder<PlanNode> builder = ImmutableList.builder(); for (PlanNode source : node.getSources()) { Optional<PlanNode> planNode = recurseToPartial(lookup.resolve(source), lookup, idAllocator); if (!planNode.isPresent()) { return Optional.empty(); } builder.add(planNode.get()); } return Optional.of(node.replaceChildren(builder.build())); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(AggregationNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { if (node.getGroupingSetCount() != 1) { return Optional.empty(); } if (node.getStep() != SINGLE) { return Optional.empty(); } return Optional.of(groupBy( statsProvider.getStats(node.getSource()), node.getGroupingKeys(), node.getAggregations())); }
private AggregationNode map(AggregationNode node, PlanNode source, PlanNodeId newNodeId) { ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder(); for (Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { aggregations.put(map(entry.getKey()), map(entry.getValue())); } return new AggregationNode( newNodeId, source, aggregations.build(), groupingSets( mapAndDistinct(node.getGroupingKeys()), node.getGroupingSetCount(), node.getGlobalGroupingSets()), ImmutableList.of(), node.getStep(), node.getHashSymbol().map(this::map), node.getGroupIdSymbol().map(this::map)); }
@Override public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); AggregationNode aggregationNode = (AggregationNode) node; if (aggregationNode.getStep() != step) { return NO_MATCH; } return match(); } }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getGroupingKeys()); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder(); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { Symbol symbol = entry.getKey(); if (context.get().contains(symbol)) { Aggregation aggregation = entry.getValue(); expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall())); aggregation.getMask().ifPresent(expectedInputs::add); aggregations.put(symbol, aggregation); } } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new AggregationNode(node.getId(), source, aggregations.build(), node.getGroupingSets(), ImmutableList.of(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol()); }
@Override public Void visitAggregation(AggregationNode node, Integer indent) { String type = ""; if (node.getStep() != AggregationNode.Step.SINGLE) { type = format("(%s)", node.getStep().toString()); } if (node.isStreamable()) { type = format("%s(STREAMING)", type); } String key = ""; if (!node.getGroupingKeys().isEmpty()) { key = node.getGroupingKeys().toString(); } print(indent, "- Aggregate%s%s%s => [%s]", type, key, formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { if (entry.getValue().getMask().isPresent()) { print(indent + 2, "%s := %s (mask = %s)", entry.getKey(), entry.getValue().getCall(), entry.getValue().getMask().get()); } else { print(indent + 2, "%s := %s", entry.getKey(), entry.getValue().getCall()); } } return processChildren(node, indent + 1); }
@Override public Void visitAggregation(AggregationNode node, Integer indent) { output( indent, "%s aggregation over (%s)", node.getStep().name().toLowerCase(ENGLISH), node.getGroupingKeys().stream() .map(Object::toString) .sorted() .collect(joining(", "))); return visitPlan(node, indent + 1); }
@Override protected Optional<PlanNode> pushDownProjectOff( PlanNodeIdAllocator idAllocator, AggregationNode aggregationNode, Set<Symbol> referencedOutputs) { Map<Symbol, AggregationNode.Aggregation> prunedAggregations = Maps.filterKeys( aggregationNode.getAggregations(), referencedOutputs::contains); if (prunedAggregations.size() == aggregationNode.getAggregations().size()) { return Optional.empty(); } // PruneAggregationSourceColumns will subsequently project off any newly unused inputs. return Optional.of( new AggregationNode( aggregationNode.getId(), aggregationNode.getSource(), prunedAggregations, aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), aggregationNode.getStep(), aggregationNode.getHashSymbol(), aggregationNode.getGroupIdSymbol())); } }
@Override public PlanNodeCostEstimate visitAggregation(AggregationNode node, Void context) { if (node.getStep() != FINAL && node.getStep() != SINGLE) { return PlanNodeCostEstimate.unknown(); } PlanNodeStatsEstimate aggregationStats = getStats(node); PlanNodeStatsEstimate sourceStats = getStats(node.getSource()); double cpuCost = sourceStats.getOutputSizeInBytes(node.getSource().getOutputSymbols(), types); double memoryCost = aggregationStats.getOutputSizeInBytes(node.getOutputSymbols(), types); return new PlanNodeCostEstimate(cpuCost, memoryCost, 0); }
private AggregationNode replaceAggregationSource( AggregationNode aggregation, PlanNode source, List<Symbol> groupingKeys) { return new AggregationNode( aggregation.getId(), source, aggregation.getAggregations(), singleGroupingSet(groupingKeys), ImmutableList.of(), aggregation.getStep(), aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); }
@Override public GroupedExecutionProperties visitAggregation(AggregationNode node, Void context) { GroupedExecutionProperties properties = node.getSource().accept(this, null); if (groupedExecutionForAggregation && properties.isCurrentNodeCapable()) { switch (node.getStep()) { case SINGLE: case FINAL: return new GroupedExecutionProperties(true, true, properties.capableTableScanNodes); case PARTIAL: case INTERMEDIATE: return properties; } } return GroupedExecutionProperties.notCapable(); }
@Override public PlanWithProperties visitAggregation(AggregationNode node, HashComputationSet parentPreference) { Optional<HashComputation> groupByHash = Optional.empty(); if (!node.isStreamable() && !canSkipHashGeneration(node.getGroupingKeys())) { groupByHash = computeHash(node.getGroupingKeys()); } // aggregation does not pass through preferred hash symbols HashComputationSet requiredHashes = new HashComputationSet(groupByHash); PlanWithProperties child = planAndEnforce(node.getSource(), requiredHashes, false, requiredHashes); Optional<Symbol> hashSymbol = groupByHash.map(child::getRequiredHashSymbol); return new PlanWithProperties( new AggregationNode( node.getId(), child.getNode(), node.getAggregations(), node.getGroupingSets(), node.getPreGroupedSymbols(), node.getStep(), hashSymbol, node.getGroupIdSymbol()), hashSymbol.isPresent() ? ImmutableMap.of(groupByHash.get(), hashSymbol.get()) : ImmutableMap.of()); }
@Override public Void visitAggregation(AggregationNode node, Void context) { visitPlan(node, context); AggregationNode.Step step = node.getStep(); switch (step) { case SINGLE: checkFunctionSignature(node.getAggregations()); checkFunctionCall(node.getAggregations()); break; case FINAL: checkFunctionSignature(node.getAggregations()); break; } return null; }
@Override public Result apply(AggregationNode parent, Captures captures, Context context) { ProjectNode child = captures.get(CHILD); boolean changed = false; Map<Symbol, AggregationNode.Aggregation> aggregations = new LinkedHashMap<>(parent.getAggregations()); for (Entry<Symbol, AggregationNode.Aggregation> entry : parent.getAggregations().entrySet()) { Symbol symbol = entry.getKey(); AggregationNode.Aggregation aggregation = entry.getValue(); if (isCountOverConstant(aggregation, child.getAssignments())) { changed = true; aggregations.put(symbol, new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), new Signature("count", AGGREGATE, parseTypeSignature(StandardTypes.BIGINT)), aggregation.getMask())); } } if (!changed) { return Result.empty(); } return Result.ofPlanNode(new AggregationNode( parent.getId(), child, aggregations, parent.getGroupingSets(), ImmutableList.of(), parent.getStep(), parent.getHashSymbol(), parent.getGroupIdSymbol())); }
private static int countFinalAggregationNodes(Plan plan) { return searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == FINAL) .count(); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Boolean> context) { boolean distinct = isDistinctOperator(node); PlanNode rewrittenNode = context.rewrite(node.getSource(), distinct); if (context.get() && distinct) { // Assumes underlying node has same output symbols as this distinct node return rewrittenNode; } return new AggregationNode( node.getId(), rewrittenNode, node.getAggregations(), node.getGroupingSets(), ImmutableList.of(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol()); }
private static int countSingleStreamingAggregations(Plan plan) { return searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == SINGLE && ((AggregationNode) node).isStreamable()) .count(); } }