public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanNode child, PartitioningScheme partitioningScheme) { if (partitioningScheme.getPartitioning().getHandle().isSingleNode()) { return gatheringExchange(id, scope, child); } return new ExchangeNode( id, ExchangeNode.Type.REPARTITION, scope, partitioningScheme, ImmutableList.of(child), ImmutableList.of(partitioningScheme.getOutputLayout()).asList(), Optional.empty()); }
private PlanNode addGatheringIntermediate(AggregationNode aggregation, PlanNodeIdAllocator idAllocator) { verify(aggregation.getGroupingKeys().isEmpty(), "Should be an un-grouped aggregation"); ExchangeNode gatheringExchange = ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, aggregation); return new AggregationNode( idAllocator.getNextId(), gatheringExchange, outputsAsInputs(aggregation.getAggregations()), aggregation.getGroupingSets(), aggregation.getPreGroupedSymbols(), AggregationNode.Step.INTERMEDIATE, aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTopN(TopNNode node, PreferredProperties preferredProperties) { PlanWithProperties child; switch (node.getStep()) { case SINGLE: case FINAL: child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } break; case PARTIAL: child = planChild(node, PreferredProperties.any()); break; default: throw new UnsupportedOperationException(format("Unsupported step for TopN [%s]", node.getStep())); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitOutput(OutputNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode() && isForceSingleNodeOutput(session)) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
ExchangeNode exchangeNode = gatheringExchange(idAllocator.getNextId(), LOCAL, planWithProperties.getNode()); return deriveProperties(exchangeNode, planWithProperties.getProperties()); ExchangeNode exchangeNode = gatheringExchange( idAllocator.getNextId(), LOCAL,
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public Result apply(AggregationNode aggregation, Captures captures, Context context) { Lookup lookup = context.getLookup(); PlanNodeIdAllocator idAllocator = context.getIdAllocator(); Session session = context.getSession(); Optional<PlanNode> rewrittenSource = recurseToPartial(lookup.resolve(aggregation.getSource()), lookup, idAllocator); if (!rewrittenSource.isPresent()) { return Result.empty(); } PlanNode source = rewrittenSource.get(); if (getTaskConcurrency(session) > 1) { source = ExchangeNode.partitionedExchange( idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, source, new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), source.getOutputSymbols())); source = new AggregationNode( idAllocator.getNextId(), source, inputsAsOutputs(aggregation.getAggregations()), aggregation.getGroupingSets(), aggregation.getPreGroupedSymbols(), AggregationNode.Step.INTERMEDIATE, aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); source = ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, source); } return Result.ofPlanNode(aggregation.replaceChildren(ImmutableList.of(source))); }
if (!right.getProperties().isSingleNode()) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties());
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange( idAllocator.getNextId(), REMOTE, new DistinctLimitNode(idAllocator.getNextId(), child.getNode(), node.getLimit(), true, node.getDistinctSymbols(), node.getHashSymbol())), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitLimit(LimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( new LimitNode(idAllocator.getNextId(), child.getNode(), node.getCount(), true), child.getProperties()); child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties());
addExchange = partial -> gatheringExchange(idAllocator.getNextId(), REMOTE, partial);
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties());
(!isColocatedJoinEnabled(session) && hasMultipleSources(source.getNode(), filteringSource.getNode()))) { filteringSource = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, filteringSource.getNode()), filteringSource.getProperties());
private PlanWithProperties planReplicatedJoin(JoinNode node, PlanWithProperties left) { // Broadcast Join PlanWithProperties right = node.getRight().accept(this, PreferredProperties.any()); if (left.getProperties().isSingleNode()) { if (!right.getProperties().isSingleNode() || (!isColocatedJoinEnabled(session) && hasMultipleSources(left.getNode(), right.getNode()))) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } } else { right = withDerivedProperties( replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } return buildJoin(node, left, right, JoinNode.DistributionType.REPLICATED); }
@Override public PlanWithProperties visitOutput(OutputNode node, Context context) { PlanWithProperties child = planChild(node, context.withPreferredProperties(PreferredProperties.any())); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }