@Override public Map<PlanNodeId, SplitSource> visitExchange(ExchangeNode node, Void context) { return processSources(node.getSources(), context); }
@Override public Range<Long> visitExchange(ExchangeNode node, Void context) { if (node.getSources().size() == 1) { return getOnlyElement(node.getSources()).accept(this, null); } return Range.atLeast(0L); }
@Override public Void visitExchange(ExchangeNode node, Void context) { for (PlanNode subPlanNode : node.getSources()) { subPlanNode.accept(this, context); } return null; }
@Override public Map<PlanNodeId, SplitSource> visitExchange(ExchangeNode node, Void context) { return processSources(node.getSources(), context); }
@Override public Void visitExchange(ExchangeNode node, Void context) { for (PlanNode subPlanNode : node.getSources()) { subPlanNode.accept(this, context); } return null; }
@Override public Range<Long> visitExchange(ExchangeNode node, Void context) { if (node.getSources().size() == 1) { return getOnlyElement(node.getSources()).accept(this, null); } return Range.atLeast(0L); }
@Override public Set<PlanFragmentId> visitExchange(ExchangeNode node, PlanFragmentId currentFragmentId) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the phased execution scheduler"); ImmutableSet.Builder<PlanFragmentId> allSources = ImmutableSet.builder(); // Link the source fragments together, so we only schedule one at a time. Set<PlanFragmentId> previousSources = ImmutableSet.of(); for (PlanNode subPlanNode : node.getSources()) { Set<PlanFragmentId> currentSources = subPlanNode.accept(this, currentFragmentId); allSources.addAll(currentSources); addEdges(previousSources, currentSources); previousSources = currentSources; } return allSources.build(); }
@Override public Set<PlanFragmentId> visitExchange(ExchangeNode node, PlanFragmentId currentFragmentId) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the phased execution scheduler"); ImmutableSet.Builder<PlanFragmentId> allSources = ImmutableSet.builder(); // Link the source fragments together, so we only schedule one at a time. Set<PlanFragmentId> previousSources = ImmutableSet.of(); for (PlanNode subPlanNode : node.getSources()) { Set<PlanFragmentId> currentSources = subPlanNode.accept(this, currentFragmentId); allSources.addAll(currentSources); addEdges(previousSources, currentSources); previousSources = currentSources; } return allSources.build(); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(ExchangeNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getInputs().get(i), node.getOutputSymbols()); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndMaxDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } verify(estimate.isPresent()); return estimate; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(ExchangeNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getInputs().get(i), node.getOutputSymbols()); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndMaxDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } verify(estimate.isPresent()); return estimate; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(Symbol::toSymbolReference) .map(ArgumentBinding::expressionBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(ArgumentBinding::columnBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
for (int i = 0; i < node.getSources().size(); i++) { Map<Symbol, SymbolReference> outputsToInputs = new HashMap<>(); for (int index = 0; index < node.getInputs().get(i).size(); index++) { PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) {
@Override public PlanNode visitExchange(ExchangeNode node, RewriteContext<Void> context) List<PlanNode> sources = node.getSources().stream() .map(context::rewrite) .collect(toImmutableList());
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }