public static Map<Symbol, Symbol> exchangeInputToOutput(ExchangeNode node, int sourceIndex) { List<Symbol> inputSymbols = node.getInputs().get(sourceIndex); Map<Symbol, Symbol> inputToOutput = new HashMap<>(); for (int i = 0; i < node.getOutputSymbols().size(); i++) { inputToOutput.put(inputSymbols.get(i), node.getOutputSymbols().get(i)); } return inputToOutput; }
@Override public Expression visitExchange(ExchangeNode node, Void context) { return deriveCommonPredicates(node, source -> { Map<Symbol, SymbolReference> mappings = new HashMap<>(); for (int i = 0; i < node.getInputs().get(source).size(); i++) { mappings.put( node.getOutputSymbols().get(i), node.getInputs().get(source).get(i).toSymbolReference()); } return mappings.entrySet(); }); }
public static Map<Symbol, Symbol> exchangeInputToOutput(ExchangeNode node, int sourceIndex) { List<Symbol> inputSymbols = node.getInputs().get(sourceIndex); Map<Symbol, Symbol> inputToOutput = new HashMap<>(); for (int i = 0; i < node.getOutputSymbols().size(); i++) { inputToOutput.put(inputSymbols.get(i), node.getOutputSymbols().get(i)); } return inputToOutput; }
@Override public Expression visitExchange(ExchangeNode node, Void context) { return deriveCommonPredicates(node, source -> { Map<Symbol, SymbolReference> mappings = new HashMap<>(); for (int i = 0; i < node.getInputs().get(source).size(); i++) { mappings.put( node.getOutputSymbols().get(i), node.getInputs().get(source).get(i).toSymbolReference()); } return mappings.entrySet(); }); }
private List<Symbol> canonicalizeExchangeNodeInputs(ExchangeNode node, int symbolIndex) { return node.getInputs().stream() .map(input -> canonicalize(input.get(symbolIndex))) .collect(toImmutableList()); }
private static Map<Symbol, SymbolReference> extractExchangeOutputToInput(ExchangeNode exchange, int sourceIndex) { Map<Symbol, SymbolReference> outputToInputMap = new HashMap<>(); for (int i = 0; i < exchange.getOutputSymbols().size(); i++) { outputToInputMap.put(exchange.getOutputSymbols().get(i), exchange.getInputs().get(sourceIndex).get(i).toSymbolReference()); } return outputToInputMap; } }
private List<Symbol> canonicalizeExchangeNodeInputs(ExchangeNode node, int symbolIndex) { return node.getInputs().stream() .map(input -> canonicalize(input.get(symbolIndex))) .collect(toImmutableList()); }
private static Map<Symbol, SymbolReference> extractExchangeOutputToInput(ExchangeNode exchange, int sourceIndex) { Map<Symbol, SymbolReference> outputToInputMap = new HashMap<>(); for (int i = 0; i < exchange.getOutputSymbols().size(); i++) { outputToInputMap.put(exchange.getOutputSymbols().get(i), exchange.getInputs().get(sourceIndex).get(i).toSymbolReference()); } return outputToInputMap; } }
private void mapExchangeNodeOutputToInputSymbols(ExchangeNode node) { checkState(node.getInputs().size() == 1); for (int symbolIndex = 0; symbolIndex < node.getOutputSymbols().size(); symbolIndex++) { Symbol canonicalOutput = canonicalize(node.getOutputSymbols().get(symbolIndex)); Symbol canonicalInput = canonicalize(node.getInputs().get(0).get(symbolIndex)); if (!canonicalOutput.equals(canonicalInput)) { map(canonicalOutput, canonicalInput); } } }
private void mapExchangeNodeOutputToInputSymbols(ExchangeNode node) { checkState(node.getInputs().size() == 1); for (int symbolIndex = 0; symbolIndex < node.getOutputSymbols().size(); symbolIndex++) { Symbol canonicalOutput = canonicalize(node.getOutputSymbols().get(symbolIndex)); Symbol canonicalInput = canonicalize(node.getInputs().get(0).get(symbolIndex)); if (!canonicalOutput.equals(canonicalInput)) { map(canonicalOutput, canonicalInput); } } }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(ExchangeNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getInputs().get(i), node.getOutputSymbols()); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndMaxDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } verify(estimate.isPresent()); return estimate; }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(ExchangeNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getInputs().get(i), node.getOutputSymbols()); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndMaxDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } verify(estimate.isPresent()); return estimate; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
private void mapExchangeNodeSymbols(ExchangeNode node) { if (node.getInputs().size() == 1) { mapExchangeNodeOutputToInputSymbols(node); return; } // Mapping from list [node.getInput(0).get(symbolIndex), node.getInput(1).get(symbolIndex), ...] to node.getOutputSymbols(symbolIndex). // All symbols are canonical. Map<List<Symbol>, Symbol> inputsToOutputs = new HashMap<>(); // Map each same list of input symbols [I1, I2, ..., In] to the same output symbol O for (int symbolIndex = 0; symbolIndex < node.getOutputSymbols().size(); symbolIndex++) { Symbol canonicalOutput = canonicalize(node.getOutputSymbols().get(symbolIndex)); List<Symbol> canonicalInputs = canonicalizeExchangeNodeInputs(node, symbolIndex); Symbol output = inputsToOutputs.get(canonicalInputs); if (output == null || canonicalOutput.equals(output)) { inputsToOutputs.put(canonicalInputs, canonicalOutput); } else { map(canonicalOutput, output); } } }
private void mapExchangeNodeSymbols(ExchangeNode node) { if (node.getInputs().size() == 1) { mapExchangeNodeOutputToInputSymbols(node); return; } // Mapping from list [node.getInput(0).get(symbolIndex), node.getInput(1).get(symbolIndex), ...] to node.getOutputSymbols(symbolIndex). // All symbols are canonical. Map<List<Symbol>, Symbol> inputsToOutputs = new HashMap<>(); // Map each same list of input symbols [I1, I2, ..., In] to the same output symbol O for (int symbolIndex = 0; symbolIndex < node.getOutputSymbols().size(); symbolIndex++) { Symbol canonicalOutput = canonicalize(node.getOutputSymbols().get(symbolIndex)); List<Symbol> canonicalInputs = canonicalizeExchangeNodeInputs(node, symbolIndex); Symbol output = inputsToOutputs.get(canonicalInputs); if (output == null || canonicalOutput.equals(output)) { inputsToOutputs.put(canonicalInputs, canonicalOutput); } else { map(canonicalOutput, output); } } }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
@Override public MatchResult visitExchange(ExchangeNode node, PlanMatchPattern pattern) { List<List<Symbol>> allInputs = node.getInputs(); List<Symbol> outputs = node.getOutputSymbols(); MatchResult result = super.visitExchange(node, pattern); if (!result.isMatch()) { return result; } SymbolAliases newAliases = result.getAliases(); for (List<Symbol> inputs : allInputs) { Assignments.Builder assignments = Assignments.builder(); for (int i = 0; i < inputs.size(); ++i) { assignments.put(outputs.get(i), inputs.get(i).toSymbolReference()); } newAliases = newAliases.updateAssignments(assignments.build()); } return match(newAliases); }
@Override public MatchResult visitExchange(ExchangeNode node, PlanMatchPattern pattern) { List<List<Symbol>> allInputs = node.getInputs(); List<Symbol> outputs = node.getOutputSymbols(); MatchResult result = super.visitExchange(node, pattern); if (!result.isMatch()) { return result; } SymbolAliases newAliases = result.getAliases(); for (List<Symbol> inputs : allInputs) { Assignments.Builder assignments = Assignments.builder(); for (int i = 0; i < inputs.size(); ++i) { assignments.put(outputs.get(i), inputs.get(i).toSymbolReference()); } newAliases = newAliases.updateAssignments(assignments.build()); } return match(newAliases); }