public HashComputationSet withHashComputation(Optional<HashComputation> hashComputation) { if (!hashComputation.isPresent()) { return this; } return new HashComputationSet(ImmutableSet.<HashComputation>builder() .addAll(hashes) .add(hashComputation.get()) .build()); } }
public HashComputationSet translate(Function<Symbol, Optional<Symbol>> translator) { Set<HashComputation> newHashes = hashes.stream() .map(hash -> hash.translate(translator)) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableSet()); return new HashComputationSet(newHashes); }
public HashComputationSet pruneSymbols(List<Symbol> symbols) { Set<Symbol> uniqueSymbols = ImmutableSet.copyOf(symbols); return new HashComputationSet(hashes.stream() .filter(hash -> hash.canComputeWith(uniqueSymbols)) .collect(toImmutableSet())); }
PlanWithProperties probe = planAndEnforce( node.getProbeSource(), new HashComputationSet(probeHashComputation), true, new HashComputationSet(probeHashComputation)); Symbol probeHashSymbol = probe.getRequiredHashSymbol(probeHashComputation.get()); HashComputationSet requiredHashes = new HashComputationSet(indexHashComputation); PlanWithProperties index = planAndEnforce(node.getIndexSource(), requiredHashes, true, requiredHashes); Symbol indexHashSymbol = index.getRequiredHashSymbol(indexHashComputation.get());
PlanWithProperties left = planAndEnforce(node.getLeft(), new HashComputationSet(), true, new HashComputationSet()); PlanWithProperties right = planAndEnforce(node.getRight(), new HashComputationSet(), true, new HashComputationSet()); checkState(left.getHashSymbols().isEmpty() && right.getHashSymbols().isEmpty()); return new PlanWithProperties( PlanWithProperties left = planAndEnforce(node.getLeft(), new HashComputationSet(leftHashComputation), true, new HashComputationSet(leftHashComputation)); Symbol leftHashSymbol = left.getRequiredHashSymbol(leftHashComputation.get()); PlanWithProperties right = planAndEnforce(node.getRight(), new HashComputationSet(rightHashComputation), true, new HashComputationSet(rightHashComputation)); Symbol rightHashSymbol = right.getRequiredHashSymbol(rightHashComputation.get());
@Override public PlanWithProperties visitSemiJoin(SemiJoinNode node, HashComputationSet parentPreference) { Optional<HashComputation> sourceHashComputation = computeHash(ImmutableList.of(node.getSourceJoinSymbol())); PlanWithProperties source = planAndEnforce( node.getSource(), new HashComputationSet(sourceHashComputation), true, new HashComputationSet(sourceHashComputation)); Symbol sourceHashSymbol = source.getRequiredHashSymbol(sourceHashComputation.get()); Optional<HashComputation> filterHashComputation = computeHash(ImmutableList.of(node.getFilteringSourceJoinSymbol())); HashComputationSet requiredHashes = new HashComputationSet(filterHashComputation); PlanWithProperties filteringSource = planAndEnforce(node.getFilteringSource(), requiredHashes, true, requiredHashes); Symbol filteringSourceHashSymbol = filteringSource.getRequiredHashSymbol(filterHashComputation.get()); return new PlanWithProperties( new SemiJoinNode( node.getId(), source.getNode(), filteringSource.getNode(), node.getSourceJoinSymbol(), node.getFilteringSourceJoinSymbol(), node.getSemiJoinOutput(), Optional.of(sourceHashSymbol), Optional.of(filteringSourceHashSymbol), node.getDistributionType()), source.getHashSymbols()); }
@Override public PlanWithProperties visitRowNumber(RowNumberNode node, HashComputationSet parentPreference) { if (node.getPartitionBy().isEmpty()) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getPartitionBy()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new RowNumberNode( node.getId(), child.getNode(), node.getPartitionBy(), node.getRowNumberSymbol(), node.getMaxRowCountPerPartition(), Optional.of(hashSymbol)), child.getHashSymbols()); }
@Override public PlanWithProperties visitWindow(WindowNode node, HashComputationSet parentPreference) { if (node.getPartitionBy().isEmpty()) { return planSimpleNodeWithProperties(node, parentPreference, true); } Optional<HashComputation> hashComputation = computeHash(node.getPartitionBy()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), true, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new WindowNode( node.getId(), child.getNode(), node.getSpecification(), node.getWindowFunctions(), Optional.of(hashSymbol), node.getPrePartitionedInputs(), node.getPreSortedOrderPrefix()), child.getHashSymbols()); }
@Override public PlanWithProperties visitAggregation(AggregationNode node, HashComputationSet parentPreference) { Optional<HashComputation> groupByHash = Optional.empty(); if (!node.isStreamable() && !canSkipHashGeneration(node.getGroupingKeys())) { groupByHash = computeHash(node.getGroupingKeys()); } // aggregation does not pass through preferred hash symbols HashComputationSet requiredHashes = new HashComputationSet(groupByHash); PlanWithProperties child = planAndEnforce(node.getSource(), requiredHashes, false, requiredHashes); Optional<Symbol> hashSymbol = groupByHash.map(child::getRequiredHashSymbol); return new PlanWithProperties( new AggregationNode( node.getId(), child.getNode(), node.getAggregations(), node.getGroupingSets(), node.getPreGroupedSymbols(), node.getStep(), hashSymbol, node.getGroupIdSymbol()), hashSymbol.isPresent() ? ImmutableMap.of(groupByHash.get(), hashSymbol.get()) : ImmutableMap.of()); }
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, HashComputationSet parentPreference) { // skip hash symbol generation for single bigint if (canSkipHashGeneration(node.getDistinctSymbols())) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); // TODO: we need to reason about how pre-computed hashes from child relate to distinct symbols. We should be able to include any precomputed hash // that's functionally dependent on the distinct field in the set of distinct fields of the new node to be able to propagate it downstream. // Currently, such precomputed hashes will be dropped by this operation. return new PlanWithProperties( new DistinctLimitNode(node.getId(), child.getNode(), node.getLimit(), node.isPartial(), node.getDistinctSymbols(), Optional.of(hashSymbol)), ImmutableMap.of(hashComputation.get(), hashSymbol)); }
@Override public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, HashComputationSet parentPreference) { if (node.getPartitionBy().isEmpty()) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getPartitionBy()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new TopNRowNumberNode( node.getId(), child.getNode(), node.getSpecification(), node.getRowNumberSymbol(), node.getMaxRowCountPerPartition(), node.isPartial(), Optional.of(hashSymbol)), child.getHashSymbols()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, HashComputationSet parentPreference) { // skip hash symbol generation for single bigint if (canSkipHashGeneration(node.getDistinctSymbols())) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new MarkDistinctNode(node.getId(), child.getNode(), node.getMarkerSymbol(), node.getDistinctSymbols(), Optional.of(hashSymbol)), child.getHashSymbols()); }
private PlanWithProperties planSimpleNodeWithProperties( PlanNode node, HashComputationSet preferredHashes, boolean alwaysPruneExtraHashSymbols) { if (node.getSources().isEmpty()) { return new PlanWithProperties(node, ImmutableMap.of()); } // There is not requirement to produce hash symbols and only preference for symbols PlanWithProperties source = planAndEnforce(Iterables.getOnlyElement(node.getSources()), new HashComputationSet(), alwaysPruneExtraHashSymbols, preferredHashes); PlanNode result = replaceChildren(node, ImmutableList.of(source.getNode())); // return only hash symbols that are passed through the new node Map<HashComputation, Symbol> hashSymbols = new HashMap<>(source.getHashSymbols()); hashSymbols.values().retainAll(result.getOutputSymbols()); return new PlanWithProperties(result, hashSymbols); }
@Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { requireNonNull(plan, "plan is null"); requireNonNull(session, "session is null"); requireNonNull(types, "types is null"); requireNonNull(symbolAllocator, "symbolAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); if (SystemSessionProperties.isOptimizeHashGenerationEnabled(session)) { PlanWithProperties result = plan.accept(new Rewriter(idAllocator, symbolAllocator, types), new HashComputationSet()); return result.getNode(); } return plan; }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, HashComputationSet parentPreference) { // this plan node can only have a single input symbol, so do not add extra hash symbols return planSimpleNodeWithProperties(node, new HashComputationSet(), true); }
@Override public PlanWithProperties visitSpatialJoin(SpatialJoinNode node, HashComputationSet parentPreference) { PlanWithProperties left = planAndEnforce(node.getLeft(), new HashComputationSet(), true, new HashComputationSet()); PlanWithProperties right = planAndEnforce(node.getRight(), new HashComputationSet(), true, new HashComputationSet()); verify(left.getHashSymbols().isEmpty(), "probe side of the spatial join should not include hash symbols"); verify(right.getHashSymbols().isEmpty(), "build side of the spatial join should not include hash symbols"); return new PlanWithProperties( replaceChildren(node, ImmutableList.of(left.getNode(), right.getNode())), ImmutableMap.of()); }