public PlanWithProperties(PlanNode node) { this(node, ActualProperties.builder().build()); }
public ActualProperties build() { List<LocalProperty<Symbol>> localProperties = this.localProperties; if (unordered) { localProperties = filteredCopy(this.localProperties, property -> !property.isOrderSensitive()); } return new ActualProperties(global, localProperties, constants); } }
probeProperties = probeProperties.translate(column -> filterOrRewrite(node.getOutputSymbols(), node.getCriteria(), column)); buildProperties = buildProperties.translate(column -> filterOrRewrite(node.getOutputSymbols(), node.getCriteria(), column)); constants.putAll(probeProperties.getConstants()); constants.putAll(buildProperties.getConstants()); return ActualProperties.builder() .global(probeProperties) .local(ImmutableList.of()) return ActualProperties.builderFrom(probeProperties) .constants(constants) .unordered(unordered) .build(); case LEFT: return ActualProperties.builderFrom(probeProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column))) .unordered(unordered) .build(); case RIGHT: buildProperties = buildProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column)); return ActualProperties.builderFrom(buildProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column))) .local(ImmutableList.of()) .unordered(unordered) .build(); case FULL: if (probeProperties.getNodePartitioning().isPresent()) { Partitioning nodePartitioning = probeProperties.getNodePartitioning().get(); ImmutableList.Builder<Expression> coalesceExpressions = ImmutableList.builder();
private boolean shouldRepartitionForIndexJoin(List<Symbol> joinColumns, PreferredProperties parentPreferredProperties, ActualProperties probeProperties) { // See if distributed index joins are enabled if (!distributedIndexJoins) { return false; } // No point in repartitioning if the plan is not distributed if (probeProperties.isSingleNode()) { return false; } Optional<PreferredProperties.PartitioningProperties> parentPartitioningPreferences = parentPreferredProperties.getGlobalProperties() .flatMap(PreferredProperties.Global::getPartitioningProperties); // Disable repartitioning if it would disrupt a parent's partitioning preference when streaming is enabled boolean parentAlreadyPartitionedOnChild = parentPartitioningPreferences .map(partitioning -> probeProperties.isStreamPartitionedOn(partitioning.getPartitioningColumns())) .orElse(false); if (preferStreamingOperators && parentAlreadyPartitionedOnChild) { return false; } // Otherwise, repartition if we need to align with the join columns if (!probeProperties.isStreamPartitionedOn(joinColumns)) { return true; } // If we are already partitioned on the join columns because the data has been forced effectively into one stream, // then we should repartition if that would make a difference (from the single stream state). return probeProperties.isEffectivelySingleStream() && probeProperties.isStreamRepartitionEffective(joinColumns); }
@Override public ActualProperties visitIndexJoin(IndexJoinNode node, List<ActualProperties> inputProperties) { // TODO: include all equivalent columns in partitioning properties ActualProperties probeProperties = inputProperties.get(0); ActualProperties indexProperties = inputProperties.get(1); switch (node.getType()) { case INNER: return ActualProperties.builderFrom(probeProperties) .constants(ImmutableMap.<Symbol, NullableValue>builder() .putAll(probeProperties.getConstants()) .putAll(indexProperties.getConstants()) .build()) .build(); case SOURCE_OUTER: return ActualProperties.builderFrom(probeProperties) .constants(probeProperties.getConstants()) .build(); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); } }
@Override public ActualProperties visitTableWriter(TableWriterNode node, List<ActualProperties> inputProperties) { ActualProperties properties = Iterables.getOnlyElement(inputProperties); if (properties.isCoordinatorOnly()) { return ActualProperties.builder() .global(coordinatorSingleStreamPartition()) .build(); } return ActualProperties.builder() .global(properties.isSingleNode() ? singleStreamPartition() : arbitraryPartition()) .build(); }
if (left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { Partitioning rightPartitioning = left.getProperties().translate(createTranslator(leftToRight)).getNodePartitioning().get(); right = node.getRight().accept(this, PreferredProperties.partitioned(rightPartitioning)); if (!right.getProperties().isCompatibleTablePartitioningWith(left.getProperties(), rightToLeft::get, metadata, session)) { right = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, right.getNode(), new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())), right = node.getRight().accept(this, PreferredProperties.partitioned(ImmutableSet.copyOf(rightSymbols))); if (right.getProperties().isNodePartitionedOn(rightSymbols) && !right.getProperties().isSingleNode()) { Partitioning leftPartitioning = right.getProperties().translate(createTranslator(rightToLeft)).getNodePartitioning().get(); left = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, left.getNode(), new PartitioningScheme(leftPartitioning, left.getNode().getOutputSymbols())), verify(left.getProperties().isCompatibleTablePartitioningWith(right.getProperties(), leftToRight::get, metadata, session)); Partitioning rightPartitioning = left.getProperties().translate(createTranslator(leftToRight)).getNodePartitioning().get(); right = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, right.getNode(), new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())),
for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { Map<Symbol, Symbol> inputToOutput = exchangeInputToOutput(node, sourceIndex); ActualProperties translated = inputProperties.get(sourceIndex).translate(symbol -> Optional.ofNullable(inputToOutput.get(symbol))); entries = (entries == null) ? translated.getConstants().entrySet() : Sets.intersection(entries, translated.getConstants().entrySet()); ActualProperties.Builder builder = ActualProperties.builder(); builder.local(localProperties.build()); builder.constants(constants); case GATHER: boolean coordinatorOnly = node.getPartitioningScheme().getPartitioning().getHandle().isCoordinatorOnly(); return ActualProperties.builder() .global(coordinatorOnly ? coordinatorSingleStreamPartition() : singleStreamPartition()) .local(localProperties.build()) .build(); case REPARTITION: return ActualProperties.builder() .global(partitionedOn( node.getPartitioningScheme().getPartitioning(), case REPLICATE: return ActualProperties.builder() .global(arbitraryPartition()) .constants(constants)
@Override public ActualProperties visitSpatialJoin(SpatialJoinNode node, List<ActualProperties> inputProperties) { ActualProperties probeProperties = inputProperties.get(0); ActualProperties buildProperties = inputProperties.get(1); switch (node.getType()) { case INNER: probeProperties = probeProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column)); buildProperties = buildProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column)); Map<Symbol, NullableValue> constants = new HashMap<>(); constants.putAll(probeProperties.getConstants()); constants.putAll(buildProperties.getConstants()); return ActualProperties.builderFrom(probeProperties) .constants(constants) .build(); case LEFT: return ActualProperties.builderFrom(probeProperties.translate(column -> filterIfMissing(node.getOutputSymbols(), column))) .build(); default: throw new IllegalArgumentException("Unsupported spatial join type: " + node.getType()); } }
return ActualProperties.builderFrom(probeProperties) .constants(ImmutableMap.<Symbol, NullableValue>builder() .putAll(probeProperties.getConstants()) .putAll(buildProperties.getConstants()) .build()) .build(); case LEFT: return ActualProperties.builderFrom(probeProperties) .constants(probeProperties.getConstants()) .build(); case RIGHT: return ActualProperties.builderFrom(buildProperties) .constants(buildProperties.getConstants()) .local(ImmutableList.of()) .build(); return ActualProperties.builder() .global(probeProperties.isSingleNode() ? singleStreamPartition() : arbitraryPartition()) .build(); default:
if (!source.getProperties().isCompatibleTablePartitioningWith(childPartitioning, nullsAndAnyReplicated, metadata, session)) { source = withDerivedProperties( partitionedExchange( ActualProperties.builder() .global(partitionedOn(desiredParentPartitioning, Optional.of(desiredParentPartitioning))) .build() .withReplicatedNulls(parentPartitioningPreference.isNullsAndAnyReplicated())); PlanWithProperties child = node.getSources().get(i).accept(this, PreferredProperties.any()); plannedChildren.add(child); if (child.getProperties().isSingleNode()) { unpartitionedChildren.add(child.getNode()); unpartitionedOutputLayouts.add(node.sourceOutputLayout(i)); ActualProperties.builder() .global(singleStreamPartition()) .build());
right = node.getRight().accept(this, context.withPreferredProperties(PreferredProperties.hashPartitioned(rightSymbols))); if (!left.getProperties().isNodePartitionedOn(leftSymbols) || (distributedJoins && left.getProperties().isSingleNode())) { left = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), left.getNode(), leftSymbols, node.getLeftHashSymbol()), if (left.getProperties().isSingleNode()) { if (!right.getProperties().isSingleNode()) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), right.getNode()), if (!left.getProperties().isNodePartitionedWith(right.getProperties(), leftToRight::get)) { Function<Symbol, Optional<Symbol>> leftToRightTranslator = leftSymbol -> leftToRight.get(leftSymbol).stream().findAny(); Optional<List<PartitionFunctionArgumentBinding>> rightPartitionColumns = left.getProperties().translate(leftToRightTranslator).getNodePartitioningColumns(); left.getProperties().getNodePartitioningHandle().get(), node.getRight().getOutputSymbols(), rightPartitionColumns.get(), right = node.getRight().accept(this, context.withPreferredProperties(PreferredProperties.any())); if (left.getProperties().isSingleNode() && !right.getProperties().isSingleNode()) { else if (!left.getProperties().isSingleNode() && (!left.getProperties().isNodePartitionedOn(FIXED_HASH_DISTRIBUTION, leftSymbols) || !right.getProperties().isNodePartitionedOn(FIXED_HASH_DISTRIBUTION, rightSymbols))) { right = withDerivedProperties( replicatedExchange(idAllocator.getNextId(), right.getNode()),
private static boolean meetsPartitioningRequirements(PreferredProperties preferred, ActualProperties actual) { if (!preferred.getGlobalProperties().isPresent()) { return true; } PreferredProperties.Global preferredGlobal = preferred.getGlobalProperties().get(); if (!preferredGlobal.isDistributed()) { return actual.isSingleNode(); } if (!preferredGlobal.getPartitioningProperties().isPresent()) { return !actual.isSingleNode(); } return actual.isStreamPartitionedOn(preferredGlobal.getPartitioningProperties().get().getPartitioningColumns()); }
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
for (int i = 0; i < sources.size(); i++) { PlanWithProperties child = sources.get(i).accept(this, context.withPreferredProperties(PreferredProperties.any())); if (child.getProperties().isSingleNode()) { unpartitionedChildren.add(child.getNode()); unpartitionedOutputLayouts.add(node.sourceOutputLayout(i)); ActualProperties.builder() .global(singleStreamPartition()) .build()); if (!source.getProperties().isNodePartitionedOn(FIXED_HASH_DISTRIBUTION, sourceHashColumns)) { source = withDerivedProperties( partitionedExchange( ActualProperties.builder() .global(partitionedOn(FIXED_HASH_DISTRIBUTION, hashArguments, Optional.of(hashArguments))) .build());
checkArgument(forceSingleNode || globalProperties.isSingleNode(), "Final aggregation with default value not separated from partial aggregation by remote hash exchange");
public ActualProperties translate(Function<Symbol, Optional<Symbol>> translator) { return builder() .global(global.translate(new Partitioning.Translator(translator, symbol -> Optional.ofNullable(constants.get(symbol)), expression -> Optional.empty()))) .local(LocalProperties.translate(localProperties, translator)) .constants(translateConstants(translator)) .build(); }
public static ActualProperties deriveProperties(PlanNode node, List<ActualProperties> inputProperties, Metadata metadata, Session session, TypeProvider types, SqlParser parser) { ActualProperties output = node.accept(new Visitor(metadata, session, types, parser), inputProperties); output.getNodePartitioning().ifPresent(partitioning -> verify(node.getOutputSymbols().containsAll(partitioning.getColumns()), "Node-level partitioning properties contain columns not present in node's output")); verify(node.getOutputSymbols().containsAll(output.getConstants().keySet()), "Node-level constant properties contain columns not present in node's output"); Set<Symbol> localPropertyColumns = output.getLocalProperties().stream() .flatMap(property -> property.getColumns().stream()) .collect(Collectors.toSet()); verify(node.getOutputSymbols().containsAll(localPropertyColumns), "Node-level local properties contain columns not present in node's output"); return output; }
@Override public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List<Symbol> leftSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getLeft) .collect(toImmutableList()); List<Symbol> rightSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableList()); JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); if (distributionType == JoinNode.DistributionType.REPLICATED) { PlanWithProperties left = node.getLeft().accept(this, PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) if (!node.getCriteria().isEmpty() && left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { return planPartitionedJoin(node, leftSymbols, rightSymbols, left); } return planReplicatedJoin(node, left); } else { return planPartitionedJoin(node, leftSymbols, rightSymbols); } }
@Override public ActualProperties visitAssignUniqueId(AssignUniqueId node, List<ActualProperties> inputProperties) { ActualProperties properties = Iterables.getOnlyElement(inputProperties); ImmutableList.Builder<LocalProperty<Symbol>> newLocalProperties = ImmutableList.builder(); newLocalProperties.addAll(properties.getLocalProperties()); newLocalProperties.add(new GroupingProperty<>(ImmutableList.of(node.getIdColumn()))); node.getSource().getOutputSymbols().stream() .forEach(column -> newLocalProperties.add(new ConstantProperty<>(column))); if (properties.getNodePartitioning().isPresent()) { // preserve input (possibly preferred) partitioning return ActualProperties.builderFrom(properties) .local(newLocalProperties.build()) .build(); } return ActualProperties.builderFrom(properties) .global(partitionedOn(ARBITRARY_DISTRIBUTION, ImmutableList.of(node.getIdColumn()), Optional.empty())) .local(newLocalProperties.build()) .build(); }