public static StreamPreferredProperties defaultParallelism(Session session) { if (getTaskConcurrency(session) > 1 && !preferStreamingOperators(session)) { return new StreamPreferredProperties(Optional.of(MULTIPLE), Optional.empty(), false); } return any(); }
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, StreamPreferredProperties parentPreferences) { // final limit requires that all data be in one stream StreamPreferredProperties requiredProperties; StreamPreferredProperties preferredProperties; if (node.isPartial()) { requiredProperties = parentPreferences.withoutPreference().withDefaultParallelism(session); preferredProperties = parentPreferences.withDefaultParallelism(session); } else { // a final changes the input organization completely, so we do not pass through parent preferences requiredProperties = singleStream(); preferredProperties = defaultParallelism(session); } return planAndEnforceChildren(node, requiredProperties, preferredProperties); }
@Override public PlanWithProperties visitTableWriter(TableWriterNode node, StreamPreferredProperties parentPreferences) { StreamPreferredProperties requiredProperties; StreamPreferredProperties preferredProperties; if (getTaskWriterCount(session) > 1) { requiredProperties = fixedParallelism(); preferredProperties = fixedParallelism(); } else { requiredProperties = singleStream(); preferredProperties = defaultParallelism(session); } return planAndEnforceChildren(node, requiredProperties, preferredProperties); }
public StreamPreferredProperties translate(Function<Symbol, Optional<Symbol>> translator) { return new StreamPreferredProperties( distribution, partitioningColumns.flatMap(partitioning -> translateSymbols(partitioning, translator)), orderSensitive); }
@Override public PlanWithProperties visitOutput(OutputNode node, StreamPreferredProperties parentPreferences) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); }
public StreamPreferredProperties withParallelism() { // do not override an existing parallel preference if (isParallelPreferred()) { return this; } return new StreamPreferredProperties(Optional.of(MULTIPLE), Optional.empty(), orderSensitive); }
@Override public PlanWithProperties visitJoin(JoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties probe; if (isSpillEnabled(session)) { probe = planAndEnforce( node.getLeft(), fixedParallelism(), parentPreferences.constrainTo(node.getLeft().getOutputSymbols()).withFixedParallelism()); } else { probe = planAndEnforce( node.getLeft(), defaultParallelism(session), parentPreferences.constrainTo(node.getLeft().getOutputSymbols()).withDefaultParallelism(session)); } // this build consumes the input completely, so we do not pass through parent preferences List<Symbol> buildHashSymbols = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight); StreamPreferredProperties buildPreference; if (getTaskConcurrency(session) > 1) { buildPreference = exactlyPartitionedOn(buildHashSymbols); } else { buildPreference = singleStream(); } PlanWithProperties build = planAndEnforce(node.getRight(), buildPreference, buildPreference); return rebaseAndDeriveProperties(node, ImmutableList.of(probe, build)); }
return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); PlanWithProperties child = planAndEnforce(node.getSource(), any(), defaultParallelism(session)); PlanWithProperties exchange = deriveProperties( partitionedExchange( .constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(groupingKeys);
@Override public PlanWithProperties visitSemiJoin(SemiJoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties source = planAndEnforce( node.getSource(), defaultParallelism(session), parentPreferences.constrainTo(node.getSource().getOutputSymbols()).withDefaultParallelism(session)); // this filter source consumes the input completely, so we do not pass through parent preferences PlanWithProperties filteringSource = planAndEnforce(node.getFilteringSource(), singleStream(), singleStream()); return rebaseAndDeriveProperties(node, ImmutableList.of(source, filteringSource)); }
private PlanWithProperties enforce(PlanWithProperties planWithProperties, StreamPreferredProperties requiredProperties) if (requiredProperties.isSatisfiedBy(planWithProperties.getProperties())) { return planWithProperties; if (requiredProperties.isSingleStreamPreferred()) { ExchangeNode exchangeNode = gatheringExchange(idAllocator.getNextId(), LOCAL, planWithProperties.getNode()); return deriveProperties(exchangeNode, planWithProperties.getProperties()); Optional<List<Symbol>> requiredPartitionColumns = requiredProperties.getPartitioningColumns(); if (!requiredPartitionColumns.isPresent()) { if (requiredProperties.isParallelPreferred()) {
.constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getPartitionBy());
@Override public PlanWithProperties visitExchange(ExchangeNode node, StreamPreferredProperties parentPreferences) { checkArgument(node.getScope() != LOCAL, "AddLocalExchanges can not process a plan containing a local exchange"); // this node changes the input organization completely, so we do not pass through parent preferences if (node.getOrderingScheme().isPresent()) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); } return planAndEnforceChildren(node, any(), defaultParallelism(session)); }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties probe = planAndEnforce( node.getProbeSource(), defaultParallelism(session), parentPreferences.constrainTo(node.getProbeSource().getOutputSymbols()).withDefaultParallelism(session)); // index source does not support local parallel and must produce a single stream StreamProperties indexStreamProperties = derivePropertiesRecursively(node.getIndexSource(), metadata, session, types, parser); checkArgument(indexStreamProperties.getDistribution() == SINGLE, "index source must be single stream"); PlanWithProperties index = new PlanWithProperties(node.getIndexSource(), indexStreamProperties); return rebaseAndDeriveProperties(node, ImmutableList.of(probe, index)); }
.map(source -> source.accept(this, defaultParallelism(session))) .collect(toImmutableList()); if (preferredProperties.isSingleStreamPreferred()) { ExchangeNode exchangeNode = new ExchangeNode( idAllocator.getNextId(), Optional<List<Symbol>> preferredPartitionColumns = preferredProperties.getPartitioningColumns(); if (preferredPartitionColumns.isPresent()) { ExchangeNode exchangeNode = new ExchangeNode(
public static StreamPreferredProperties exactlyPartitionedOn(Collection<Symbol> partitionSymbols) { if (partitionSymbols.isEmpty()) { return singleStream(); } // this must be the exact partitioning symbols, in the exact order return new StreamPreferredProperties(Optional.of(FIXED), true, Optional.of(ImmutableList.copyOf(partitionSymbols)), false); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, StreamPreferredProperties parentPreferences) { // table commit requires that all data be in one stream // this node changes the input organization completely, so we do not pass through parent preferences return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); }
private PlanWithProperties planAndEnforce(PlanNode node, StreamPreferredProperties requiredProperties, StreamPreferredProperties preferredProperties) { // verify properties are in terms of symbols produced by the node List<Symbol> outputSymbols = node.getOutputSymbols(); checkArgument(requiredProperties.getPartitioningColumns().map(outputSymbols::containsAll).orElse(true)); checkArgument(preferredProperties.getPartitioningColumns().map(outputSymbols::containsAll).orElse(true)); // plan the node using the preferred properties PlanWithProperties result = node.accept(this, preferredProperties); // enforce the required properties result = enforce(result, requiredProperties); checkState(requiredProperties.isSatisfiedBy(result.getProperties()), "required properties not enforced"); return result; }
public StreamPreferredProperties withOrderSensitivity() { return new StreamPreferredProperties(distribution, false, Optional.empty(), true); }
public StreamPreferredProperties withFixedParallelism() { if (distribution.isPresent() && distribution.get() == FIXED) { return this; } return fixedParallelism(); }
private PlanWithProperties planAndEnforceChildren(PlanNode node, StreamPreferredProperties requiredProperties, StreamPreferredProperties preferredProperties) { // plan and enforce each child, but strip any requirement not in terms of symbols produced from the child // Note: this assumes the child uses the same symbols as the parent List<PlanWithProperties> children = node.getSources().stream() .map(source -> planAndEnforce( source, requiredProperties.constrainTo(source.getOutputSymbols()), preferredProperties.constrainTo(source.getOutputSymbols()))) .collect(toImmutableList()); return rebaseAndDeriveProperties(node, children); }