private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan) { PlanFragment fragment = subPlan.getFragment(); GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null); if (properties.isSubTreeUseful()) { fragment = fragment.withGroupedExecution(properties.getCapableTableScanNodes()); } ImmutableList.Builder<SubPlan> result = ImmutableList.builder(); for (SubPlan child : subPlan.getChildren()) { result.add(analyzeGroupedExecution(session, child)); } return new SubPlan(fragment, result.build()); }
/** * Flattens the subplan and returns all PlanFragments in the tree */ public List<PlanFragment> getAllFragments() { ImmutableList.Builder<PlanFragment> fragments = ImmutableList.builder(); fragments.add(getFragment()); for (SubPlan child : getChildren()) { fragments.addAll(child.getAllFragments()); } return fragments.build(); }
private void sanityCheckFragmentedPlan(SubPlan subPlan, int maxStageCount) { subPlan.sanityCheck(); int fragmentCount = subPlan.getAllFragments().size(); if (fragmentCount > maxStageCount) { throw new PrestoException(QUERY_HAS_TOO_MANY_STAGES, format( "Number of stages in the query (%s) exceeds the allowed maximum (%s). " + "If the query contains multiple DISTINCTs, please set the use_mark_distinct session property to false. " + "If the query contains multiple CTEs that are referenced more than once, please create temporary table(s) for one or more of the CTEs.", fragmentCount, maxStageCount)); } }
private StageExecutionPlan doPlan(SubPlan root, Session session, ImmutableList.Builder<SplitSource> allSplitSources) { PlanFragment currentFragment = root.getFragment(); // get splits for this fragment, this is lazy so split assignments aren't actually calculated here Map<PlanNodeId, SplitSource> splitSources = currentFragment.getRoot().accept(new Visitor(session, currentFragment.getStageExecutionStrategy(), allSplitSources), null); // create child stages ImmutableList.Builder<StageExecutionPlan> dependencies = ImmutableList.builder(); for (SubPlan childPlan : root.getChildren()) { dependencies.add(doPlan(childPlan, session, allSplitSources)); } return new StageExecutionPlan( currentFragment, splitSources, dependencies.build()); }
private SubPlan reassignPartitioningHandleIfNecessary(Session session, SubPlan subPlan) { return reassignPartitioningHandleIfNecessaryHelper(session, subPlan, subPlan.getFragment().getPartitioning()); }
public static String textDistributedPlan(SubPlan plan, FunctionRegistry functionRegistry, Session session, boolean verbose) { StringBuilder builder = new StringBuilder(); for (PlanFragment fragment : plan.getAllFragments()) { builder.append(formatFragment(functionRegistry, session, fragment, Optional.empty(), Optional.empty(), verbose, plan.getAllFragments())); } return builder.toString(); }
public void sanityCheck() { Multiset<PlanFragmentId> exchangeIds = fragment.getRemoteSourceNodes().stream() .map(RemoteSourceNode::getSourceFragmentIds) .flatMap(List::stream) .collect(toImmutableMultiset()); Multiset<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableMultiset()); Preconditions.checkState(exchangeIds.equals(childrenIds), "Subplan exchange ids don't match child fragment ids (%s vs %s)", exchangeIds, childrenIds); for (SubPlan child : children) { child.sanityCheck(); } } }
private SubPlan buildFragment(PlanNode root, FragmentProperties properties, PlanFragmentId fragmentId) { Set<Symbol> dependencies = SymbolsExtractor.extractOutputSymbols(root); List<PlanNodeId> schedulingOrder = scheduleOrder(root); boolean equals = properties.getPartitionedSources().equals(ImmutableSet.copyOf(schedulingOrder)); checkArgument(equals, "Expected scheduling order (%s) to contain an entry for all partitioned sources (%s)", schedulingOrder, properties.getPartitionedSources()); PlanFragment fragment = new PlanFragment( fragmentId, root, Maps.filterKeys(types.allTypes(), in(dependencies)), properties.getPartitioningHandle(), schedulingOrder, properties.getPartitioningScheme(), StageExecutionStrategy.ungroupedExecution(), statsAndCosts.getForSubplan(root)); return new SubPlan(fragment, properties.getChildren()); }
private StageExecutionPlan doPlan(SubPlan root, Session session, ImmutableList.Builder<SplitSource> allSplitSources) { PlanFragment currentFragment = root.getFragment(); // get splits for this fragment, this is lazy so split assignments aren't actually calculated here Map<PlanNodeId, SplitSource> splitSources = currentFragment.getRoot().accept(new Visitor(session, currentFragment.getStageExecutionDescriptor(), allSplitSources), null); // create child stages ImmutableList.Builder<StageExecutionPlan> dependencies = ImmutableList.builder(); for (SubPlan childPlan : root.getChildren()) { dependencies.add(doPlan(childPlan, session, allSplitSources)); } return new StageExecutionPlan( currentFragment, splitSources, dependencies.build()); }
private SubPlan reassignPartitioningHandleIfNecessary(Session session, SubPlan subPlan) { return reassignPartitioningHandleIfNecessaryHelper(session, subPlan, subPlan.getFragment().getPartitioning()); }
public static String textDistributedPlan(SubPlan plan, FunctionRegistry functionRegistry, Session session, boolean verbose) { StringBuilder builder = new StringBuilder(); for (PlanFragment fragment : plan.getAllFragments()) { builder.append(formatFragment(functionRegistry, session, fragment, Optional.empty(), Optional.empty(), verbose, plan.getAllFragments())); } return builder.toString(); }
public void sanityCheck() { Multiset<PlanFragmentId> exchangeIds = fragment.getRemoteSourceNodes().stream() .map(RemoteSourceNode::getSourceFragmentIds) .flatMap(List::stream) .collect(toImmutableMultiset()); Multiset<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableMultiset()); Preconditions.checkState(exchangeIds.equals(childrenIds), "Subplan exchange ids don't match child fragment ids (%s vs %s)", exchangeIds, childrenIds); for (SubPlan child : children) { child.sanityCheck(); } } }
private SubPlan buildFragment(PlanNode root, FragmentProperties properties, PlanFragmentId fragmentId) { Set<Symbol> dependencies = SymbolsExtractor.extractOutputSymbols(root); List<PlanNodeId> schedulingOrder = scheduleOrder(root); boolean equals = properties.getPartitionedSources().equals(ImmutableSet.copyOf(schedulingOrder)); checkArgument(equals, "Expected scheduling order (%s) to contain an entry for all partitioned sources (%s)", schedulingOrder, properties.getPartitionedSources()); PlanFragment fragment = new PlanFragment( fragmentId, root, Maps.filterKeys(types.allTypes(), in(dependencies)), properties.getPartitioningHandle(), schedulingOrder, properties.getPartitioningScheme(), ungroupedExecution(), statsAndCosts.getForSubplan(root)); return new SubPlan(fragment, properties.getChildren()); }
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan) { PlanFragment fragment = subPlan.getFragment(); GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null); if (properties.isSubTreeUseful()) { fragment = fragment.withGroupedExecution(properties.getCapableTableScanNodes()); } ImmutableList.Builder<SubPlan> result = ImmutableList.builder(); for (SubPlan child : subPlan.getChildren()) { result.add(analyzeGroupedExecution(session, child)); } return new SubPlan(fragment, result.build()); }
private static void printSubPlan(SubPlan plan, Map<PlanFragmentId, PlanFragment> fragmentsById, PlanNodeIdGenerator idGenerator, StringBuilder output) { PlanFragment fragment = plan.getFragment(); printFragmentNodes(output, fragment, idGenerator); fragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null); for (SubPlan child : plan.getChildren()) { printSubPlan(child, fragmentsById, idGenerator, output); } }
private PlanNodeCostEstimate calculateCumulativeCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); }
/** * Flattens the subplan and returns all PlanFragments in the tree */ public List<PlanFragment> getAllFragments() { ImmutableList.Builder<PlanFragment> fragments = ImmutableList.builder(); fragments.add(getFragment()); for (SubPlan child : getChildren()) { fragments.addAll(child.getAllFragments()); } return fragments.build(); }
private void sanityCheckFragmentedPlan(SubPlan subPlan, int maxStageCount) { subPlan.sanityCheck(); int fragmentCount = subPlan.getAllFragments().size(); if (fragmentCount > maxStageCount) { throw new PrestoException(QUERY_HAS_TOO_MANY_STAGES, format( "Number of stages in the query (%s) exceeds the allowed maximum (%s). " + "If the query contains multiple DISTINCTs, please set the use_mark_distinct session property to false. " + "If the query contains multiple CTEs that are referenced more than once, please create temporary table(s) for one or more of the CTEs.", fragmentCount, maxStageCount)); } }
public static String printDistributed(SubPlan plan) { List<PlanFragment> fragments = plan.getAllFragments(); Map<PlanFragmentId, PlanFragment> fragmentsById = Maps.uniqueIndex(fragments, PlanFragment::getId); PlanNodeIdGenerator idGenerator = new PlanNodeIdGenerator(); StringBuilder output = new StringBuilder(); output.append("digraph distributed_plan {\n"); printSubPlan(plan, fragmentsById, idGenerator, output); output.append("}\n"); return output.toString(); }
private SubPlan reassignPartitioningHandleIfNecessaryHelper(Session session, SubPlan subPlan, PartitioningHandle newOutputPartitioningHandle) PlanFragment fragment = subPlan.getFragment(); for (SubPlan child : subPlan.getChildren()) { childrenBuilder.add(reassignPartitioningHandleIfNecessaryHelper(session, child, fragment.getPartitioning())); return new SubPlan(newFragment, childrenBuilder.build());