/** * possiblePlans should be provided in layout preference order */ private PlanWithProperties pickPlan(List<PlanWithProperties> possiblePlans, PreferredProperties preferredProperties) { checkArgument(!possiblePlans.isEmpty()); if (preferStreamingOperators) { possiblePlans = new ArrayList<>(possiblePlans); Collections.sort(possiblePlans, Comparator.comparing(PlanWithProperties::getProperties, streamingExecutionPreference(preferredProperties))); // stable sort; is Collections.min() guaranteed to be stable? } return possiblePlans.get(0); }
@VisibleForTesting static Comparator<ActualProperties> streamingExecutionPreference(PreferredProperties preferred) { // Calculating the matches can be a bit expensive, so cache the results between comparisons LoadingCache<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>> matchCache = CacheBuilder.newBuilder() .build(CacheLoader.from(actualProperties -> LocalProperties.match(actualProperties, preferred.getLocalProperties()))); return (actual1, actual2) -> { List<Optional<LocalProperty<Symbol>>> matchLayout1 = matchCache.getUnchecked(actual1.getLocalProperties()); List<Optional<LocalProperty<Symbol>>> matchLayout2 = matchCache.getUnchecked(actual2.getLocalProperties()); return ComparisonChain.start() .compareTrueFirst(hasLocalOptimization(preferred.getLocalProperties(), matchLayout1), hasLocalOptimization(preferred.getLocalProperties(), matchLayout2)) .compareTrueFirst(meetsPartitioningRequirements(preferred, actual1), meetsPartitioningRequirements(preferred, actual2)) .compare(matchLayout1, matchLayout2, matchedLayoutPreference()) .result(); }; }
estimatedExchangesCostCalculator, ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, sqlParser)));
public void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern) { List<PlanOptimizer> optimizers = ImmutableList.of( new UnaliasSymbolReferences(), new AddExchanges(getQueryRunner().getMetadata(), new SqlParser()), new PruneUnreferencedOutputs(), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getCostCalculator(), ImmutableSet.of(new RemoveRedundantIdentityProjections()))); assertPlan(sql, pattern, optimizers); } }
@Test public void testPickLayoutAnyPreference() Comparator<ActualProperties> preference = streamingExecutionPreference(PreferredProperties.any());
@VisibleForTesting static Comparator<ActualProperties> streamingExecutionPreference(PreferredProperties preferred) { // Calculating the matches can be a bit expensive, so cache the results between comparisons LoadingCache<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>> matchCache = CacheBuilder.newBuilder() .build(new CacheLoader<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>>() { @Override public List<Optional<LocalProperty<Symbol>>> load(List<LocalProperty<Symbol>> actualProperties) { return LocalProperties.match(actualProperties, preferred.getLocalProperties()); } }); return (actual1, actual2) -> { List<Optional<LocalProperty<Symbol>>> matchLayout1 = matchCache.getUnchecked(actual1.getLocalProperties()); List<Optional<LocalProperty<Symbol>>> matchLayout2 = matchCache.getUnchecked(actual2.getLocalProperties()); return ComparisonChain.start() .compareTrueFirst(hasLocalOptimization(preferred.getLocalProperties(), matchLayout1), hasLocalOptimization(preferred.getLocalProperties(), matchLayout2)) .compareTrueFirst(meetsPartitioningRequirements(preferred, actual1), meetsPartitioningRequirements(preferred, actual2)) .compare(matchLayout1, matchLayout2, matchedLayoutPreference()) .result(); }; }
builder.add(new AddExchanges(metadata, sqlParser)); builder.add(new AddIntermediateAggregation(metadata)); // Must run after AddExchanges
@Test public void testPickLayoutPartitionedPreference() Comparator<ActualProperties> preference = streamingExecutionPreference(PreferredProperties.distributed());
@Test public void testPickLayoutPartitionedOnSingle() Comparator<ActualProperties> preference = streamingExecutionPreference( PreferredProperties.partitioned(ImmutableSet.of(symbol("a"))));
@Test public void testPickLayoutUnpartitionedPreference() Comparator<ActualProperties> preference = streamingExecutionPreference(PreferredProperties.undistributed());
@Test public void testPickLayoutPartitionedWithGroup() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.partitionedWithLocal( ImmutableSet.of(symbol("a")),
@Test public void testPickLayoutGroupedMultiple() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.local(ImmutableList.of(grouped("a", "b"))));
@Test public void testPickLayoutGroupedWithSort() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.local(ImmutableList.of(grouped("a"), sorted("b", ASC_NULLS_FIRST))));
@Test public void testPickLayoutGrouped() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.local(ImmutableList.of(grouped("a"))));
@Test public void testPickLayoutPartitionedOnMultiple() Comparator<ActualProperties> preference = streamingExecutionPreference( PreferredProperties.partitioned(ImmutableSet.of(symbol("a"), symbol("b"))));
@Test public void testPickLayoutGroupedMultipleProperties() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.local(ImmutableList.of(grouped("a"), grouped("b"))));
@Test public void testPickLayoutUnpartitionedWithGroupAndSort() Comparator<ActualProperties> preference = streamingExecutionPreference (PreferredProperties.undistributedWithLocal(ImmutableList.of(grouped("a"), sorted("b", ASC_NULLS_FIRST))));
/** * possiblePlans should be provided in layout preference order */ private PlanWithProperties pickPlan(List<PlanWithProperties> possiblePlans, Context context) { checkArgument(!possiblePlans.isEmpty()); if (preferStreamingOperators) { possiblePlans = new ArrayList<>(possiblePlans); Collections.sort(possiblePlans, Comparator.comparing(PlanWithProperties::getProperties, streamingExecutionPreference(context.getPreferredProperties()))); // stable sort; is Collections.min() guaranteed to be stable? } return possiblePlans.get(0); }
throws Exception Comparator<ActualProperties> preference = streamingExecutionPreference(PreferredProperties.any());
throws Exception Comparator<ActualProperties> preference = streamingExecutionPreference( PreferredProperties.partitioned(ImmutableSet.of(symbol("a"))));