if (rightRowCount < PrelUtil.getSettings(join.getCluster()).getBroadcastThreshold() && ! left.getTraitSet().getTrait(DistributionTraitDef.INSTANCE).equals(DistributionTrait.SINGLETON) && (join.getJoinType() == JoinRelType.INNER || join.getJoinType() == JoinRelType.LEFT)) { if (rightRowCount <= plannerSettings.getOptions().getOption(PlannerSettings.BROADCAST_MIN_THRESHOLD)) { logger.debug("Enable broadcast plan? true (rightRowCount %d smaller than minimum broadcast threshold)", rightRowCount); return true; final double broadcastFactor = plannerSettings.getBroadcastFactor(); final double leftRowCount = mq.getRowCount(left); final int numEndPoints = plannerSettings.numEndPoints(); final long maxWidthPerNode = plannerSettings.getNumCoresPerExecutor(); final long maxWidthPerQuery = plannerSettings.getOptions().getOption(ExecConstants.MAX_WIDTH_GLOBAL); final long sliceTarget = plannerSettings.getSliceTarget(); final double minFactor = Doubles.min(leftRowCount * 1.0 / sliceTarget, numEndPoints * maxWidthPerNode, maxWidthPerQuery); final boolean enableBroadCast = (minFactor * broadcastFactor < leftRowCount);
@Before public void setUp() { optionManager = mock(OptionManager.class); when(optionManager.getOption("planner.timeout_per_phase_ms")).thenReturn(OptionValue.createLong(OptionType.QUERY, "planner.timeout_per_phase_ms", 100)); settings = new PlannerSettings(DremioTest.DEFAULT_SABOT_CONFIG, optionManager, null); }
protected double getFilterReduction(){ if(filter != null){ double selectivity = 0.15d; double max = PrelUtil.getPlannerSettings(getCluster()).getFilterMaxSelectivityEstimateFactor(); double min = PrelUtil.getPlannerSettings(getCluster()).getFilterMinSelectivityEstimateFactor(); if(selectivity < min) { selectivity = min; } if(selectivity > max) { selectivity = max; } return selectivity; }else { return 1d; } }
private static RelTraitSet traits( RelOptCluster cluster, double rowCount, int splitCount, RelTraitSet traitSet) { PlannerSettings settings = PrelUtil.getPlannerSettings(cluster.getPlanner()); boolean smallInput = rowCount < (double)settings.getSliceTarget(); DistributionTrait distribution; if(settings.isMultiPhaseAggEnabled() && !settings.isSingleMode() && !smallInput && splitCount > 1) { distribution = DistributionTrait.ANY; } else { distribution = DistributionTrait.SINGLETON; } return traitSet.plus(distribution); }
protected static boolean isSingleton(RelOptRuleCall call) { PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); if (settings.isSingleMode()) { return true; } RelNode child = call.rel(0).getInputs().get(0); // if small input, then singleton return call.getMetadataQuery().getRowCount(child) < settings.getSliceTarget(); }
@Override public RuleSet getRules(OptimizerRulesContext context) { final ImmutableList.Builder<RelOptRule> rules = ImmutableList.builder(); rules.add(AggregateReduceFunctionsRule.NO_REDUCE_SUM); if (context.getPlannerSettings() .getOptions() .getOption(PlannerSettings.JDBC_PUSH_DOWN_PLUS)) { rules.add( SimpleFilterJoinRule.CALCITE_INSTANCE, JOIN_CONDITION_PUSH_CALCITE_RULE, PushFilterPastProjectRule.CALCITE_INSTANCE ); } return RuleSets.ofList(rules.build()); } },
/** * Cost of doing Top-N is proportional to M log N where M is the total number of * input rows and N is the limit for Top-N. This makes Top-N preferable to Sort * since cost of full Sort is proportional to M log M . */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice. return super.computeSelfCost(planner).multiplyBy(0.05); } RelNode child = this.getInput(); double inputRows = relMetadataQuery.getRowCount(child); int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DremioCost.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(limit)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice. return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = relMetadataQuery.getRowCount(child); // int rowWidth = child.getRowType().getPrecision(); int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DremioCost.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(inputRows)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints // TODO: use rowWidth instead of avgFieldWidth * numFields // avgFieldWidth * numFields * inputRows double numFields = this.getRowType().getFieldCount(); long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).getNumVal(); double memCost = fieldWidth * numFields * inputRows; Factory costFactory = (Factory) planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0, memCost); }
@Override public Set<RelOptRule> getRules(OptimizerRulesContext optimizerContext, PlannerPhase phase, SourceType pluginType) { switch(phase){ case LOGICAL: ImmutableSet.Builder<RelOptRule> builder = ImmutableSet.builder(); builder.add(new HiveScanDrule(pluginType)); builder.add(EliminateEmptyScans.INSTANCE); final PlannerSettings plannerSettings = optimizerContext.getPlannerSettings(); if(plannerSettings.isPartitionPruningEnabled()){ builder.add(new PruneScanRuleFilterOnProject<>(pluginType, HiveScanDrel.class, optimizerContext)); builder.add(new PruneScanRuleFilterOnScan<>(pluginType, HiveScanDrel.class, optimizerContext)); } final OptionManager options = plannerSettings.getOptions(); if (options.getOption(HivePluginOptions.HIVE_ORC_READER_VECTORIZE) && options.getOption(HivePluginOptions.ENABLE_FILTER_PUSHDOWN_HIVE_ORC)) { builder.add(new ORCFilterPushDownRule(pluginType)); } return builder.build(); case PHYSICAL: return ImmutableSet.<RelOptRule>of( new HiveScanPrule(pluginType) ); default: return ImmutableSet.<RelOptRule>of(); } }
if (settings.isUnionAllDistributeEnabled()) { final PlannerSettings plannerSettings = PrelUtil.getPlannerSettings(call.getPlanner()); final RelTraitSet traitsChild; if (plannerSettings.getOptions().getOption(PlannerSettings.ENABLE_UNIONALL_ROUND_ROBIN)) { traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.PHYSICAL).plus(DistributionTrait.ROUND_ROBIN); } else {
this.plannerSettings = new PlannerSettings(sabotContext.getConfig(), queryOptions, sabotContext.getClusterResourceInformation()); this.plannerSettings.setNumEndPoints(sabotContext.getExecutors().size()); this.table = new OperatorTable(sabotContext.getFunctionImplementationRegistry()); PlannerSettings.getInitialPlanningMemorySize(), plannerSettings.getPlanningMemoryLimit()); this.bufferManager = new BufferManagerImpl(allocator);
if (context.getPlannerSettings().isHashJoinSwapEnabled()) { phyRelNode = SwapHashJoinVisitor.swapHashJoin(phyRelNode, context.getPlannerSettings() .getHashJoinSwapMarginFactor()); long targetSliceSize = config.getContext().getPlannerSettings().getSliceTarget(); phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize); if (context.getPlannerSettings().isGlobalDictionariesEnabled()) { phyRelNode = GlobalDictionaryVisitor.useGlobalDictionaries(phyRelNode);
/** * A SingleMergeExchange processes a total of M rows coming from N * sorted input streams (from N senders) and merges them into a single * output sorted stream. For costing purposes we can assume each sender * is sending M/N rows to a single receiver. * (See DremioCost for symbol notations) * C = CPU cost of SV remover for M/N rows * + Network cost of sending M/N rows to 1 destination. * So, C = (s * M/N) + (w * M/N) * Cost of merging M rows coming from N senders = (M log2 N) * c * Total cost = N * C + (M log2 N) * c */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = mq.getRowCount(child); int rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH; double svrCpuCost = DremioCost.SVR_CPU_COST * inputRows; double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth; int numEndPoints = PrelUtil.getSettings(getCluster()).numEndPoints(); double mergeCpuCost = DremioCost.COMPARE_CPU_COST * inputRows * (Math.log(numEndPoints)/Math.log(2)); Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, svrCpuCost + mergeCpuCost, 0, networkCost); }
public static RelTraitSet fixTraits(RelOptPlanner cluster, RelTraitSet set) { if (getPlannerSettings(cluster).isSingleMode()) { return set.replace(DistributionTrait.ANY); } else { return set; } }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double leftRowCount = mq.getRowCount(this.getLeft()); double rightRowCount = mq.getRowCount(this.getRight()); double nljFactor = PrelUtil.getSettings(getCluster()).getNestedLoopJoinFactor(); // cpu cost of evaluating each leftkey=rightkey join condition double joinConditionCost = DremioCost.COMPARE_CPU_COST * this.getLeftKeys().size(); double cpuCost = joinConditionCost * (leftRowCount * rightRowCount) * nljFactor; Factory costFactory = (Factory) planner.getCostFactory(); return costFactory.makeCost(leftRowCount * rightRowCount, cpuCost, 0, 0, 0); }
public boolean isTrivial(){ Integer last = getLast(); if(last != null && last <= PrelUtil.getPlannerSettings(getCluster()).getSliceTarget() ){ return true; } else { return false; } }
@Override public Set<RelOptRule> getRules(OptimizerRulesContext optimizerContext, PlannerPhase phase, SourceType pluginType) { switch(phase){ case LOGICAL: ImmutableSet.Builder<RelOptRule> builder = ImmutableSet.builder(); builder.add(new FileSystemDrule(pluginType)); if(optimizerContext.getPlannerSettings().isPartitionPruningEnabled()){ builder.add(new PruneScanRuleFilterOnProject<>(pluginType, FilesystemScanDrel.class, optimizerContext)); builder.add(new PruneScanRuleFilterOnScan<>(pluginType, FilesystemScanDrel.class, optimizerContext)); } return builder.build(); case PHYSICAL: return ImmutableSet.<RelOptRule>of( new EasyFilesystemScanPrule(pluginType), new ParquetFilesystemScanPrule(pluginType), ConvertCountToDirectScan.getAggOnScan(pluginType), ConvertCountToDirectScan.getAggProjOnScan(pluginType) ); default: return ImmutableSet.<RelOptRule>of(); } }
protected static boolean create2PhasePlan(RelOptRuleCall call, AggregateRel aggregate) { PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); if (! settings.isMultiPhaseAggEnabled() || isSingleton(call)) { return false; } for (AggregateCall aggCall : aggregate.getAggCallList()) { String name = aggCall.getAggregation().getName(); if (!twoPhaseFunctions.contains(name)) { return false; } } return true; }
@Override public Set<RelOptRule> getRules(OptimizerRulesContext optimizerContext, PlannerPhase phase, SourceType pluginType) { final OptionManager options = optimizerContext.getPlannerSettings().getOptions(); switch(phase){ case LOGICAL: return ImmutableSet.<RelOptRule>of(new ElasticScanRule(pluginType)); case PHYSICAL: ImmutableSet.Builder<RelOptRule> builder = ImmutableSet.builder(); builder.add(new ElasticScanPrule(optimizerContext.getFunctionRegistry())); if (options.getOption(ExecConstants.ELASTIC_RULES_PROJECT)) { builder.add(new ElasticProjectRule(optimizerContext.getFunctionRegistry())); } if (options.getOption(ExecConstants.ELASTIC_RULES_FILTER)) { builder.add(ElasticFilterRule.INSTANCE); } if (options.getOption(ExecConstants.ELASTIC_RULES_LIMIT)) { builder.add(ElasticLimitRule.INSTANCE); } if (options.getOption(ExecConstants.ELASTIC_RULES_SAMPLE)) { builder.add(ElasticSampleRule.INSTANCE); } return builder.build(); default: return ImmutableSet.of(); } }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double totalInputRowCount = 0; for (int i = 0; i < this.getInputs().size(); i++) { totalInputRowCount += mq.getRowCount(this.getInputs().get(i)); } double cpuCost = totalInputRowCount * DremioCost.BASE_CPU_COST; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0); }