public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); if (scan instanceof DbGroupScan) { String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null) { IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); populateStats(condition, indexes, scanRel, context); logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); return true; } } return false; }
&& !condition.isAlwaysTrue()) { RexBuilder builder = scanRel.getCluster().getRexBuilder(); PlannerSettings settings = PrelUtil.getSettings(scanRel.getCluster()); String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice. return super.computeSelfCost(planner, mq).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = mq.getRowCount(child); // int rowWidth = child.getRowType().getPrecision(); int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DrillCostBase.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(inputRows)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints // TODO: use rowWidth instead of avgFieldWidth * numFields // avgFieldWidth * numFields * inputRows double numFields = this.getRowType().getFieldCount(); long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val; double memCost = fieldWidth * numFields * inputRows; DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0, memCost); }
final List<RelNode> inputs = union.getInputs(); List<RelNode> convertedInputList = Lists.newArrayList(); PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); boolean allHashDistributed = true; convertedChild = convert(child, PrelUtil.fixTraits(call, traitsChild)); } else { RelTraitSet traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL); convertedChild = convert(child, PrelUtil.fixTraits(call, traitsChild)); allHashDistributed = false;
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); if (PrelUtil.getSettings(getCluster()).isSingleMode()) { return childPOP; } SingleMergeExchange g = new SingleMergeExchange(childPOP, PrelUtil.getOrdering(this.collation, getInput().getRowType())); return creator.addMetadata(this, g); }
@Override public Iterator<Prel> iterator() { return PrelUtil.iter(getInput()); }
@Override public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); OrderedMuxExchange p = new OrderedMuxExchange(childPOP, PrelUtil.getOrdering(fieldCollation, getInput().getRowType())); return creator.addMetadata(this, p); }
public static RelTraitSet fixTraits(RelOptRuleCall call, RelTraitSet set) { return fixTraits(call.getPlanner(), set); }
List<RelDataTypeField> origRelDataTypes = new ArrayList<>(); int i = 0; final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());
@Override public RelOptCost computeSelfCost(final RelOptPlanner planner, RelMetadataQuery mq) { final PlannerSettings settings = PrelUtil.getPlannerSettings(planner); final ScanStats stats = this.getGroupScan().getScanStats(settings); final int columnCount = this.getRowType().getFieldCount(); if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost()); } double rowCount = stats.getRecordCount(); double cpuCost = rowCount * Math.max(columnCount, 1); if (stats.getCpuCost() > 0) { cpuCost *= stats.getCpuCost(); } double ioCost = stats.getDiskCost(); DrillCostBase.DrillCostFactory costFactory = (DrillCostBase.DrillCostFactory)planner.getCostFactory(); return costFactory.makeCost(rowCount, cpuCost, ioCost, 0); }
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); if (PrelUtil.getSettings(getCluster()).isSingleMode()) { return childPOP; } HashToMergeExchange g = new HashToMergeExchange(childPOP, HashPrelUtil.getHashExpression(this.distFields, getInput().getRowType()), PrelUtil.getOrdering(this.collation, getInput().getRowType())); return creator.addMetadata(this, g); }
@Override public Iterator<Prel> iterator() { return PrelUtil.iter(getInput()); }
@Override public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); TopN topN = new TopN(childPOP, PrelUtil.getOrdering(this.collation, getInput().getRowType()), false, this.limit); return creator.addMetadata(this, topN); }
private void createTransformRequest(RelOptRuleCall call, DrillAggregateRel aggregate, RelNode input, RelTraitSet traits) throws InvalidRelException { final RelNode convertedInput = convert(input, PrelUtil.fixTraits(call, traits)); HashAggPrel newAgg = new HashAggPrel( aggregate.getCluster(), traits, convertedInput, aggregate.indicator, aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList(), OperatorPhase.PHASE_1of1); call.transformTo(newAgg); }
@Override public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(1); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for Hive based partition pruning if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); } else { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); } }
@Override public RelOptCost computeSelfCost(final RelOptPlanner planner, RelMetadataQuery mq) { final PlannerSettings settings = PrelUtil.getPlannerSettings(planner); final ScanStats stats = this.getGroupScan().getScanStats(settings); final int columnCount = this.getRowType().getFieldCount(); if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost()); } // double rowCount = RelMetadataQuery.getRowCount(this); double rowCount = stats.getRecordCount(); // As DRILL-4083 points out, when columnCount == 0, cpuCost becomes zero, // which makes the costs of HiveScan and HiveDrillNativeParquetScan the same double cpuCost = rowCount * Math.max(columnCount, 1); // For now, assume cpu cost is proportional to row count. // If a positive value for CPU cost is given multiply the default CPU cost by given CPU cost. if (stats.getCpuCost() > 0) { cpuCost *= stats.getCpuCost(); } double ioCost = stats.getDiskCost(); DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory(); return costFactory.makeCost(rowCount, cpuCost, ioCost, 0); }
double forcedScalingFactor = PrelUtil.getSettings(scanRel.getCluster()).getIndexStatsRowCountScalingFactor();
@Override public Iterator<Prel> iterator() { return PrelUtil.iter(getInput()); }
@Override public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); org.apache.drill.exec.physical.config.Sort g = new ExternalSort(childPOP, PrelUtil.getOrdering(this.collation, getInput().getRowType()), false); return creator.addMetadata(this, g); }
@Override public void onMatch(RelOptRuleCall call) { final DrillUnionRel union = call.rel(0); final List<RelNode> inputs = union.getInputs(); List<RelNode> convertedInputList = Lists.newArrayList(); RelTraitSet traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL); try { for (RelNode input : inputs) { RelNode convertedInput = convert(input, PrelUtil.fixTraits(call, traits)); convertedInputList.add(convertedInput); } traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON); UnionDistinctPrel unionDistinct = new UnionDistinctPrel(union.getCluster(), traits, convertedInputList, false /* compatibility already checked during logical phase */); call.transformTo(unionDistinct); } catch (InvalidRelException e) { tracer.warn(e.toString()); } }