public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); if (scan instanceof DbGroupScan) { String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null) { IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); populateStats(condition, indexes, scanRel, context); logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); return true; } } return false; }
RexBuilder builder = scanRel.getCluster().getRexBuilder(); PlannerSettings settings = PrelUtil.getSettings(scanRel.getCluster()); newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); } else { newRowType = scanRel.getRowType(); StatisticsPayload payload = new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize); addToCache(idxCondition, idx, context, payload, jTabGrpScan, scanRel, newRowType); addBaseConditions(idxCondition, payload, false, baseConditionMap, scanRel.getRowType()); RexNode idxTotColCondition = IndexPlanUtils.getTotalFilter(idxLeadColCondition, idxTotRemColCondition, builder); FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); RelDataType newRowType = scanRel.getRowType(); if (functionInfo.hasFunctional()) { newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); convertToLogicalExpression(condition, scanRel.getRowType(), settings, builder)), null, scanRel); addToCache(condition, null, null, new MapRDBStatisticsPayload(rowCount, ftsLeadingKeyPayload.getRowCount(), ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType()); ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType());
RelDataType dbscanRowType = convertRowType(origScan.getRowType(), origScan.getCluster().getTypeFactory()); ScanPrel dbScan = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), restrictedGroupScan, dbscanRowType, origScan.getTable()); lastRelNode = dbScan;
public static ScanPrel buildCoveringIndexScan(DrillScanRelBase origScan, IndexGroupScan indexGroupScan, IndexCallContext indexContext, IndexDescriptor indexDesc) { FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo(); //to record the new (renamed)paths added List<SchemaPath> rewrittenPaths = Lists.newArrayList(); DbGroupScan dbGroupScan = (DbGroupScan) getGroupScan(origScan); indexGroupScan.setColumns( rewriteFunctionColumn(dbGroupScan.getColumns(), functionInfo, rewrittenPaths)); DrillDistributionTrait partition = scanIsPartition(getGroupScan(origScan))? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, indexContext, functionInfo, rewrittenPaths); // add a default collation trait otherwise Calcite runs into a ClassCastException, which at first glance // seems like a Calcite bug RelTraitSet indexScanTraitSet = origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL). plus(RelCollationTraitDef.INSTANCE.getDefault()).plus(partition); // Create the collation traits for index scan based on the index columns under the // condition that the index actually has collation property (e.g hash indexes don't) if (indexDesc.getCollation() != null) { RelCollation collationTrait = buildCollationCoveringIndexScan(indexDesc, indexContext); indexScanTraitSet = indexScanTraitSet.plus(collationTrait); } ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), indexScanTraitSet, indexGroupScan, newRowType, origScan.getTable()); return indexScanPrel; }
finalRel = new ProjectPrel(indexContext.getScan().getCluster(), indexScanTraitSet.plus(collation), indexScanPrel, indexContext.getLowerProject().getProjects(), indexContext.getLowerProject().getRowType()); finalRel.toString(), indexContext.getScan().toString(), indexContext.getLowerProject()!=null?indexContext.getLowerProject().getDigest(): indexContext.getScan().getDigest(), finalRel.getDigest()); return finalRel;
return fullTableScanPayload.getLeadingRowCount(); } else if (condition != null) { conditionAsStr = convertRexToString(condition, scanRel.getRowType()); payloadMap = statsCache.get(conditionAsStr); if (payloadMap != null) {
/** * This constructor is to build selector for no index condition case (no filter) * @param indexContext */ public IndexSelector(IndexCallContext indexContext) { this.indexCondition = null; this.otherRemainderCondition = null; this.indexContext = indexContext; this.totalRows = Statistics.ROWCOUNT_UNKNOWN; this.stats = indexContext.getGroupScan().getStatistics(); this.rexBuilder = indexContext.getScan().getCluster().getRexBuilder(); this.builder = null; this.primaryTableScan = indexContext.getScan(); this.indexPropList = Lists.newArrayList(); }
RelCollation collation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(origProject), null, origScan, functionInfo, indexContext); indexProjectPrel = new ProjectPrel(origScan.getCluster(), indexFilterTraitSet.plus(collation), indexFilterPrel, IndexPlanUtils.getProjects(origProject), origProject.getRowType()); finalRel.toString(), origScan.toString(), upperProject==null?indexContext.getFilter().getDigest(): upperProject.getDigest(), finalRel.getDigest()); return finalRel;
@Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw).item("groupscan", getGroupScan().getDigest()); }
public static GroupScan getGroupScan(DrillScanRelBase relNode) { return relNode.getGroupScan(); }
DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), indexScan, indexScanRowType, origScan.getTable()); FilterPrel indexFilterPrel = new FilterPrel(indexScanPrel.getCluster(), indexScanPrel.getTraitSet(), indexScanPrel, FunctionalIndexHelper.convertConditionForIndexScan(condition, origScan, finalRel.toString(), origScan.toString()); return finalRel;
public RelNode buildOriginalProject (RelNode newRel) { RelDataType origRowType = origProject == null ? origScan.getRowType() : origProject.getRowType(); final RelDataTypeFactory.FieldInfoBuilder finalFieldTypeBuilder = origScan.getCluster().getTypeFactory().builder(); List<RelDataTypeField> hjRowFields = newRel.getRowType().getFieldList(); int toRemoveRowKeyCount = 1; if (getRowKeyIndex(origRowType, origScan) < 0 ) { toRemoveRowKeyCount = 2; } finalFieldTypeBuilder.addAll(hjRowFields.subList(0, hjRowFields.size()-toRemoveRowKeyCount)); final RelDataType finalProjectRowType = finalFieldTypeBuilder.build(); List<RexNode> resetExprs = Lists.newArrayList(); for (int idx=0; idx<hjRowFields.size()-toRemoveRowKeyCount; ++idx) { resetExprs.add(RexInputRef.of(idx, newRel.getRowType())); } final ProjectPrel resetProjectPrel = new ProjectPrel(newRel.getCluster(), newRel.getTraitSet(), newRel, resetExprs, finalProjectRowType); newRel = resetProjectPrel; RelNode finalRel = Prule.convert(newRel, newRel.getTraitSet()); return finalRel; }
/** * Build the collation property for index scan * @param indexDesc the index for which we are building index plan * @param context the context of this index planning process * @return the output RelCollation for the scan on index */ public static RelCollation buildCollationCoveringIndexScan(IndexDescriptor indexDesc, IndexCallContext context) { Map<LogicalExpression, Integer> rowTypeExprs = getExprsFromRowType(context.getScan().getRowType()); return buildCollationForExpressions(rowTypeExprs, indexDesc, context); }
IndexGroupScan idxScan, FunctionalIndexInfo functionInfo) { RelDataTypeFactory typeFactory = origScan.getCluster().getTypeFactory(); List<RelDataTypeField> fields = new ArrayList<>();
@Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw).item("groupscan", this.getGroupScan().getDigest()); }
DrillDistributionTrait.DistributionType.RANGE_DISTRIBUTED, ImmutableList.copyOf(rangeDistFields), ((DbGroupScan)((DrillScanRelBase) leftScan).getGroupScan()).getRangePartitionFunction(rangeDistRefList));
RelDataType dbscanRowType = convertRowType(origScan.getRowType(), origScan.getCluster().getTypeFactory()); RelDataType indexScanRowType = FunctionalIndexHelper.convertRowTypeForIndexScan( origScan, indexContext.getOrigMarker(), indexGroupScan, functionInfo); DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), indexGroupScan, indexScanRowType, origScan.getTable()); DbGroupScan origDbGroupScan = (DbGroupScan)IndexPlanUtils.getGroupScan(origScan); DrillScanRel rightIdxRel = new DrillScanRel(origScan.getCluster(), origScan.getTraitSet(), origScan.getTable(), origScan.getRowType(), indexContext.getScanColumns()); double rightIdxRowCount = indexGroupScan.getRowCount(indexCondition, rightIdxRel); restrictedGroupScan.setRowCount(null, rightIdxRowCount, rightIdxRowCount); RelTraitSet origScanTraitSet = origScan.getTraitSet(); RelTraitSet restrictedScanTraitSet = origScanTraitSet.plus(Prel.DRILL_PHYSICAL); ScanPrel dbScan = new ScanPrel(origScan.getCluster(), restrictedScanTraitSet, restrictedGroupScan, dbscanRowType, origScan.getTable()); RelNode lastLeft = dbScan; RelDataType origRowType = origProject == null ? origScan.getRowType() : origProject.getRowType(); origScan.getCluster().getTypeFactory().builder(); finalRel.toString(), origScan.toString()); return finalRel;
public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); if (scan instanceof DbGroupScan) { String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null) { IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); populateStats(condition, indexes, scanRel, context); logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); return true; } } return false; }
RexBuilder builder = scanRel.getCluster().getRexBuilder(); PlannerSettings settings = PrelUtil.getSettings(scanRel.getCluster()); newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); } else { newRowType = scanRel.getRowType(); StatisticsPayload payload = new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize); addToCache(idxCondition, idx, context, payload, jTabGrpScan, scanRel, newRowType); addBaseConditions(idxCondition, payload, false, baseConditionMap, scanRel.getRowType()); RexNode idxTotColCondition = IndexPlanUtils.getTotalFilter(idxLeadColCondition, idxTotRemColCondition, builder); FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); RelDataType newRowType = scanRel.getRowType(); if (functionInfo.hasFunctional()) { newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); convertToLogicalExpression(condition, scanRel.getRowType(), settings, builder)), null, scanRel); addToCache(condition, null, null, new MapRDBStatisticsPayload(rowCount, ftsLeadingKeyPayload.getRowCount(), ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType()); ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType());
return fullTableScanPayload.getLeadingRowCount(); } else if (condition != null) { conditionAsStr = convertRexToString(condition, scanRel.getRowType()); payloadMap = statsCache.get(conditionAsStr); if (payloadMap != null) {