GroupScan grpScan = IndexPlanUtils.getGroupScan(scanRel); baseConditionMap = new HashMap<>(); for (IndexDescriptor idx : firstKeyIdxConditionMap.keySet()) { if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { continue; if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { continue; RexNode idxRemColCondition = IndexPlanUtils.getLeadingPrefixMap(leadingPrefixMap, idx.getIndexColumns(), infoBuilder, idxCondition); RexNode idxLeadColCondition = IndexPlanUtils.getLeadingColumnsFilter( IndexPlanUtils.getLeadingFilters(leadingPrefixMap, idx.getIndexColumns()), builder); RexNode idxTotRemColCondition = IndexPlanUtils.getTotalRemainderFilter(idxRemColCondition, idxIncColCondition, builder); RexNode idxTotColCondition = IndexPlanUtils.getTotalFilter(idxLeadColCondition, idxTotRemColCondition, builder); FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); RelDataType newRowType = scanRel.getRowType();
IndexPlanUtils.buildCoveringIndexScan(origScan, indexGroupScan, indexContext, indexDesc); coveringCondition = IndexPlanUtils.getTotalFilter(indexCondition, remainderCondition, indexScanPrel.getCluster().getRexBuilder()); RexNode newIndexCondition = rewriteFunctionalCondition(coveringCondition, indexScanPrel.getRowType(), functionInfo); RelCollation collation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(origProject), null, origScan, functionInfo, indexContext); indexProjectPrel = new ProjectPrel(origScan.getCluster(), indexFilterTraitSet.plus(collation), indexFilterPrel, IndexPlanUtils.getProjects(origProject), origProject.getRowType()); IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(upperProject), origProject, origScan, functionInfo, indexContext); finalRel, IndexPlanUtils.getProjects(upperProject), upperProject.getRowType()); DrillParseContext parseContxt = new DrillParseContext(PrelUtil.getPlannerSettings(newProject.getCluster())); for(RexNode projectRex: newProject.getProjects()) { RexNode newRex = IndexPlanUtils.rewriteFunctionalRex(indexContext, parseContxt, null, origScan, projectRex, indexScanPrel.getRowType(), functionInfo); newProjects.add(newRex);
IndexPlanUtils.updateSortExpression(indexContext, indexContext.getSort() != null ? indexContext.getCollation().getFieldCollations() : null); IndexPlanUtils.buildCoveringIndexScan(origScan, indexGroupScan, indexContext, indexDesc); ((IndexGroupScan)indexScanPrel.getGroupScan()).setStatistics(((DbGroupScan)IndexPlanUtils.getGroupScan(origScan)).getStatistics()); RelTraitSet indexScanTraitSet = indexScanPrel.getTraitSet(); if (indexContext.getLowerProject() != null) { RelCollation collation = IndexPlanUtils.buildCollationProject(indexContext.getLowerProject().getProjects(), null, indexContext.getScan(), functionInfo, indexContext); finalRel = new ProjectPrel(indexContext.getScan().getCluster(), indexScanTraitSet.plus(collation), DrillParseContext parseContxt = new DrillParseContext(PrelUtil.getPlannerSettings(newProject.getCluster())); for(RexNode projectRex: newProject.getProjects()) { RexNode newRex = IndexPlanUtils.rewriteFunctionalRex(indexContext, parseContxt, null, origScan, projectRex, indexScanPrel.getRowType(), functionInfo); newProjects.add(newRex);
origScan, indexContext.getOrigMarker(), indexGroupScan, functionInfo); DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan))? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; DbGroupScan origDbGroupScan = (DbGroupScan)IndexPlanUtils.getGroupScan(origScan); if (indexDesc.getCollation() != null && !settings.isIndexForceSortNonCovering()) { collation = IndexPlanUtils.buildCollationNonCoveringIndexScan(indexDesc, indexScanRowType, dbscanRowType, indexContext); if (restrictedScanTraitSet.contains(RelCollationTraitDef.INSTANCE)) { // replace existing trait restrictedScanTraitSet = restrictedScanTraitSet.plus(partition).replace(collation); leftProjectExprs.addAll(IndexPlanUtils.getProjects(origProject)); collation = IndexPlanUtils.buildCollationProject(leftProjectExprs, null, dbScan, functionInfo, indexContext); newCollation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(upperProject), origProject, origScan, functionInfo, indexContext); ProjectPrel cap = new ProjectPrel(upperProject.getCluster(), newProjectTraits, newRel, IndexPlanUtils.getProjects(upperProject), upperProject.getRowType()); newRel = cap;
public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); if (scan instanceof DbGroupScan) { String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null) { IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); populateStats(condition, indexes, scanRel, context); logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); return true; } } return false; }
public static ScanPrel buildCoveringIndexScan(DrillScanRelBase origScan, IndexGroupScan indexGroupScan, IndexCallContext indexContext, IndexDescriptor indexDesc) { FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo(); //to record the new (renamed)paths added List<SchemaPath> rewrittenPaths = Lists.newArrayList(); DbGroupScan dbGroupScan = (DbGroupScan) getGroupScan(origScan); indexGroupScan.setColumns( rewriteFunctionColumn(dbGroupScan.getColumns(), functionInfo, rewrittenPaths)); DrillDistributionTrait partition = scanIsPartition(getGroupScan(origScan))? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, indexContext, functionInfo, rewrittenPaths); // add a default collation trait otherwise Calcite runs into a ClassCastException, which at first glance // seems like a Calcite bug RelTraitSet indexScanTraitSet = origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL). plus(RelCollationTraitDef.INSTANCE.getDefault()).plus(partition); // Create the collation traits for index scan based on the index columns under the // condition that the index actually has collation property (e.g hash indexes don't) if (indexDesc.getCollation() != null) { RelCollation collationTrait = buildCollationCoveringIndexScan(indexDesc, indexContext); indexScanTraitSet = indexScanTraitSet.plus(collationTrait); } ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), indexScanTraitSet, indexGroupScan, newRowType, origScan.getTable()); return indexScanPrel; }
IndexPlanUtils.updateSortExpression(indexContext, indexContext.sort != null ? indexContext.sort.collation.getFieldCollations() : null); logger.info("index_plan_info indexDescriptor: {}", indexDesc.toString()); if (IndexPlanUtils.conditionIndexed(indexableExprMarker, indexDesc) != IndexPlanUtils.ConditionIndexed.NONE) { if (isValidIndexHint && !indexContext.indexHint.equals(indexDesc.getIndexName())) { logger.info("index_plan_info: Index {} is being discarded due to index Hint", indexDesc.getIndexName()); selector.addIndex(indexDesc, IndexPlanUtils.isCoveringIndex(indexContext, functionInfo), indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : scan.getRowType().getFieldCount());
private Pair<RelNode, DbGroupScan> buildRestrictedDBScan(RexNode remnant, boolean isAnyIndexAsync) { DbGroupScan origDbGroupScan = (DbGroupScan)IndexPlanUtils.getGroupScan(origScan); List<SchemaPath> cols = new ArrayList<SchemaPath>(origDbGroupScan.getColumns()); if (!checkRowKey(cols)) { return null; DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan))? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON; leftProjectExprs.addAll(IndexPlanUtils.getProjects(origProject));
/** * For a particular table scan for table T1 and an index on that table, find out if it is a covering index * @return */ static public boolean isCoveringIndex(IndexCallContext indexContext, FunctionalIndexInfo functionInfo) { if (functionInfo.hasFunctional()) { // need info from full query return queryCoveredByIndex(indexContext, functionInfo); } DbGroupScan groupScan = (DbGroupScan) getGroupScan(indexContext.getScan()); List<LogicalExpression> tableCols = Lists.newArrayList(); tableCols.addAll(groupScan.getColumns()); return functionInfo.getIndexDesc().isCoveringIndex(tableCols); }
(DbGroupScan)IndexPlanUtils.getGroupScan(origScan)); if (upperProject != null) { ProjectPrel cap = new ProjectPrel(finalRel.getCluster(), finalRel.getTraitSet(), finalRel, IndexPlanUtils.getProjects(upperProject), upperProject.getRowType()); finalRel = cap;
RelDataType indexScanRowType = FunctionalIndexHelper.convertRowTypeForIndexScan( origScan, indexContext.getOrigMarker(), indexScan, functionInfo); DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan))? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
public RexNode getLeadingColumnsFilter() { return IndexPlanUtils.getLeadingColumnsFilter(leadingFilters, rexBuilder); }
public RexNode getTotalRemainderFilter() { return IndexPlanUtils.getTotalRemainderFilter(indexColumnsRemainderFilter, otherColumnsRemainderFilter, rexBuilder); }
/** * Build collation property for project, the one closer to the Scan * @param projectRexs the expressions to project * @param project the project between projectRexs and input, it could be null if no such intermediate project(lower project) * @param input the input RelNode to the project, usually it is the scan operator. * @param indexInfo the index for which we are building index plan * @param context the context of this index planning process * @return the output RelCollation */ public static RelCollation buildCollationProject(List<RexNode> projectRexs, DrillProjectRelBase project, RelNode input, FunctionalIndexInfo indexInfo, IndexCallContext context) { Map<LogicalExpression, Integer> projectExprs = getProjectExprs(projectRexs, project, input); return buildCollationForExpressions(projectExprs, indexInfo.getIndexDesc(), context); }
/** * Build the collation property for index scan * @param indexDesc the index for which we are building index plan * @param context the context of this index planning process * @return the output RelCollation for the scan on index */ public static RelCollation buildCollationCoveringIndexScan(IndexDescriptor indexDesc, IndexCallContext context) { Map<LogicalExpression, Integer> rowTypeExprs = getExprsFromRowType(context.getScan().getRowType()); return buildCollationForExpressions(rowTypeExprs, indexDesc, context); }
leadingFilters = IndexPlanUtils.getLeadingFilters(leadingPrefixMap, indexDescriptor.getIndexColumns());
/** * This method analyzes an index's columns and starting from the first column, checks * which part of the filter condition matches that column. This process continues with * subsequent columns. The goal is to identify the portion of the filter condition that * match the prefix columns. If there are additional conditions that don't match prefix * columns, that condition is set as a remainder condition. * @param indexProps */ public void analyzePrefixMatches(IndexProperties indexProps) { RexNode initCondition = indexCondition.isAlwaysTrue() ? null : indexCondition; Map<LogicalExpression, RexNode> leadingPrefixMap = Maps.newHashMap(); List<LogicalExpression> indexCols = indexProps.getIndexDesc().getIndexColumns(); boolean satisfiesCollation = false; if (indexCols.size() > 0) { if (initCondition != null) { // check filter condition initCondition = IndexPlanUtils.getLeadingPrefixMap(leadingPrefixMap, indexCols, builder, indexCondition); } if (requiredCollation()) { satisfiesCollation = buildAndCheckCollation(indexProps); } } indexProps.setProperties(leadingPrefixMap, satisfiesCollation, initCondition /* the remainder condition for indexed columns */, stats); }
public RelOptCost getSelfCost(RelOptPlanner planner) { if (selfCost != null) { return selfCost; } selfCost = indexDescriptor.getCost(this, planner, numProjectedFields, IndexPlanUtils.getGroupScan(primaryTableScan)); return selfCost; }
GroupScan grpScan = IndexPlanUtils.getGroupScan(scanRel); baseConditionMap = new HashMap<>(); for (IndexDescriptor idx : firstKeyIdxConditionMap.keySet()) { if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { continue; if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { continue; RexNode idxRemColCondition = IndexPlanUtils.getLeadingPrefixMap(leadingPrefixMap, idx.getIndexColumns(), infoBuilder, idxCondition); RexNode idxLeadColCondition = IndexPlanUtils.getLeadingColumnsFilter( IndexPlanUtils.getLeadingFilters(leadingPrefixMap, idx.getIndexColumns()), builder); RexNode idxTotRemColCondition = IndexPlanUtils.getTotalRemainderFilter(idxRemColCondition, idxIncColCondition, builder); RexNode idxTotColCondition = IndexPlanUtils.getTotalFilter(idxLeadColCondition, idxTotRemColCondition, builder); FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); RelDataType newRowType = scanRel.getRowType();
public static int getRowKeyIndex(RelDataType rowType, DrillScanRelBase origScan) { List<String> fieldNames = rowType.getFieldNames(); int idx = 0; for (String field : fieldNames) { if (field.equalsIgnoreCase(((DbGroupScan)IndexPlanUtils.getGroupScan(origScan)).getRowKeyName())) { return idx; } idx++; } return -1; }