private boolean columnsIndexed(Collection<LogicalExpression> expressions, boolean allColsIndexed) { List<LogicalExpression> decodedCols = new DecodePathinExpr().parseExpressions(expressions); if (allColsIndexed) { return columnsInIndexFields(decodedCols, indexedFields); } else { return someColumnsInIndexFields(decodedCols, indexedFields); } }
@Override public boolean isCoveringIndex(List<LogicalExpression> expressions) { List<LogicalExpression> decodedCols = new DecodePathinExpr().parseExpressions(expressions); return columnsInIndexFields(decodedCols, allFields); }
public FunctionalIndexInfo getFunctionalInfo() { if (this.functionalInfo == null) { this.functionalInfo = new MapRDBFunctionalIndexInfo(this); } return this.functionalInfo; }
private RexNode convertCastForFIdx(RexCall condition, IndexDescriptor index, IndexCallContext context, RelNode origScan) { if (index == null) { return condition; } FunctionalIndexInfo functionInfo = index.getFunctionalInfo(); if (!functionInfo.hasFunctional()) { return condition; } // The functional index has a different row-type than the original scan. Use the index row-type when // converting the condition RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, context, functionInfo); RexBuilder builder = origScan.getCluster().getRexBuilder(); return FunctionalIndexHelper.convertConditionForIndexScan(condition, origScan, newRowType, builder, functionInfo); }
/** * This function computes statistics when there is no query condition * @param jTabGrpScan - The current group scan * @param indexes - The collection of indexes to use for getting statistics * @param scanRel - The current scanRel * @param context - The index plan call context */ private void populateStatsForNoFilter(JsonTableGroupScan jTabGrpScan, IndexCollection indexes, RelNode scanRel, IndexCallContext context) { // Get the stats payload for full table (has total rows in the table) StatisticsPayload ftsPayload = jTabGrpScan.getFirstKeyEstimatedStats(null, null, scanRel); addToCache(null, null, context, ftsPayload, jTabGrpScan, scanRel, scanRel.getRowType()); addToCache(null, jTabGrpScan.getAverageRowSizeStats(null), ftsPayload); // Get the stats for all indexes for (IndexDescriptor idx: indexes) { StatisticsPayload idxPayload = jTabGrpScan.getFirstKeyEstimatedStats(null, idx, scanRel); StatisticsPayload idxRowSizePayload = jTabGrpScan.getAverageRowSizeStats(idx); RelDataType newRowType; FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); if (functionInfo.hasFunctional()) { newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); } else { newRowType = scanRel.getRowType(); } addToCache(null, idx, context, idxPayload, jTabGrpScan, scanRel, newRowType); addToCache(idx, idxRowSizePayload, ftsPayload); } }
@Override public String buildUniqueIndexIdentifier(IndexDescriptor idx) { if (idx == null) { return null; } else { return idx.getTableName() + "_" + idx.getIndexName(); } }
private void addToCache(IndexDescriptor idx, StatisticsPayload payload, StatisticsPayload ftsPayload) { String tabIdxIdentifier = buildUniqueIndexIdentifier(idx); if (fIStatsCache.get(tabIdxIdentifier) == null) { if (ftsPayload.getAvgRowSize() >= payload.getAvgRowSize()) { fIStatsCache.put(tabIdxIdentifier, payload); logger.debug("Statistics: fIStatsCache:<{}, {}>",tabIdxIdentifier, payload); } else { StatisticsPayload cappedPayload = new MapRDBStatisticsPayload(ROWCOUNT_UNKNOWN, ROWCOUNT_UNKNOWN, ftsPayload.getAvgRowSize()); fIStatsCache.put(tabIdxIdentifier,cappedPayload); logger.debug("Statistics: fIStatsCache:<{}, {}> (Capped)",tabIdxIdentifier, cappedPayload); } } else { logger.debug("Statistics: Average row size already exists for :<{}, {}>. Skip!",tabIdxIdentifier, payload); } }
private LogicalExpression getIndexExpression(IndexFieldDesc desc) throws InvalidIndexDefinitionException { final String fieldName = desc.getFieldPath().asPathString(); final String functionDef = desc.getFunctionName(); if ((functionDef != null)) { // this is a function String[] tokens = functionDef.split("\\s+"); if (tokens[0].equalsIgnoreCase("cast")) { if (tokens.length != 3) { throw new InvalidIndexDefinitionException("cast function definition not recognized: " + functionDef); } LogicalExpression idxExpr = castFunctionSQLSyntax(fieldName, tokens[2]); if (idxExpr == null) { throw new InvalidIndexDefinitionException("got null expression for function definition: " + functionDef); } return idxExpr; } else { throw new InvalidIndexDefinitionException("function definition is not supported for indexing: " + functionDef); } } // else it is a schemaPath return fieldName2SchemaPath(fieldName); }
@Override public IndexCollection getSecondaryIndexCollection(RelNode scanRel) { IndexDiscover discover = IndexDiscoverFactory.getIndexDiscover( getStorageConfig(), this, scanRel, MapRDBIndexDiscover.class); if (discover == null) { logger.error("Null IndexDiscover was found for {}!", scanRel); } return discover.getTableIndex(getTableName()); }
private LogicalExpression castFunctionSQLSyntax(String field, String type) throws InvalidIndexDefinitionException { // get castTypeStr so we can construct SQL syntax string before MapRDB could provide such syntax String castTypeStr = getDrillTypeStr(type); if(castTypeStr == null) { // no cast throw new InvalidIndexDefinitionException("cast function type not recognized: " + type + "for field " + field); } try { String castFunc = String.format("cast( %s as %s)", field, castTypeStr); return LogicalExpressionParser.parse(castFunc); } catch (Exception ex) { logger.error("parse failed: {}", ex); } return null; }
@Override public IndexCollection getTableIndex(String tableName) { return getTableIndexFromMFS(tableName); }
@Override public boolean someColumnsIndexed(Collection<LogicalExpression> columns) { return columnsIndexed(columns, false); }
private List<LogicalExpression> field2SchemaPath(Collection<IndexFieldDesc> descCollection) throws InvalidIndexDefinitionException { List<LogicalExpression> listSchema = new ArrayList<>(); for (IndexFieldDesc field : descCollection) { listSchema.add(getIndexExpression(field)); } return listSchema; }
/** * Get the estimated statistics after applying the {@link RexNode} condition. DO NOT call this API directly. * Call the stats API instead which modifies the counts based on preference options. * @param condition, filter to apply * @param index, to use for generating the estimate * @return row count post filtering */ public MapRDBStatisticsPayload getFirstKeyEstimatedStats(QueryCondition condition, IndexDescriptor index, RelNode scanRel) { IndexDesc indexDesc = null; if (index != null) { indexDesc = (IndexDesc)((MapRDBIndexDescriptor)index).getOriginalDesc(); } return getFirstKeyEstimatedStatsInternal(condition, indexDesc, scanRel); }
public JsonTableGroupScan(String userName, AbstractStoragePlugin storagePlugin, MapRDBFormatPlugin formatPlugin, JsonScanSpec scanSpec, List<SchemaPath> columns) { super(storagePlugin, formatPlugin, columns, userName); this.scanSpec = scanSpec; this.stats = new MapRDBStatistics(); this.forcedRowCountMap = new HashMap<>(); init(); }
FileSelection deriveFSSelection(DrillFileSystem fs, IndexDescriptor idxDesc) throws IOException { String tableName = idxDesc.getTableName(); String[] tablePath = tableName.split(DrillFileUtils.SEPARATOR); String tableParent = tableName.substring(0, tableName.lastIndexOf(DrillFileUtils.SEPARATOR)); return FileSelection.create(fs, tableParent, tablePath[tablePath.length - 1], false); }
private CollationContext buildCollationContext(List<LogicalExpression> indexFields, List<RelFieldCollation> indexFieldCollations) { assert indexFieldCollations.size() <= indexFields.size(); Map<LogicalExpression, RelFieldCollation> collationMap = Maps.newHashMap(); for (int i = 0; i < indexFieldCollations.size(); i++) { collationMap.put(indexFields.get(i), indexFieldCollations.get(i)); } return new CollationContext(collationMap, indexFieldCollations); }
private boolean hasCastExpression(RexNode condition) { if (condition instanceof RexCall) { if (((RexCall) condition).getOperator().getKind() == SqlKind.CAST) { return true; } for (RexNode op : ((RexCall) condition).getOperands()) { if (hasCastExpression(op)) { return true; } } } return false; } /*
public MapRDBFunctionalIndexInfo(IndexDescriptor indexDesc) { this.indexDesc = indexDesc; columnToConvert = Maps.newHashMap(); exprToConvert = Maps.newHashMap(); pathsInExpr = Maps.newHashMap(); // keep the order of new paths, it may be related to the naming policy newPathsForIndexedFunction = Sets.newLinkedHashSet(); allPathsInFunction = Sets.newHashSet(); init(); }
@Override public boolean allColumnsIndexed(Collection<LogicalExpression> expressions) { List<LogicalExpression> decodedCols = new DecodePathinExpr().parseExpressions(expressions); return columnsInIndexFields(decodedCols, indexedFields); }