/** * Return true if the input datamap contains the column that needed in * specified expression */ private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) { if (mapMeta.getIndexedColumns().size() == 0 || columnExpressions.size() == 0) { return false; } boolean contains = true; for (ColumnExpression expression : columnExpressions) { if (!mapMeta.getIndexedColumnNames().contains(expression.getColumnName()) || !mapMeta.getOptimizedOperation().containsAll(expressionTypes)) { contains = false; break; } } return contains; }
/** * Return a new write for this datamap */ @Override public DataMapWriter createWriter(Segment segment, String shardName, SegmentProperties segmentProperties) { LOGGER.info("lucene data write to " + shardName); return new LuceneDataMapWriter(getCarbonTable().getTablePath(), dataMapName, dataMapMeta.getIndexedColumns(), segment, shardName, flushCacheSize, storeBlockletWise); }
@Override public DataMapBuilder createBuilder(Segment segment, String shardName, SegmentProperties segmentProperties) { return new LuceneDataMapBuilder(getCarbonTable().getTablePath(), dataMapName, segment, shardName, dataMapMeta.getIndexedColumns(), flushCacheSize, storeBlockletWise); }
@Override public DataMapBuilder createBuilder(Segment segment, String shardName, SegmentProperties segmentProperties) throws IOException { return new BloomDataMapBuilder(getCarbonTable().getTablePath(), this.dataMapName, this.dataMapMeta.getIndexedColumns(), segment, shardName, segmentProperties, this.bloomFilterSize, this.bloomFilterFpp, bloomCompress); }
/** * Register a DataMapWriter */ private void register(DataMapFactory factory, String segmentId, String taskNo, SegmentProperties segmentProperties) { assert (factory != null); assert (segmentId != null); DataMapMeta meta = factory.getMeta(); if (meta == null) { // if data map does not have meta, no need to register return; } List<CarbonColumn> columns = factory.getMeta().getIndexedColumns(); List<DataMapWriter> writers = registry.get(columns); DataMapWriter writer = null; try { writer = factory.createWriter(new Segment(segmentId), taskNo, segmentProperties); } catch (IOException e) { LOG.error("Failed to create DataMapWriter: " + e.getMessage()); throw new DataMapWriterException(e); } if (writers != null) { writers.add(writer); } else { writers = new ArrayList<>(); writers.add(writer); registry.put(columns, writers); } LOG.info("DataMapWriter " + writer + " added"); }
@Override public DataMapWriter createWriter(Segment segment, String shardName, SegmentProperties segmentProperties) throws IOException { LOGGER.info( String.format("Data of BloomCoarseGranDataMap %s for table %s will be written to %s", this.dataMapName, getCarbonTable().getTableName() , shardName)); return new BloomDataMapWriter(getCarbonTable().getTablePath(), this.dataMapName, this.dataMapMeta.getIndexedColumns(), segment, shardName, segmentProperties, this.bloomFilterSize, this.bloomFilterFpp, bloomCompress); }
@Override public void clear(Segment segment) { Set<String> shards = segmentMap.remove(segment.getSegmentNo()); if (shards != null) { for (String shard : shards) { for (CarbonColumn carbonColumn : dataMapMeta.getIndexedColumns()) { cache.invalidate(new BloomCacheKeyValue.CacheKey(shard, carbonColumn.getColName())); } } } }
private TableDataMap chooseDataMap(List<TableDataMap> allDataMap, List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) { List<DataMapTuple> tuples = new ArrayList<>(); for (TableDataMap dataMap : allDataMap) { if (null != dataMap.getDataMapFactory().getMeta() && contains( dataMap.getDataMapFactory().getMeta(), columnExpressions, expressionTypes)) { tuples.add( new DataMapTuple(dataMap.getDataMapFactory().getMeta().getIndexedColumns().size(), dataMap)); } } if (tuples.size() > 0) { Collections.sort(tuples); return tuples.get(0).dataMap; } return null; }
@Override public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); String indexPath = ((BloomDataMapDistributable) distributable).getIndexPath(); Set<String> filteredShards = ((BloomDataMapDistributable) distributable).getFilteredShards(); BloomCoarseGrainDataMap bloomDM = new BloomCoarseGrainDataMap(); bloomDM.init(new BloomDataMapModel(indexPath, cache, FileFactory.getConfiguration())); bloomDM.initIndexColumnConverters(getCarbonTable(), dataMapMeta.getIndexedColumns()); bloomDM.setFilteredShard(filteredShards); dataMaps.add(bloomDM); return dataMaps; }
@Override public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); try { Set<String> shardPaths = segmentMap.get(segment.getSegmentNo()); if (shardPaths == null) { shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo()); segmentMap.put(segment.getSegmentNo(), shardPaths); } Set<String> filteredShards = segment.getFilteredIndexShardNames(); for (String shard : shardPaths) { if (shard.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shard).getName())) { // Filter out the tasks which are filtered through Main datamap. // for merge shard, shard pruning delay to be done before pruning blocklet BloomCoarseGrainDataMap bloomDM = new BloomCoarseGrainDataMap(); bloomDM.init(new BloomDataMapModel(shard, cache, segment.getConfiguration())); bloomDM.initIndexColumnConverters(getCarbonTable(), dataMapMeta.getIndexedColumns()); bloomDM.setFilteredShard(filteredShards); dataMaps.add(bloomDM); } } } catch (Exception e) { throw new IOException("Error occurs while init Bloom DataMap", e); } return dataMaps; }