/** * This method will get the value for the given key. If value does not exist * for the given key, it will check and load the value. * * @param key * @return * @throws IOException in case memory is not sufficient to load data into memory */ public V get(K key) throws IOException { V value = cache.get(key); if (value != null) { segmentSet.add(key); } return value; }
/** * This method will remove the cache for a given key * * @param keys */ public void invalidateAll(List<K> keys) { for (K key : keys) { cache.invalidate(key); } }
/** * This method will return the value for the given key. It will not check and load * the data for the given key * * @param key * @return */ public V getIfPresent(K key) { V value = cache.getIfPresent(key); if (value != null) { segmentSet.add(key); } return value; }
@Override public void clear(Segment segment) { Set<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo()); if (blockIndexes != null) { for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) { TableBlockIndexUniqueIdentifierWrapper blockIndexWrapper = new TableBlockIndexUniqueIdentifierWrapper(blockIndex, this.getCarbonTable()); BlockletDataMapIndexWrapper wrapper = cache.getIfPresent(blockIndexWrapper); if (null != wrapper) { List<BlockDataMap> dataMaps = wrapper.getDataMaps(); for (DataMap dataMap : dataMaps) { if (dataMap != null) { cache.invalidate(blockIndexWrapper); dataMap.clear(); } } } } } }
@Override public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment); List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>(identifiers.size()); for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) { tableBlockIndexUniqueIdentifierWrappers.add( new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable())); } List<BlockletDataMapIndexWrapper> blockletDataMapIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers); for (BlockletDataMapIndexWrapper wrapper : blockletDataMapIndexWrappers) { dataMaps.addAll(wrapper.getDataMaps()); } return dataMaps; }
/** * This method will clear the access count for a given list of segments * * @param segmentList */ public void clearAccessCount(List<K> segmentList) { cache.clearAccessCount(segmentList); // remove from segment set so that access count is not decremented again during close operation segmentSet.removeAll(segmentList); }
@Override public void cache(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper blockletDataMapIndexWrapper) throws IOException, MemoryException { cache.put(tableBlockIndexUniqueIdentifierWrapper, blockletDataMapIndexWrapper); }
.createCache(CacheType.FORWARD_DICTIONARY); List<Dictionary> columnDictionaryList = forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers); Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size()); for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
/** * the method is used to clear access count of the unused segments cacheable object */ public void close() { List<K> segmentArrayList = new ArrayList<>(segmentSet.size()); segmentArrayList.addAll(segmentSet); cache.clearAccessCount(segmentArrayList); cache = null; }
public static Dictionary getDictionary(DictionaryColumnUniqueIdentifier columnIdentifier) throws IOException { Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY); return dictCache.get(columnIdentifier); }
@Override public void clear(Segment segment) { Set<String> shards = segmentMap.remove(segment.getSegmentNo()); if (shards != null) { for (String shard : shards) { for (CarbonColumn carbonColumn : dataMapMeta.getIndexedColumns()) { cache.invalidate(new BloomCacheKeyValue.CacheKey(shard, carbonColumn.getColName())); } } } }
@Override public List<DataMapDistributable> getAllUncachedDistributables( List<DataMapDistributable> distributables) throws IOException { List<DataMapDistributable> distributablesToBeLoaded = new ArrayList<>(distributables.size()); for (DataMapDistributable distributable : distributables) { Segment segment = distributable.getSegment(); Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableBlockIndexUniqueIdentifiers(segment); // filter out the tableBlockIndexUniqueIdentifiers based on distributable TableBlockIndexUniqueIdentifier validIdentifier = BlockletDataMapUtil .filterIdentifiersBasedOnDistributable(tableBlockIndexUniqueIdentifiers, (BlockletDataMapDistributable) distributable); if (null == cache.getIfPresent( new TableBlockIndexUniqueIdentifierWrapper(validIdentifier, this.getCarbonTable()))) { ((BlockletDataMapDistributable) distributable) .setTableBlockIndexUniqueIdentifier(validIdentifier); distributablesToBeLoaded.add(distributable); } } return distributablesToBeLoaded; } }
cache.getAll(tableBlockIndexUniqueIdentifierWrappers); for (BlockletDataMapIndexWrapper wrapper : blockletDataMapIndexWrappers) { Segment segment = segmentMap.get(wrapper.getSegmentId());
private ExtendedBlocklet getExtendedBlocklet( Set<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper, Blocklet blocklet) throws IOException { for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : identifiersWrapper) { BlockletDataMapIndexWrapper wrapper = cache.get(identifierWrapper); List<BlockDataMap> dataMaps = wrapper.getDataMaps(); for (DataMap dataMap : dataMaps) { if (((BlockDataMap) dataMap) .getTableTaskInfo(BlockletDataMapRowIndexes.SUMMARY_INDEX_FILE_NAME) .startsWith(blocklet.getFilePath())) { return ((BlockDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId()); } } } throw new IOException("Blocklet not found: " + blocklet.toString()); }
/** * This method will remove dictionary cache from driver for both reverse and forward dictionary * * @param carbonTableIdentifier * @param columnId */ public static void removeDictionaryColumnFromCache(AbsoluteTableIdentifier carbonTableIdentifier, String columnId) { Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, new ColumnIdentifier(columnId, null, null)); dictCache.invalidate(dictionaryColumnUniqueIdentifier); dictCache = CacheProvider.getInstance().createCache(CacheType.FORWARD_DICTIONARY); dictCache.invalidate(dictionaryColumnUniqueIdentifier); } }
List<BlockletDataMapIndexWrapper> wrappers = cache.getAll(identifiersWrapper); for (BlockletDataMapIndexWrapper wrapper : wrappers) { dataMaps.addAll(wrapper.getDataMaps());
BloomCacheKeyValue.CacheValue cacheValue = cache.get(cacheKey); List<CarbonBloomFilter> bloomIndexList = cacheValue.getBloomFilters(); for (CarbonBloomFilter bloomFilter : bloomIndexList) {
Dictionary dict = (Dictionary) dictCache.get( new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, dims.get(i).getDataType()));
Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance() .createCache(CacheType.REVERSE_DICTIONARY); dictionary = dictCache.get(identifier);