cache.getAll(tableBlockIndexUniqueIdentifierWrappers); for (BlockletDataMapIndexWrapper wrapper : blockletDataMapIndexWrappers) { Segment segment = segmentMap.get(wrapper.getSegmentId()); List<CoarseGrainDataMap> datamapList = dataMaps.get(segment); if (null == datamapList) { datamapList = new ArrayList<CoarseGrainDataMap>(); datamapList.addAll(wrapper.getDataMaps()); dataMaps.put(segment, datamapList);
@Override public void put(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper wrapper) throws IOException, MemoryException { // As dataMap will use unsafe memory, it is not recommended to overwrite an existing entry // as in that case clearing unsafe memory need to be taken card. If at all datamap entry // in the cache need to be overwritten then use the invalidate interface // and then use the put interface if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) { List<BlockDataMap> dataMaps = wrapper.getDataMaps(); try { for (BlockDataMap blockletDataMap : dataMaps) { blockletDataMap.convertToUnsafeDMStore(); } // Locking is not required here because in LRU cache map add method is synchronized to add // only one entry at a time and if a key already exists it will not overwrite the entry lruCache.put(tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier(), wrapper, wrapper.getMemorySize()); } catch (Throwable e) { // clear all the memory acquired by data map in case of any failure for (DataMap blockletDataMap : dataMaps) { blockletDataMap.clear(); } throw new IOException("Problem in adding datamap to cache.", e); } } }
@Override public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment); List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>(identifiers.size()); for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) { tableBlockIndexUniqueIdentifierWrappers.add( new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable())); } List<BlockletDataMapIndexWrapper> blockletDataMapIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers); for (BlockletDataMapIndexWrapper wrapper : blockletDataMapIndexWrappers) { dataMaps.addAll(wrapper.getDataMaps()); } return dataMaps; }
dataMaps.add(blockletDataMap); blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(identifier.getSegmentId(), dataMaps); } else { new BlockletDataMapIndexWrapper(identifier.getSegmentId(), dataMaps); blockletDataMapIndexWrapper.getMemorySize());
List<BlockDataMap> dataMaps = blockletDataMapIndexWrapper.getDataMaps(); for (DataMap dataMap : dataMaps) { dataMap.clear();
private ExtendedBlocklet getExtendedBlocklet( Set<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper, Blocklet blocklet) throws IOException { for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : identifiersWrapper) { BlockletDataMapIndexWrapper wrapper = cache.get(identifierWrapper); List<BlockDataMap> dataMaps = wrapper.getDataMaps(); for (DataMap dataMap : dataMaps) { if (((BlockDataMap) dataMap) .getTableTaskInfo(BlockletDataMapRowIndexes.SUMMARY_INDEX_FILE_NAME) .startsWith(blocklet.getFilePath())) { return ((BlockDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId()); } } } throw new IOException("Blocklet not found: " + blocklet.toString()); }
List<BlockletDataMapIndexWrapper> wrappers = cache.getAll(identifiersWrapper); for (BlockletDataMapIndexWrapper wrapper : wrappers) { dataMaps.addAll(wrapper.getDataMaps());
@Override public void clear(Segment segment) { Set<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo()); if (blockIndexes != null) { for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) { TableBlockIndexUniqueIdentifierWrapper blockIndexWrapper = new TableBlockIndexUniqueIdentifierWrapper(blockIndex, this.getCarbonTable()); BlockletDataMapIndexWrapper wrapper = cache.getIfPresent(blockIndexWrapper); if (null != wrapper) { List<BlockDataMap> dataMaps = wrapper.getDataMaps(); for (DataMap dataMap : dataMaps) { if (dataMap != null) { cache.invalidate(blockIndexWrapper); dataMap.clear(); } } } } } }
/** * method invalidate the segment cache for segment * * @param tableSegmentUniqueIdentifierWrapper */ @Override public void invalidate( TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) { BlockletDataMapIndexWrapper blockletDataMapIndexWrapper = getIfPresent(tableSegmentUniqueIdentifierWrapper); if (null != blockletDataMapIndexWrapper) { // clear the segmentProperties cache List<BlockDataMap> dataMaps = blockletDataMapIndexWrapper.getDataMaps(); if (null != dataMaps && !dataMaps.isEmpty()) { String segmentId = tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getSegmentId(); // as segmentId will be same for all the dataMaps and segmentProperties cache is // maintained at segment level so it need to be called only once for clearing SegmentPropertiesAndSchemaHolder.getInstance() .invalidate(segmentId, dataMaps.get(0).getSegmentPropertiesIndex(), tableSegmentUniqueIdentifierWrapper.isAddTableBlockToUnsafeAndLRUCache()); } } lruCache.remove(tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier()); }