/** * This method will the index files tableBlockIndexUniqueIdentifiers of a merge index file * * @param identifier * @param segmentIndexFileStore * @return * @throws IOException */ public static List<TableBlockIndexUniqueIdentifier> getIndexFileIdentifiersFromMergeFile( TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore segmentIndexFileStore) throws IOException { List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = new ArrayList<>(); String mergeFilePath = identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(); segmentIndexFileStore.readMergeFile(mergeFilePath); List<String> indexFiles = segmentIndexFileStore.getCarbonMergeFileToIndexFilesMap().get(mergeFilePath); for (String indexFile : indexFiles) { tableBlockIndexUniqueIdentifiers.add( new TableBlockIndexUniqueIdentifier(identifier.getIndexFilePath(), indexFile, identifier.getIndexFileName(), identifier.getSegmentId())); } return tableBlockIndexUniqueIdentifiers; }
/** * Below method will be used to load the segment of segments * One segment may have multiple task , so table segment will be loaded * based on task id and will return the map of taksId to table segment * map * * @return map of taks id to segment mapping * @throws IOException */ private BlockDataMap loadAndGetDataMap(TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore, Map<String, BlockMetaInfo> blockMetaInfoMap, CarbonTable carbonTable, boolean addTableBlockToUnsafe, Configuration configuration) throws IOException, MemoryException { String uniqueTableSegmentIdentifier = identifier.getUniqueTableSegmentIdentifier(); Object lock = segmentLockMap.get(uniqueTableSegmentIdentifier); if (lock == null) { lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier); } BlockDataMap dataMap; synchronized (lock) { dataMap = (BlockDataMap) BlockletDataMapFactory.createDataMap(carbonTable); dataMap.init(new BlockletDataMapModel(carbonTable, identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), blockMetaInfoMap, identifier.getSegmentId(), addTableBlockToUnsafe, configuration)); } return dataMap; }
TableBlockIndexUniqueIdentifier identifier = identifierWrapper.getTableBlockIndexUniqueIdentifier(); String lruCacheKey = identifier.getUniqueTableSegmentIdentifier(); BlockletDataMapIndexWrapper blockletDataMapIndexWrapper = (BlockletDataMapIndexWrapper) lruCache.get(lruCacheKey); new SegmentIndexFileStore(identifierWrapper.getConfiguration()); Set<String> filesRead = new HashSet<>(); String segmentFilePath = identifier.getIndexFilePath(); if (segInfoCache == null) { segInfoCache = new HashMap<String, Map<String, BlockMetaInfo>>(); if (identifier.getMergeIndexFileName() == null) { Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletDataMapUtil .getBlockMetaInfoMap(identifierWrapper, indexFileStore, filesRead, dataMaps.add(blockletDataMap); blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(identifier.getSegmentId(), dataMaps); } else { new BlockletDataMapIndexWrapper(identifier.getSegmentId(), dataMaps); lruCache.put(identifier.getUniqueTableSegmentIdentifier(), blockletDataMapIndexWrapper, blockletDataMapIndexWrapper.getMemorySize());
identifierWrapper.getTableBlockIndexUniqueIdentifier(); List<ColumnSchema> tableColumnList = null; if (identifier.getMergeIndexFileName() != null && indexFileStore.getFileData(identifier.getIndexFileName()) == null) { CarbonFile indexMergeFile = FileFactory.getCarbonFile( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getMergeIndexFileName(), identifierWrapper.getConfiguration()); if (indexMergeFile.exists() && !filesRead.contains(indexMergeFile.getPath())) { indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { indexMergeFile }); if (indexFileStore.getFileData(identifier.getIndexFileName()) == null) { indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { FileFactory.getCarbonFile( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), identifierWrapper.getConfiguration()) }); new DataFileFooterConverter(identifierWrapper.getConfiguration()); List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), isTransactionalTable); for (DataFileFooter footer : indexInfo) { if ((!isTransactionalTable) && (tableColumnList.size() != 0) && !isSameColumnSchemaList(footer.getColumnInTable(), tableColumnList)) { LOG.error("Schema of " + identifier.getIndexFileName() + " doesn't match with the table's schema"); throw new IOException("All the files doesn't have same schema. "
for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) { String indexFilePath = tableBlockIndexUniqueIdentifier.getIndexFilePath(); String fileName = tableBlockIndexUniqueIdentifier.getIndexFileName(); carbonIndexFiles[identifierCounter++] = FileFactory .getCarbonFile(indexFilePath + CarbonCommonConstants.FILE_SEPARATOR + fileName);
/** * method invalidate the segment cache for segment * * @param tableSegmentUniqueIdentifierWrapper */ @Override public void invalidate( TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) { BlockletDataMapIndexWrapper blockletDataMapIndexWrapper = getIfPresent(tableSegmentUniqueIdentifierWrapper); if (null != blockletDataMapIndexWrapper) { // clear the segmentProperties cache List<BlockDataMap> dataMaps = blockletDataMapIndexWrapper.getDataMaps(); if (null != dataMaps && !dataMaps.isEmpty()) { String segmentId = tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getSegmentId(); // as segmentId will be same for all the dataMaps and segmentProperties cache is // maintained at segment level so it need to be called only once for clearing SegmentPropertiesAndSchemaHolder.getInstance() .invalidate(segmentId, dataMaps.get(0).getSegmentPropertiesIndex(), tableSegmentUniqueIdentifierWrapper.isAddTableBlockToUnsafeAndLRUCache()); } } lruCache.remove(tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier()); }
/** * returns the SegmentTaskIndexWrapper * * @param tableSegmentUniqueIdentifierWrapper * @return */ @Override public BlockletDataMapIndexWrapper getIfPresent( TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) { return (BlockletDataMapIndexWrapper) lruCache.get( tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier()); }
public static Set<TableBlockIndexUniqueIdentifier> getTableBlockUniqueIdentifiers(Segment segment) throws IOException { Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = new HashSet<>(); Map<String, String> indexFiles = segment.getCommittedIndexFile(); for (Map.Entry<String, String> indexFileEntry : indexFiles.entrySet()) { Path indexFile = new Path(indexFileEntry.getKey()); tableBlockIndexUniqueIdentifiers.add( new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(), indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo())); } return tableBlockIndexUniqueIdentifiers; }
/** * This method will filter out the TableBlockIndexUniqueIdentifier belongs to that distributable * * @param tableBlockIndexUniqueIdentifiers * @param distributable * @return */ public static TableBlockIndexUniqueIdentifier filterIdentifiersBasedOnDistributable( Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers, BlockletDataMapDistributable distributable) { TableBlockIndexUniqueIdentifier validIdentifier = null; String fileName = CarbonTablePath.DataFileUtil.getFileName(distributable.getFilePath()); for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) { if (fileName.equals(tableBlockIndexUniqueIdentifier.getIndexFileName())) { validIdentifier = tableBlockIndexUniqueIdentifier; break; } } return validIdentifier; }
/** * The method clears the access count of table segments * * @param tableSegmentUniqueIdentifiersWrapper */ @Override public void clearAccessCount( List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiersWrapper) { for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : tableSegmentUniqueIdentifiersWrapper) { BlockDataMap cacheable = (BlockDataMap) lruCache.get( identifierWrapper.getTableBlockIndexUniqueIdentifier().getUniqueTableSegmentIdentifier()); cacheable.clear(); } } }
String parent = indexPath.getParent().toString(); identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper( new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo), this.getCarbonTable())); } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) { for (String indexFile : indexFiles) { identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper( new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(), segmentNo), this.getCarbonTable()));
@Override public void put(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper wrapper) throws IOException, MemoryException { // As dataMap will use unsafe memory, it is not recommended to overwrite an existing entry // as in that case clearing unsafe memory need to be taken card. If at all datamap entry // in the cache need to be overwritten then use the invalidate interface // and then use the put interface if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) { List<BlockDataMap> dataMaps = wrapper.getDataMaps(); try { for (BlockDataMap blockletDataMap : dataMaps) { blockletDataMap.convertToUnsafeDMStore(); } // Locking is not required here because in LRU cache map add method is synchronized to add // only one entry at a time and if a key already exists it will not overwrite the entry lruCache.put(tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier(), wrapper, wrapper.getMemorySize()); } catch (Throwable e) { // clear all the memory acquired by data map in case of any failure for (DataMap blockletDataMap : dataMaps) { blockletDataMap.clear(); } throw new IOException("Problem in adding datamap to cache.", e); } } }