/** * This method will filter out the TableBlockIndexUniqueIdentifier belongs to that distributable * * @param tableBlockIndexUniqueIdentifiers * @param distributable * @return */ public static TableBlockIndexUniqueIdentifier filterIdentifiersBasedOnDistributable( Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers, BlockletDataMapDistributable distributable) { TableBlockIndexUniqueIdentifier validIdentifier = null; String fileName = CarbonTablePath.DataFileUtil.getFileName(distributable.getFilePath()); for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) { if (fileName.equals(tableBlockIndexUniqueIdentifier.getIndexFileName())) { validIdentifier = tableBlockIndexUniqueIdentifier; break; } } return validIdentifier; }
/** * This method will the index files tableBlockIndexUniqueIdentifiers of a merge index file * * @param identifier * @param segmentIndexFileStore * @return * @throws IOException */ public static List<TableBlockIndexUniqueIdentifier> getIndexFileIdentifiersFromMergeFile( TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore segmentIndexFileStore) throws IOException { List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = new ArrayList<>(); String mergeFilePath = identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(); segmentIndexFileStore.readMergeFile(mergeFilePath); List<String> indexFiles = segmentIndexFileStore.getCarbonMergeFileToIndexFilesMap().get(mergeFilePath); for (String indexFile : indexFiles) { tableBlockIndexUniqueIdentifiers.add( new TableBlockIndexUniqueIdentifier(identifier.getIndexFilePath(), indexFile, identifier.getIndexFileName(), identifier.getSegmentId())); } return tableBlockIndexUniqueIdentifiers; }
tableBlockIndexUniqueIdentifiers) { String indexFilePath = tableBlockIndexUniqueIdentifier.getIndexFilePath(); String fileName = tableBlockIndexUniqueIdentifier.getIndexFileName(); carbonIndexFiles[identifierCounter++] = FileFactory .getCarbonFile(indexFilePath + CarbonCommonConstants.FILE_SEPARATOR + fileName);
/** * Below method will be used to load the segment of segments * One segment may have multiple task , so table segment will be loaded * based on task id and will return the map of taksId to table segment * map * * @return map of taks id to segment mapping * @throws IOException */ private BlockDataMap loadAndGetDataMap(TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore, Map<String, BlockMetaInfo> blockMetaInfoMap, CarbonTable carbonTable, boolean addTableBlockToUnsafe, Configuration configuration) throws IOException, MemoryException { String uniqueTableSegmentIdentifier = identifier.getUniqueTableSegmentIdentifier(); Object lock = segmentLockMap.get(uniqueTableSegmentIdentifier); if (lock == null) { lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier); } BlockDataMap dataMap; synchronized (lock) { dataMap = (BlockDataMap) BlockletDataMapFactory.createDataMap(carbonTable); dataMap.init(new BlockletDataMapModel(carbonTable, identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), blockMetaInfoMap, identifier.getSegmentId(), addTableBlockToUnsafe, configuration)); } return dataMap; }
List<ColumnSchema> tableColumnList = null; if (identifier.getMergeIndexFileName() != null && indexFileStore.getFileData(identifier.getIndexFileName()) == null) { CarbonFile indexMergeFile = FileFactory.getCarbonFile( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier if (indexFileStore.getFileData(identifier.getIndexFileName()) == null) { indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { FileFactory.getCarbonFile( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), identifierWrapper.getConfiguration()) }); List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo( identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), isTransactionalTable); for (DataFileFooter footer : indexInfo) { if ((!isTransactionalTable) && (tableColumnList.size() != 0) && !isSameColumnSchemaList(footer.getColumnInTable(), tableColumnList)) { LOG.error("Schema of " + identifier.getIndexFileName() + " doesn't match with the table's schema"); throw new IOException("All the files doesn't have same schema. "