/** * Below method will be used to get the index info from index file * * @param filePath file path of the index file * @return list of index info * @throws IOException problem while reading the index file */ public List<DataFileFooter> getIndexInfo(String filePath, byte[] fileData) throws IOException { return getIndexInfo(filePath, fileData, true); }
readIndexHeader.getTable_columns(); for (int i = 0; i < table_columns.size(); i++) { columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i))); SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info()); BlockletIndex blockletIndex = null; int counter = 0; blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index()); dataFileFooter = new DataFileFooter(); TableBlockInfo tableBlockInfo = tableBlockInfoList.get(index); tableBlockInfo.setVersion( ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion())); int blockletSize = getBlockletSize(readBlockIndexInfo); tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize); dataFileFooter.setBlockletIndex(blockletIndex);
readIndexHeader.getTable_columns(); for (int i = 0; i < table_columns.size(); i++) { columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i))); SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info()); BlockletIndex blockletIndex = null; DataFileFooter dataFileFooter = null; blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index()); dataFileFooter = new DataFileFooter(); TableBlockInfo tableBlockInfo = getTableBlockInfo(readBlockIndexInfo, readIndexHeader, parentPath); dataFileFooter.setBlockletIndex(blockletIndex); dataFileFooter.setColumnInTable(columnSchemaList);
dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset()); if (isPresenceMetaPresent) { dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence())); List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size()); for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) { encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
private static DataFileFooter getDataFileFooter(TableBlockInfo tableBlockInfo, boolean forceReadDataFileFooter) throws IOException { BlockletDetailInfo detailInfo = tableBlockInfo.getDetailInfo(); if (detailInfo == null || forceReadDataFileFooter) { AbstractDataFileFooterConverter fileFooterConverter = DataFileFooterConverterFactory.getInstance() .getDataFileFooterConverter(tableBlockInfo.getVersion()); return fileFooterConverter.readDataFileFooter(tableBlockInfo); } else { DataFileFooter fileFooter = new DataFileFooter(); fileFooter.setSchemaUpdatedTimeStamp(detailInfo.getSchemaUpdatedTimeStamp()); ColumnarFormatVersion version = ColumnarFormatVersion.valueOf(detailInfo.getVersionNumber()); AbstractDataFileFooterConverter dataFileFooterConverter = DataFileFooterConverterFactory.getInstance().getDataFileFooterConverter(version); List<ColumnSchema> schema = dataFileFooterConverter.getSchema(tableBlockInfo); fileFooter.setColumnInTable(schema); SegmentInfo segmentInfo = new SegmentInfo(); segmentInfo.setColumnCardinality(detailInfo.getDimLens()); fileFooter.setSegmentInfo(segmentInfo); return fileFooter; } }
List<Encoding> encoders = new ArrayList<Encoding>(); for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) { encoders.add(fromExternalToWrapperEncoding(encoder)); if (null != parentColumnTableRelation) { wrapperColumnSchema.setParentColumnTableRelations( fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation));
/** * This method will create a table block info object from index file info * * @param readBlockIndexInfo * @param readIndexHeader * @param parentPath * @return */ public TableBlockInfo getTableBlockInfo(BlockIndex readBlockIndexInfo, org.apache.carbondata.format.IndexHeader readIndexHeader, String parentPath) { TableBlockInfo tableBlockInfo = new TableBlockInfo(); tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset()); ColumnarFormatVersion version = ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion()); tableBlockInfo.setVersion(version); int blockletSize = getBlockletSize(readBlockIndexInfo); tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize); String fileName = readBlockIndexInfo.file_name; // Take only name of file. if (fileName.lastIndexOf("/") > 0) { fileName = fileName.substring(fileName.lastIndexOf("/")); } fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/"); tableBlockInfo.setFilePath(parentPath + fileName); return tableBlockInfo; }