@Override public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { int blockIndex = measureRawColumnChunk.getColumnIndex(); DataChunk dataChunk = measureColumnChunks.get(blockIndex); ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0); ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta, CompressorFactory.NativeSupportedCompressor.SNAPPY.getName()); ColumnPage decodedPage = codec.decode(measureRawColumnChunk.getRawData().array(), (int) measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength()); decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn()); return decodedPage; } }
/** * Read column schema from binary * * @param schemaArray * @throws IOException */ public static List<ColumnSchema> readColumnSchema(byte[] schemaArray) throws IOException { // uncompress it. schemaArray = CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().unCompressByte( schemaArray); ByteArrayInputStream schemaStream = new ByteArrayInputStream(schemaArray); DataInput schemaInput = new DataInputStream(schemaStream); List<ColumnSchema> columnSchemas = new ArrayList<>(); int size = schemaInput.readShort(); for (int i = 0; i < size; i++) { ColumnSchema columnSchema = new ColumnSchema(); columnSchema.readFields(schemaInput); columnSchemas.add(columnSchema); } return columnSchemas; }
private CompressorFactory() { for (NativeSupportedCompressor nativeSupportedCompressor : NativeSupportedCompressor.values()) { allSupportedCompressors.put(nativeSupportedCompressor.getName(), nativeSupportedCompressor.getCompressor()); } }
compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName();
/** * Convert schema to binary */ public static byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutput dataOutput = new DataOutputStream(stream); dataOutput.writeShort(columnSchemas.size()); for (ColumnSchema columnSchema : columnSchemas) { if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchema.write(dataOutput); } byte[] byteArray = stream.toByteArray(); // Compress to reduce the size of schema return CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().compressByte( byteArray); }
/** * Constructor to get minimum parameter to create instance of this class * * @param blockletInfo blocklet info * @param eachColumnValueSize size of the each column value * @param filePath file from which data will be read */ public CompressedDimensionChunkFileBasedReaderV1(final BlockletInfo blockletInfo, final int[] eachColumnValueSize, final String filePath) { super(eachColumnValueSize, filePath, blockletInfo.getNumberOfRows()); this.dimensionColumnChunk = blockletInfo.getDimensionColumnChunk(); // for v1 store, the compressor is snappy this.compressor = CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor(); }
protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk, DataChunk2 measureColumnChunk, int copyPoint, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { assert (measureColumnChunk.getEncoder_meta().size() > 0); List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta(); byte[] encodedMeta = encoder_meta.get(0).array(); ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta); ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta, CompressorFactory.NativeSupportedCompressor.SNAPPY.getName()); byte[] rawData = measureRawColumnChunk.getRawData().array(); return codec .decode(rawData, copyPoint, measureColumnChunk.data_page_length); } }
/** * Constructor to get minimum parameter to create instance of this class * * @param blockletInfo BlockletInfo * @param filePath file from which data will be read */ public CompressedMeasureChunkFileBasedReaderV2(final BlockletInfo blockletInfo, final String filePath) { super(blockletInfo, filePath); this.compressor = CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor(); }
/** * Constructor to get minimum parameter to create instance of this class * * @param blockletInfo * @param eachColumnValueSize * @param filePath */ public CompressedDimensionChunkFileBasedReaderV2(final BlockletInfo blockletInfo, final int[] eachColumnValueSize, final String filePath) { super(blockletInfo, eachColumnValueSize, filePath); // for v2 store, the compressor is snappy this.compressor = CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor(); }
private byte[] getSyncMarker(String filePath) throws IOException { CarbonHeaderReader headerReader = new CarbonHeaderReader(filePath); FileHeader header = headerReader.readHeader(); // legacy store does not have this member if (header.isSetCompressor_name()) { compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName(); } return header.getSync_marker(); }
/** * get the compressor name from chunk meta * before 1.5.0, we only support snappy and do not have compressor_name field; * after 1.5.0, we directly get the compressor from the compressor_name field */ public static String getCompressorNameFromChunkMeta(ChunkCompressionMeta chunkCompressionMeta) { if (chunkCompressionMeta.isSetCompressor_name()) { return chunkCompressionMeta.getCompressor_name(); } else { // this is for legacy store before 1.5.0 return CompressorFactory.NativeSupportedCompressor.SNAPPY.getName(); } } /**