/** * Convert schema to binary */ public static byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutput dataOutput = new DataOutputStream(stream); dataOutput.writeShort(columnSchemas.size()); for (ColumnSchema columnSchema : columnSchemas) { if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchema.write(dataOutput); } byte[] byteArray = stream.toByteArray(); // Compress to reduce the size of schema return CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().compressByte( byteArray); }
DataType dataType = columnPageEncoderMeta.getStoreDataType(); if (dataType == DataTypes.BOOLEAN) { return compressor.compressByte(getBooleanPage()); } else if (dataType == DataTypes.BYTE) { return compressor.compressByte(getBytePage()); } else if (dataType == DataTypes.SHORT) { return compressor.compressShort(getShortPage()); } else if (dataType == DataTypes.SHORT_INT) { return compressor.compressByte(getShortIntPage()); } else if (dataType == DataTypes.INT) { return compressor.compressInt(getIntPage()); return compressor.compressDouble(getDoublePage()); } else if (DataTypes.isDecimal(dataType)) { return compressor.compressByte(getDecimalPage()); } else if (dataType == DataTypes.BYTE_ARRAY && columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_PRIMITIVE) { return compressor.compressByte(getComplexChildrenLVFlattenedBytePage()); } else if (dataType == DataTypes.BYTE_ARRAY && (columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_STRUCT || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_LONG_VALUE || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_VALUE)) { return compressor.compressByte(getComplexParentFlattenedBytePage()); } else if (dataType == DataTypes.BYTE_ARRAY) { return compressor.compressByte(getLVFlattenedBytePage()); } else { throw new UnsupportedOperationException("unsupport compress column page: " + dataType);
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage = new BlockIndexerStorageForShort(inputPage.getByteArrayPage(), false, false, false); byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); byte[] compressed = compressor.compressByte(flattened); super.indexStorage = indexStorage; super.compressedDataPage = compressed; }
void apppendBlocklet(DataOutputStream outputStream) throws IOException { outputStream.write(CarbonStreamOutputFormat.CARBON_SYNC_MARKER); BlockletInfo blockletInfo = new BlockletInfo(); blockletInfo.setNum_rows(getRowIndex() + 1); BlockletHeader blockletHeader = new BlockletHeader(); blockletHeader.setBlocklet_length(getCount()); blockletHeader.setMutation(MutationType.INSERT); blockletHeader.setBlocklet_info(blockletInfo); // add blocklet level min/max blockletMinMaxIndex = generateBlockletMinMax(); if (blockletInfo.getNum_rows() > 1) { BlockletIndex blockletIndex = new BlockletIndex(); blockletIndex.setMin_max_index(CarbonMetadataUtil.convertMinMaxIndex(blockletMinMaxIndex)); blockletHeader.setBlocklet_index(blockletIndex); } byte[] headerBytes = CarbonUtil.getByteArray(blockletHeader); outputStream.writeInt(headerBytes.length); outputStream.write(headerBytes); byte[] compressed = compressor.compressByte(getBytes(), getCount()); outputStream.writeInt(compressed.length); outputStream.write(compressed); }
CompressorFactory.getInstance().getCompressor(columnCompressor).compressByte( usedDictionaryValues.toByteArray()));
private void fillNullBitSet(ColumnPage inputPage, DataChunk2 dataChunk) { PresenceMeta presenceMeta = new PresenceMeta(); presenceMeta.setPresent_bit_streamIsSet(true); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); presenceMeta.setPresent_bit_stream( compressor.compressByte(inputPage.getNullBits().toByteArray())); dataChunk.setPresence(presenceMeta); }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, true, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, false, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override protected void encodeIndexStorage(ColumnPage input) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = input.getByteArrayPage(); boolean isDictionary = input.isLocalDictGeneratedPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, isDictionary, !isDictionary, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, isDictionary); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }