private static ColumnPage getComplexLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
private static ColumnPage getLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }