@Override public void freeMemory() { byteArrayData = null; super.freeMemory(); }
/** * Create a new column page based on the LV (Length Value) encoded bytes */ static ColumnPage newComplexLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, int lvLength, String compressorName) throws MemoryException { return getComplexLVBytesColumnPage(columnSpec, lvEncodedBytes, DataTypes.BYTE_ARRAY, lvLength, compressorName); }
/** * Create a new column page based on the LV (Length Value) encoded bytes */ static ColumnPage newLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, int lvLength, String compressorName) throws MemoryException { return getLVBytesColumnPage(columnSpec, lvEncodedBytes, DataTypes.BYTE_ARRAY, lvLength, compressorName); }
/** * Create a new column page for decimal page */ public static ColumnPage newDecimalColumnPage(ColumnPageEncoderMeta meta, byte[] lvEncodedBytes, int actualDataLength) throws MemoryException { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); int size = decimalConverter.getSize(); if (size < 0) { return getLVBytesColumnPage(columnSpec, lvEncodedBytes, DataTypes.createDecimalType(columnSpec.getPrecision(), columnSpec.getScale()), CarbonCommonConstants.INT_SIZE_IN_BYTE, meta.getCompressorName()); } else { // Here the size is always fixed. return getDecimalColumnPage(meta, lvEncodedBytes, size, actualDataLength); } }
if (isUnsafeEnabled(meta)) { page = new UnsafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), page.rowOffset = rowOffset; for (int i = 0; i < rowId; i++) { page.putBytes(i, lvEncodedBytes, i * size, size);
@Override public byte[] getDecimalPage() { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < pageSize; rowId++) { int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); copyBytes(rowId, data, offset, length); offset += length; } return data; }
private static ColumnPage newDecimalPage(ColumnPageEncoderMeta meta, byte[] lvEncodedByteArray) throws MemoryException { return VarLengthColumnPageBase .newDecimalColumnPage(meta, lvEncodedByteArray, lvEncodedByteArray.length); }
private static ColumnPage getComplexLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
private static ColumnPage newLVBytesPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedByteArray, int lvLength, String compressorName) throws MemoryException { return VarLengthColumnPageBase.newLVBytesColumnPage( columnSpec, lvEncodedByteArray, lvLength, compressorName); }
private static ColumnPage newComplexLVBytesPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedByteArray, int lvLength, String compressorName) throws MemoryException { return VarLengthColumnPageBase.newComplexLVBytesColumnPage( columnSpec, lvEncodedByteArray, lvLength, compressorName); }
@Override public byte[] getComplexParentFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); copyBytes(rowId, data, offset, length); offset += length; } return data; }
if (DataTypes.isDecimal(meta.getStoreDataType())) { ColumnPage decimalColumnPage = VarLengthColumnPageBase .newDecimalColumnPage(meta, unCompressData, uncompressedLength); decimalConverter.fillVector(decimalColumnPage.getByteArrayPage(), pageSize, vectorInfo, nullBits, meta.getStoreDataType());
private static ColumnPage getLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
@Override public byte[] getComplexChildrenLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 2)]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { short length = (short) (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); ByteUtil.setShort(data, offset, length); copyBytes(rowId, data, offset + 2, length); offset += 2 + length; } return data; }
@Override public void freeMemory() { if (memoryBlock != null) { UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock); memoryBlock = null; baseAddress = null; baseOffset = 0; super.freeMemory(); } }
@Override public byte[] getLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 4)]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); ByteUtil.setInt(data, offset, length); copyBytes(rowId, data, offset + 4, length); offset += 4 + length; } return data; }