@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); return input.compress(compressor); }
@Override public byte[] compress(Compressor compressor) throws MemoryException, IOException { if (UnsafeMemoryManager.isOffHeap() && compressor.supportUnsafe()) { // use raw compression and copy to byte[] int inputSize = totalLength; long compressedMaxSize = compressor.maxCompressedLength(inputSize); MemoryBlock compressed = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, compressedMaxSize); long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset()); assert outSize < Integer.MAX_VALUE; byte[] output = new byte[(int) outSize]; CarbonUnsafe.getUnsafe() .copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output, CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize); UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed); return output; } else { return super.compress(compressor); } }
public byte[] encodeAndCompressPage(ColumnPage input, ColumnPageValueConverter converter, Compressor compressor) throws MemoryException, IOException { encodedPage = ColumnPage.newPage( new ColumnPageEncoderMeta(input.getColumnPageEncoderMeta().getColumnSpec(), targetDataType, input.getColumnPageEncoderMeta().getCompressorName()), input.getPageSize()); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForNoDictionary(getPageBasedOnDataType(input), input.getDataType(), isInvertedIndex); } ColumnPage columnPage = getSortedColumnPageIfRequired(input); columnPage.convertValue(converter); byte[] result = encodedPage.compress(compressor); return result; }