CompressedColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.chunkFactor = chunkFactor; this.compression = compression; this.flattener = flattener; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(chunkFactor * Integer.BYTES, closer).order(byteOrder); this.numInserted = 0; }
private final CompressionStrategy.Compressor compressor = compressionStrategy.getCompressor(); private final ByteBuffer compressedDataBuffer = compressor.allocateOutBuffer(bufferSize, closer);
CompressedVSizeColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.numBytes = VSizeColumnarInts.getNumBytesForMax(maxValue); this.chunkFactor = chunkFactor; int chunkBytes = chunkFactor * numBytes; this.isBigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); this.compression = compression; this.flattener = flattener; this.intBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); CompressionStrategy.Compressor compressor = compression.getCompressor(); this.endBuffer = compressor.allocateInBuffer(chunkBytes, segmentWriteOutMedium.getCloser()).order(byteOrder); this.numInserted = 0; }
BlockLayoutColumnarFloatsSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionStrategy compression ) { this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( segmentWriteOutMedium, filenameBase, compression, CompressedPools.BUFFER_SIZE ); this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); }
BlockLayoutColumnarDoublesSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionStrategy compression ) { this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( segmentWriteOutMedium, filenameBase, compression, CompressedPools.BUFFER_SIZE ); this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); }
@Override public Boolean call() { ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocate(originalData.length); compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); return true; } }
@Test public void testDirectMemoryOperations() { ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocateDirect(originalData.length); compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); }
@Test public void testBasicOperations() { ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocate(originalData.length); compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); }
BlockLayoutColumnarLongsSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionFactory.LongEncodingWriter writer, CompressionStrategy compression ) { this.sizePer = writer.getBlockSize(CompressedPools.BUFFER_SIZE); int bufferSize = writer.getNumBytes(sizePer); this.flattener = GenericIndexedWriter.ofCompressedByteBuffers(segmentWriteOutMedium, filenameBase, compression, bufferSize); this.writer = writer; this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); endBuffer = compressor.allocateInBuffer(writer.getNumBytes(sizePer), segmentWriteOutMedium.getCloser()).order(byteOrder); writer.setBuffer(endBuffer); numInsertedForNextFlush = sizePer; }
CompressedColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.chunkFactor = chunkFactor; this.compression = compression; this.flattener = flattener; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(chunkFactor * Integer.BYTES, closer).order(byteOrder); this.numInserted = 0; }
private final CompressionStrategy.Compressor compressor = compressionStrategy.getCompressor(); private final ByteBuffer compressedDataBuffer = compressor.allocateOutBuffer(bufferSize, closer);
CompressedVSizeColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.numBytes = VSizeColumnarInts.getNumBytesForMax(maxValue); this.chunkFactor = chunkFactor; int chunkBytes = chunkFactor * numBytes; this.isBigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); this.compression = compression; this.flattener = flattener; this.intBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); CompressionStrategy.Compressor compressor = compression.getCompressor(); this.endBuffer = compressor.allocateInBuffer(chunkBytes, segmentWriteOutMedium.getCloser()).order(byteOrder); this.numInserted = 0; }
BlockLayoutColumnarDoublesSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionStrategy compression ) { this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( segmentWriteOutMedium, filenameBase, compression, CompressedPools.BUFFER_SIZE ); this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); }
BlockLayoutColumnarFloatsSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionStrategy compression ) { this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( segmentWriteOutMedium, filenameBase, compression, CompressedPools.BUFFER_SIZE ); this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); }
BlockLayoutColumnarLongsSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionFactory.LongEncodingWriter writer, CompressionStrategy compression ) { this.sizePer = writer.getBlockSize(CompressedPools.BUFFER_SIZE); int bufferSize = writer.getNumBytes(sizePer); this.flattener = GenericIndexedWriter.ofCompressedByteBuffers(segmentWriteOutMedium, filenameBase, compression, bufferSize); this.writer = writer; this.compression = compression; CompressionStrategy.Compressor compressor = compression.getCompressor(); endBuffer = compressor.allocateInBuffer(writer.getNumBytes(sizePer), segmentWriteOutMedium.getCloser()).order(byteOrder); writer.setBuffer(endBuffer); numInsertedForNextFlush = sizePer; }