@Override public Boolean call() { ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocate(originalData.length); compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); return true; } }
public static CompressionStrategy[] noNoneValues() { return (CompressionStrategy[]) ArrayUtils.removeElement(CompressionStrategy.values(), NONE); }
public static Supplier<ColumnarDoubles> fromByteBuffer( ByteBuffer buffer, ByteOrder order ) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == LZF_VERSION || versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); CompressionStrategy compression = CompressionStrategy.LZF; if (versionFromBuffer == VERSION) { byte compressionId = buffer.get(); compression = CompressionStrategy.forId(compressionId); } return CompressionFactory.getDoubleSupplier( totalSize, sizePer, buffer.asReadOnlyBuffer(), order, compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); } }
CompressedColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.chunkFactor = chunkFactor; this.compression = compression; this.flattener = flattener; CompressionStrategy.Compressor compressor = compression.getCompressor(); Closer closer = segmentWriteOutMedium.getCloser(); this.endBuffer = compressor.allocateInBuffer(chunkFactor * Integer.BYTES, closer).order(byteOrder); this.numInserted = 0; }
DecompressingByteBufferObjectStrategy(ByteOrder order, CompressionStrategy compression) { this.order = order; this.decompressor = compression.getDecompressor(); }
@Override public void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) { metaOut.put(strategy.getId()); }
@Parameterized.Parameters public static Iterable<Object[]> compressionStrategies() { return Iterables.transform( Arrays.asList(CompressionStrategy.noNoneValues()), new Function<CompressionStrategy, Object[]>() { @Override public Object[] apply(CompressionStrategy compressionStrategy) { return new Object[]{compressionStrategy}; } } ); }
@JsonCreator public static CompressionStrategy fromString(String name) { return valueOf(StringUtils.toUpperCase(name)); }
@JsonValue @Override public String toString() { return StringUtils.toLowerCase(this.name()); }
private final CompressionStrategy.Compressor compressor = compressionStrategy.getCompressor(); private final ByteBuffer compressedDataBuffer = compressor.allocateOutBuffer(bufferSize, closer);
DecompressingByteBufferObjectStrategy(ByteOrder order, CompressionStrategy compression) { this.order = order; this.decompressor = compression.getDecompressor(); }
@Override public void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) { metaOut.put(CompressionFactory.setEncodingFlag(strategy.getId())); metaOut.put(CompressionFactory.LongEncodingFormat.DELTA.getId()); metaOut.put(CompressionFactory.DELTA_ENCODING_VERSION); metaOut.putLong(base); metaOut.putInt(bitsPerValue); }
@Parameterized.Parameters(name = "{index}: compression={0}, byteOrder={1}") public static Iterable<Object[]> compressionStrategiesAndByteOrders() { Set<List<Object>> combinations = Sets.cartesianProduct( Sets.newHashSet(CompressionStrategy.noNoneValues()), Sets.newHashSet(ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN) ); return Iterables.transform( combinations, (Function<List, Object[]>) input -> new Object[]{input.get(0), input.get(1)} ); }
@JsonCreator public static CompressionStrategy fromString(String name) { return valueOf(StringUtils.toUpperCase(name)); }
@JsonValue @Override public String toString() { return StringUtils.toLowerCase(this.name()); }
@Test public void testDirectMemoryOperations() { ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocateDirect(originalData.length); compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); }
CompressedVSizeColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression, final GenericIndexedWriter<ByteBuffer> flattener ) { this.numBytes = VSizeColumnarInts.getNumBytesForMax(maxValue); this.chunkFactor = chunkFactor; int chunkBytes = chunkFactor * numBytes; this.isBigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); this.compression = compression; this.flattener = flattener; this.intBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); CompressionStrategy.Compressor compressor = compression.getCompressor(); this.endBuffer = compressor.allocateInBuffer(chunkBytes, segmentWriteOutMedium.getCloser()).order(byteOrder); this.numInserted = 0; }
@Parameterized.Parameters(name = "{0} {1} {2}") public static Iterable<Object[]> compressionStrategies() { List<Object[]> data = new ArrayList<>(); for (CompressionStrategy strategy : CompressionStrategy.values()) { data.add(new Object[]{strategy, ByteOrder.BIG_ENDIAN}); data.add(new Object[]{strategy, ByteOrder.LITTLE_ENDIAN}); } return data; }