private void testSupplierSerde(CompressedColumnarLongsSupplier supplier, long[] vals) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); supplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(supplier.getSerializedSize(), bytes.length); CompressedColumnarLongsSupplier anotherSupplier = CompressedColumnarLongsSupplier.fromByteBuffer( ByteBuffer.wrap(bytes), order ); ColumnarLongs indexed = anotherSupplier.get(); assertIndexMatchesVals(indexed, vals); }
compression ); return new CompressedColumnarLongsSupplier( totalSize, sizePer,
@Setup public void setup() throws Exception { File dir = new File(dirPath); File compFile = new File(dir, file + "-" + strategy + "-" + format); bufferHandler = FileUtils.map(compFile); ByteBuffer buffer = bufferHandler.get(); supplier = CompressedColumnarLongsSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder()); }
public void testValues(long[] values) throws Exception { ColumnarLongsSerializer serializer = CompressionFactory.getLongSerializer( new OffHeapMemorySegmentWriteOutMedium(), "test", order, encodingStrategy, compressionStrategy ); serializer.open(); for (long value : values) { serializer.add(value); } Assert.assertEquals(values.length, serializer.size()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.writeTo(Channels.newChannel(baos), null); Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedColumnarLongsSupplier supplier = CompressedColumnarLongsSupplier.fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order); ColumnarLongs longs = supplier.get(); assertIndexMatchesVals(longs, values); longs.close(); }
@Override public NumericColumn get() { return LongsColumn.create(column.get(), nullValueBitmap); } }
@Override public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(new LongNumericColumnSupplier(column, bitmap)); }; } }
Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedColumnarLongsSupplier supplier = CompressedColumnarLongsSupplier .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order); ColumnarLongs longs = supplier.get();
@Override public NumericColumn get() { return LongsColumn.create(column.get(), nullValueBitmap); } }
@Override public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) { final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); LongNumericColumnSupplier columnSupplier = new LongNumericColumnSupplier( column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(columnSupplier); } };
compression ); return new CompressedColumnarLongsSupplier( totalSize, sizePer,
final BitmapSerdeFactory bitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory(); CompressedColumnarLongsSupplier timestamps = CompressedColumnarLongsSupplier.fromByteBuffer( smooshedFiles.mapFile(makeTimeFile(inDir, BYTE_ORDER).getName()), BYTE_ORDER
@Override public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(new LongNumericColumnSupplier(column, bitmap)); }; } }
@Override public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) { final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); LongNumericColumnSupplier columnSupplier = new LongNumericColumnSupplier( column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(columnSupplier); } };
final BitmapSerdeFactory bitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory(); CompressedColumnarLongsSupplier timestamps = CompressedColumnarLongsSupplier.fromByteBuffer( smooshedFiles.mapFile(makeTimeFile(inDir, BYTE_ORDER).getName()), BYTE_ORDER