public BlockLayoutColumnarDoublesSupplier( int totalSize, int sizePer, ByteBuffer fromBuffer, ByteOrder byteOrder, CompressionStrategy strategy ) { baseDoubleBuffers = GenericIndexed.read(fromBuffer, new DecompressingByteBufferObjectStrategy(byteOrder, strategy)); this.totalSize = totalSize; this.sizePer = sizePer; }
private static <T> GenericIndexed<T> read(ByteBuffer buf, ComplexMetricSerde serde) { return GenericIndexed.read(buf, serde.getObjectStrategy()); }
public BlockLayoutColumnarFloatsSupplier( int totalSize, int sizePer, ByteBuffer fromBuffer, ByteOrder byteOrder, CompressionStrategy strategy ) { baseFloatBuffers = GenericIndexed.read(fromBuffer, new DecompressingByteBufferObjectStrategy(byteOrder, strategy)); this.totalSize = totalSize; this.sizePer = sizePer; }
public BlockLayoutColumnarLongsSupplier( int totalSize, int sizePer, ByteBuffer fromBuffer, ByteOrder order, CompressionFactory.LongEncodingReader reader, CompressionStrategy strategy ) { baseLongBuffers = GenericIndexed.read(fromBuffer, new DecompressingByteBufferObjectStrategy(order, strategy)); this.totalSize = totalSize; this.sizePer = sizePer; this.baseReader = reader; }
public static CompressedColumnarIntsSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedColumnarIntsSupplier( totalSize, sizePer, GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
@Override public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) { final GenericIndexed<ArrayOfDoublesSketch> ge = GenericIndexed.read(buffer, ArrayOfDoublesSketchObjectStrategy.STRATEGY); builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge)); }
public static CompressedVSizeColumnarIntsSupplier fromByteBuffer( ByteBuffer buffer, ByteOrder order ) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { final int numBytes = buffer.get(); final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedVSizeColumnarIntsSupplier( totalSize, sizePer, numBytes, GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
@Setup(Level.Trial) public void createGenericIndexed() throws IOException { GenericIndexedWriter<byte[]> genericIndexedWriter = new GenericIndexedWriter<>( new OffHeapMemorySegmentWriteOutMedium(), "genericIndexedBenchmark", byteArrayStrategy ); genericIndexedWriter.open(); // GenericIndexObject caches prevObject for comparison, so need two arrays for correct objectsSorted computation. ByteBuffer[] elements = new ByteBuffer[2]; elements[0] = ByteBuffer.allocate(elementSize); elements[1] = ByteBuffer.allocate(elementSize); for (int i = 0; i < n; i++) { ByteBuffer element = elements[i & 1]; element.putInt(0, i); genericIndexedWriter.write(element.array()); } smooshDir = Files.createTempDir(); file = File.createTempFile("genericIndexedBenchmark", "meta"); try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); FileSmoosher fileSmoosher = new FileSmoosher(smooshDir)) { genericIndexedWriter.writeTo(fileChannel, fileSmoosher); } FileChannel fileChannel = FileChannel.open(file.toPath()); MappedByteBuffer byteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, file.length()); genericIndexed = GenericIndexed.read(byteBuffer, byteArrayStrategy, SmooshedFileMapper.load(smooshDir)); }
@Override public void deserializeColumn(final ByteBuffer buf, final ColumnBuilder columnBuilder) { columnBuilder.setComplexColumnSupplier( new ComplexColumnPartSupplier( getTypeName(), GenericIndexed.read(buf, HllSketchObjectStrategy.STRATEGY, columnBuilder.getFileMapper()) ) ); }
@Override public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) { GenericIndexed<SketchHolder> ge = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge)); }
@Override public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) { final GenericIndexed<DoublesSketch> column = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
private GenericIndexed<String> serializeAndDeserialize(GenericIndexed<String> indexed) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); final WritableByteChannel channel = Channels.newChannel(baos); indexed.writeTo(channel, null); channel.close(); final ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray()); Assert.assertEquals(indexed.getSerializedSize(), byteBuffer.remaining()); GenericIndexed<String> deserialized = GenericIndexed.read(byteBuffer, GenericIndexed.STRING_STRATEGY); Assert.assertEquals(0, byteBuffer.remaining()); return deserialized; } }
@Override public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { final GenericIndexed<ApproximateHistogram> column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
@Override public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) { final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), builder.getFileMapper()); builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
@Override public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
@Override public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
@Override public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) { final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }
@Override public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { final GenericIndexed column; if (columnBuilder.getFileMapper() == null) { column = GenericIndexed.read(byteBuffer, getObjectStrategy()); } else { column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); } columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); }