public static V3CompressedVSizeColumnarMultiIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new V3CompressedVSizeColumnarMultiIntsSerializer( new CompressedColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, CompressedColumnarIntsSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ) ); }
@Override public long getSerializedSize() throws IOException { writeLastOffset(); return 1 + offsetWriter.getSerializedSize() + valueWriter.getSerializedSize(); }
); V3CompressedVSizeColumnarMultiIntsSerializer writer = new V3CompressedVSizeColumnarMultiIntsSerializer(offsetWriter, valueWriter); V3CompressedVSizeColumnarMultiIntsSupplier supplierFromIterable = V3CompressedVSizeColumnarMultiIntsSupplier.fromIterable( segmentWriteOutMedium.getCloser() ); writer.open(); for (int[] val : vals) { writer.addValues(new ArrayBasedIndexedInts(val)); long writtenLength = writer.getSerializedSize(); final WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); writer.writeTo(writeOutBytes, smoosher); smoosher.close();
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
); V3CompressedVSizeColumnarMultiIntsSerializer writer = new V3CompressedVSizeColumnarMultiIntsSerializer(offsetWriter, valueWriter); writer.open(); for (int[] val : vals) { writer.addValues(new ArrayBasedIndexedInts(val)); final SmooshedWriter channel = smoosher.addWithSmooshedWriter("test", writer.getSerializedSize()); writer.writeTo(channel, smoosher); channel.close(); smoosher.close();
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
public static V3CompressedVSizeColumnarMultiIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new V3CompressedVSizeColumnarMultiIntsSerializer( new CompressedColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, CompressedColumnarIntsSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ) ); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writeLastOffset(); Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{VERSION})); offsetWriter.writeTo(channel, smoosher); valueWriter.writeTo(channel, smoosher); }
@Override public long getSerializedSize() throws IOException { writeLastOffset(); return 1 + offsetWriter.getSerializedSize() + valueWriter.getSerializedSize(); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writeLastOffset(); Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{VERSION})); offsetWriter.writeTo(channel, smoosher); valueWriter.writeTo(channel, smoosher); }