public static CompressedVSizeColumnarIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ); }
@Override public void addValues(IndexedInts ints) throws IOException { if (lastOffsetWritten) { throw new IllegalStateException("written out already"); } offsetWriter.addValue(offset); int numValues = ints.size(); for (int i = 0; i < numValues; i++) { valueWriter.addValue(ints.get(i)); } offset += numValues; }
@Override public long getSerializedSize() throws IOException { writeLastOffset(); return 1 + offsetWriter.getSerializedSize() + valueWriter.getSerializedSize(); }
CompressedVSizeColumnarIntsSerializer writer = new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, "test", segmentWriteOutMedium.getCloser() ); writer.open(); for (int val : vals) { writer.addValue(val); long writtenLength = writer.getSerializedSize(); final WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); writer.writeTo(writeOutBytes, smoosher); smoosher.close();
@Override public void open() throws IOException { offsetWriter.open(); valueWriter.open(); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writeLastOffset(); Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{VERSION})); offsetWriter.writeTo(channel, smoosher); valueWriter.writeTo(channel, smoosher); }
@Override public long getSerializedSize() throws IOException { writeEndBuffer(); return metaSerdeHelper.size(this) + flattener.getSerializedSize(); }
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
filenameBase, compression, sizePer(maxValue, chunkFactor)
Long.BYTES * 10000 ); CompressedVSizeColumnarIntsSerializer writer = new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, vals.length > 0 ? Ints.max(vals) : 0, genericIndexed ); writer.open(); for (int val : vals) { writer.addValue(val); writer.getSerializedSize() ); writer.writeTo(channel, smoosher); channel.close(); smoosher.close();
@Override public void open() throws IOException { offsetWriter.open(); valueWriter.open(); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writeLastOffset(); Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{VERSION})); offsetWriter.writeTo(channel, smoosher); valueWriter.writeTo(channel, smoosher); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writeEndBuffer(); metaSerdeHelper.writeTo(channel, this); flattener.writeTo(channel, smoosher); }
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
filenameBase, compression, sizePer(maxValue, chunkFactor)
public static V3CompressedVSizeColumnarMultiIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new V3CompressedVSizeColumnarMultiIntsSerializer( new CompressedColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, CompressedColumnarIntsSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ) ); }
@Override public long getSerializedSize() throws IOException { writeLastOffset(); return 1 + offsetWriter.getSerializedSize() + valueWriter.getSerializedSize(); }
@Override public void addValues(IndexedInts ints) throws IOException { if (lastOffsetWritten) { throw new IllegalStateException("written out already"); } offsetWriter.addValue(offset); int numValues = ints.size(); for (int i = 0; i < numValues; i++) { valueWriter.addValue(ints.get(i)); } offset += numValues; }
@Override public long getSerializedSize() throws IOException { writeEndBuffer(); return metaSerdeHelper.size(this) + flattener.getSerializedSize(); }
compressionStrategy ); CompressedVSizeColumnarIntsSerializer valueWriter = new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, "value",