private void makeWithSerde(final int chunkSize) throws IOException { CloseQuietly.close(columnarInts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); final CompressedVSizeColumnarIntsSupplier theSupplier = CompressedVSizeColumnarIntsSupplier.fromList( IntArrayList.wrap(vals), Ints.max(vals), chunkSize, byteOrder, compressionStrategy, closer ); theSupplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); supplier = CompressedVSizeColumnarIntsSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), byteOrder); columnarInts = supplier.get(); }
CompressedVSizeColumnarIntsSupplier.fromList( IntArrayList.wrap(vals), bound - 1, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForBytes(bytes), ByteOrder.nativeOrder(), CompressionStrategy.LZ4, this.compressed = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( bufferCompressed, ByteOrder.nativeOrder() ).get();
CompressedVSizeColumnarIntsSupplier headerSupplier = CompressedVSizeColumnarIntsSupplier.fromList( offsetList, offsetMax, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(offsetMax), byteOrder, compression, closer ); CompressedVSizeColumnarIntsSupplier valuesSupplier = CompressedVSizeColumnarIntsSupplier.fromList( values, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), byteOrder, compression,
private void setupSimple(final int chunkSize) { CloseQuietly.close(columnarInts); vals = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16}; supplier = CompressedVSizeColumnarIntsSupplier.fromList( IntArrayList.wrap(vals), Ints.max(vals), chunkSize, ByteOrder.nativeOrder(), compressionStrategy, closer ); columnarInts = supplier.get(); }
public static CompressedVSizeColumnarMultiIntsSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == version) { CompressedVSizeColumnarIntsSupplier offsetSupplier = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( buffer, order ); CompressedVSizeColumnarIntsSupplier valueSupplier = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( buffer, order ); return new CompressedVSizeColumnarMultiIntsSupplier(offsetSupplier, valueSupplier); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
public static CompressedVSizeColumnarIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ); }
CompressedVSizeColumnarIntsSupplier supplierFromByteBuffer = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( mapper.mapFile("test"), byteOrder ); ColumnarInts columnarInts = supplierFromByteBuffer.get(); for (int i = 0; i < vals.length; ++i) { assertEquals(vals[i], columnarInts.get(i));
chunkFactor <= maxIntsInBufferForBytes(numBytes), "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor ); return new CompressedVSizeColumnarIntsSupplier( list.size(), chunkFactor,
public static int maxIntsInBufferForValue(int maxValue) { return maxIntsInBufferForBytes(VSizeColumnarInts.getNumBytesForMax(maxValue)); }
@Override public long getSerializedSize() { return 1 + offsetSupplier.getSerializedSize() + valueSupplier.getSerializedSize(); }
@Test public void testLargeChunks() throws Exception { for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); setupLargeChunks(maxChunkSize, 10 * maxChunkSize, maxValue); Assert.assertEquals(10, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize, 10 * maxChunkSize + 1, maxValue); Assert.assertEquals(11, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(1, 0xFFFF, maxValue); Assert.assertEquals(0xFFFF, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize / 2, 10 * (maxChunkSize / 2) + 1, maxValue); Assert.assertEquals(11, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); } }
public static int maxIntsInBufferForBytes(int numBytes) { int maxSizePer = (CompressedPools.BUFFER_SIZE - bufferPadding(numBytes)) / numBytes; // round down to the nearest power of 2 return Integer.highestOneBit(maxSizePer); }
public static CompressedVSizeColumnarIntsSupplier fromByteBuffer( ByteBuffer buffer, ByteOrder order ) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { final int numBytes = buffer.get(); final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedVSizeColumnarIntsSupplier( totalSize, sizePer, numBytes, GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
private WritableSupplier<ColumnarInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer) { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: case UNCOMPRESSED_WITH_FLAGS: return VSizeColumnarInts.readFromByteBuffer(buffer); case COMPRESSED: return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder); default: throw new IAE("Unsupported single-value version[%s]", version); } }
@Override public ColumnarMultiInts get() { return new CompressedVSizeColumnarMultiIntsSupplier.CompressedVSizeColumnarMultiInts( offsetSupplier.get(), valueSupplier.get() ); }
@Test public void testLargeData() throws Exception { // more than one chunk for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals((rand.nextInt(5) + 5) * maxChunkSize + rand.nextInt(maxChunkSize), maxValue); checkSerializedSizeAndData(maxChunkSize); } }
chunkFactor <= maxIntsInBufferForBytes(numBytes), "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor ); return new CompressedVSizeColumnarIntsSupplier( list.size(), chunkFactor,
@Test public void testmaxIntsInBuffer() { Assert.assertEquals(CompressedPools.BUFFER_SIZE, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForBytes(1)); Assert.assertEquals(CompressedPools.BUFFER_SIZE / 2, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForBytes(2)); Assert.assertEquals(CompressedPools.BUFFER_SIZE / 4, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForBytes(4)); Assert.assertEquals(CompressedPools.BUFFER_SIZE, 0x10000); // nearest power of 2 is 2^14 Assert.assertEquals(1 << 14, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForBytes(3)); }