private void makeWithSerde(final int chunkSize) throws IOException { CloseQuietly.close(columnarInts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); final CompressedColumnarIntsSupplier theSupplier = CompressedColumnarIntsSupplier.fromIntBuffer( IntBuffer.wrap(vals), chunkSize, ByteOrder.nativeOrder(), compressionStrategy, closer ); theSupplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); supplier = CompressedColumnarIntsSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), ByteOrder.nativeOrder()); columnarInts = supplier.get(); }
public static CompressedColumnarIntsSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedColumnarIntsSupplier( totalSize, sizePer, GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
private void setupSimple(final int chunkSize) { CloseQuietly.close(columnarInts); vals = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16}; supplier = CompressedColumnarIntsSupplier.fromIntBuffer( IntBuffer.wrap(vals), chunkSize, ByteOrder.nativeOrder(), compressionStrategy, closer ); columnarInts = supplier.get(); }
compressionStrategy ); CompressedColumnarIntsSupplier supplierFromList = CompressedColumnarIntsSupplier.fromList( IntArrayList.wrap(vals), chunkFactor, smoosher.close(); assertEquals(writtenLength, supplierFromList.getSerializedSize()); CompressedColumnarIntsSupplier supplierFromByteBuffer = CompressedColumnarIntsSupplier.fromByteBuffer( ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())), byteOrder ); ColumnarInts columnarInts = supplierFromByteBuffer.get(); assertEquals(vals.length, columnarInts.size()); for (int i = 0; i < vals.length; ++i) {
CompressedColumnarIntsSupplier supplierFromByteBuffer = CompressedColumnarIntsSupplier.fromByteBuffer( mapper.mapFile("test"), byteOrder ); ColumnarInts columnarInts = supplierFromByteBuffer.get(); assertEquals(vals.length, columnarInts.size()); for (int i = 0; i < vals.length; ++i) {
@Override public ColumnarMultiInts get() { return new CompressedVSizeColumnarMultiIntsSupplier.CompressedVSizeColumnarMultiInts( offsetSupplier.get(), valueSupplier.get() ); }
@Override public long getSerializedSize() { return 1 + offsetSupplier.getSerializedSize() + valueSupplier.getSerializedSize(); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION})); offsetSupplier.writeTo(channel, smoosher); valueSupplier.writeTo(channel, smoosher); }
public static V3CompressedVSizeColumnarMultiIntsSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { CompressedColumnarIntsSupplier offsetSupplier = CompressedColumnarIntsSupplier.fromByteBuffer( buffer, order ); CompressedVSizeColumnarIntsSupplier valueSupplier = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( buffer, order ); return new V3CompressedVSizeColumnarMultiIntsSupplier(offsetSupplier, valueSupplier); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
CompressedColumnarIntsSupplier headerSupplier = CompressedColumnarIntsSupplier.fromList( offsetList, offsetChunkFactor,
@Test public void testLargeChunks() throws Exception { final int maxChunkSize = CompressedPools.BUFFER_SIZE / Long.BYTES; setupLargeChunks(maxChunkSize, 10 * maxChunkSize); Assert.assertEquals(10, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize, 10 * maxChunkSize + 1); Assert.assertEquals(11, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize - 1, 10 * (maxChunkSize - 1) + 1); Assert.assertEquals(11, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); }
final ColumnarInts columnarInts2 = supplier.get(); try { new Thread(new Runnable()
@Override public long getSerializedSize() { return 1 + offsetSupplier.getSerializedSize() + valueSupplier.getSerializedSize(); }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION})); offsetSupplier.writeTo(channel, smoosher); valueSupplier.writeTo(channel, smoosher); }
public static V3CompressedVSizeColumnarMultiIntsSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { CompressedColumnarIntsSupplier offsetSupplier = CompressedColumnarIntsSupplier.fromByteBuffer( buffer, order ); CompressedVSizeColumnarIntsSupplier valueSupplier = CompressedVSizeColumnarIntsSupplier.fromByteBuffer( buffer, order ); return new V3CompressedVSizeColumnarMultiIntsSupplier(offsetSupplier, valueSupplier); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
CompressedColumnarIntsSupplier headerSupplier = CompressedColumnarIntsSupplier.fromList( offsetList, offsetChunkFactor,
@Test public void testSanity() { setupSimple(5); Assert.assertEquals(4, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); // test powers of 2 setupSimple(4); Assert.assertEquals(4, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); setupSimple(32); Assert.assertEquals(1, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); }
@Override public ColumnarMultiInts get() { return new CompressedVSizeColumnarMultiIntsSupplier.CompressedVSizeColumnarMultiInts( offsetSupplier.get(), valueSupplier.get() ); }
); return new CompressedColumnarIntsSupplier( list.size(), chunkFactor,
@Test public void testSanityWithSerde() throws Exception { setupSimpleWithSerde(5); Assert.assertEquals(4, supplier.getBaseIntBuffers().size()); assertIndexMatchesVals(); }