@Test public void testSerialization() throws Exception { final int[] array = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; VSizeColumnarInts ints = VSizeColumnarInts.fromArray(array); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ints.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(ints.getSerializedSize(), bytes.length); VSizeColumnarInts deserialized = VSizeColumnarInts.readFromByteBuffer(ByteBuffer.wrap(bytes)); Assert.assertEquals(1, deserialized.getNumBytes()); Assert.assertEquals(array.length, deserialized.size()); for (int i = 0; i < array.length; i++) { Assert.assertEquals(array[i], deserialized.get(i)); } } }
public static VSizeColumnarInts fromArray(int[] array, int maxValue) { return fromIndexedInts(new ArrayBasedIndexedInts(array), maxValue); }
public static VSizeColumnarInts fromIndexedInts(IndexedInts ints, int maxValue) { int numBytes = getNumBytesForMax(maxValue); final ByteBuffer buffer = ByteBuffer.allocate((ints.size() * numBytes) + (4 - numBytes)); writeToBuffer(buffer, ints, numBytes, maxValue); return new VSizeColumnarInts(buffer.asReadOnlyBuffer(), numBytes); }
@Test public void testSanity() { final int[] array = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; VSizeColumnarInts ints = VSizeColumnarInts.fromArray(array); Assert.assertEquals(1, ints.getNumBytes()); Assert.assertEquals(array.length, ints.size()); for (int i = 0; i < array.length; i++) { Assert.assertEquals(array[i], ints.get(i)); } }
private void checkSerializedSizeAndData() throws Exception { int maxValue = vals.length == 0 ? 0 : Ints.max(vals); VSizeColumnarIntsSerializer writer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, maxValue); VSizeColumnarInts intsFromList = VSizeColumnarInts.fromIndexedInts(new ArrayBasedIndexedInts(vals), maxValue); writer.open(); for (int val : vals) { writer.addValue(val); } long writtenLength = writer.getSerializedSize(); WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); writer.writeTo(writeOutBytes, null); assertEquals(writtenLength, intsFromList.getSerializedSize()); // read from ByteBuffer and check values VSizeColumnarInts intsFromByteBuffer = VSizeColumnarInts.readFromByteBuffer( ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())) ); assertEquals(vals.length, intsFromByteBuffer.size()); for (int i = 0; i < vals.length; ++i) { assertEquals(vals[i], intsFromByteBuffer.get(i)); } }
).get(); final ByteBuffer bufferUncompressed = serialize(VSizeColumnarInts.fromArray(vals)); this.uncompressed = VSizeColumnarInts.readFromByteBuffer(bufferUncompressed);
@Override public VSizeColumnarInts apply(int[] input) { return VSizeColumnarInts.fromArray(input, 20); } }
VSizeColumnarInts next = objects.next(); if (numBytes == -1) { numBytes = next.getNumBytes(); if (object.getNumBytes() != numBytes) { throw new ISE("val.numBytes[%s] != numBytesInValue[%s]", object.getNumBytes(), numBytes); offset += object.getNumBytesNoPadding(); headerBytes.writeInt(offset); object.writeBytesNoPaddingTo(valueBytes);
private void assertSame(List<int[]> someInts, VSizeColumnarMultiInts indexed) { Assert.assertEquals(3, indexed.size()); for (int i = 0; i < indexed.size(); ++i) { final int[] ints = someInts.get(i); final VSizeColumnarInts vSizeColumnarInts = indexed.get(i); Assert.assertEquals(ints.length, vSizeColumnarInts.size()); Assert.assertEquals(1, vSizeColumnarInts.getNumBytes()); for (int j = 0; j < ints.length; j++) { Assert.assertEquals(ints[j], vSizeColumnarInts.get(j)); } } } }
private WritableSupplier<ColumnarInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer) { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: case UNCOMPRESSED_WITH_FLAGS: return VSizeColumnarInts.readFromByteBuffer(buffer); case COMPRESSED: return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder); default: throw new IAE("Unsupported single-value version[%s]", version); } }
public VSizeColumnarIntsSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue) { this.segmentWriteOutMedium = segmentWriteOutMedium; this.numBytes = VSizeColumnarInts.getNumBytesForMax(maxValue); }
public static VSizeColumnarInts readFromByteBuffer(ByteBuffer buffer) { byte versionFromBuffer = buffer.get(); if (VERSION == versionFromBuffer) { int numBytes = buffer.get(); int size = buffer.getInt(); ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); bufferToUse.limit(bufferToUse.position() + size); buffer.position(bufferToUse.limit()); return new VSizeColumnarInts( bufferToUse, numBytes ); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
public static VSizeColumnarInts fromArray(int[] array) { return fromArray(array, Ints.max(array)); }
VSizeColumnarInts next = objects.next(); if (numBytes == -1) { numBytes = next.getNumBytes(); if (object.getNumBytes() != numBytes) { throw new ISE("val.numBytes[%s] != numBytesInValue[%s]", object.getNumBytes(), numBytes); offset += object.getNumBytesNoPadding(); headerBytes.writeInt(offset); object.writeBytesNoPaddingTo(valueBytes);
private WritableSupplier<ColumnarInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer) { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: case UNCOMPRESSED_WITH_FLAGS: return VSizeColumnarInts.readFromByteBuffer(buffer); case COMPRESSED: return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder); default: throw new IAE("Unsupported single-value version[%s]", version); } }
private static int sizePer(int maxValue, int chunkFactor) { return chunkFactor * VSizeColumnarInts.getNumBytesForMax(maxValue) + CompressedVSizeColumnarIntsSupplier.bufferPadding(VSizeColumnarInts.getNumBytesForMax(maxValue)); }
@Override public VSizeColumnarInts get(int index) { if (index >= size) { throw new IAE("Index[%d] >= size[%d]", index, size); } ByteBuffer myBuffer = theBuffer.asReadOnlyBuffer(); int startOffset = 0; int endOffset; if (index == 0) { endOffset = myBuffer.getInt(); } else { myBuffer.position(myBuffer.position() + ((index - 1) * Integer.BYTES)); startOffset = myBuffer.getInt(); endOffset = myBuffer.getInt(); } myBuffer.position(valuesOffset + startOffset); myBuffer.limit(myBuffer.position() + (endOffset - startOffset) + bufferBytes); return myBuffer.hasRemaining() ? new VSizeColumnarInts(myBuffer, numBytes) : null; }
Iterables.transform(rows, (Function<int[], ColumnarInts>) input -> VSizeColumnarInts.fromArray(input, 20)), bound - 1, ByteOrder.nativeOrder(), VSizeColumnarMultiInts.fromIterable(Iterables.transform(rows, input -> VSizeColumnarInts.fromArray(input, 20))) ); this.uncompressed = VSizeColumnarMultiInts.readFromByteBuffer(bufferUncompressed);
public static VSizeColumnarInts fromIndexedInts(IndexedInts ints, int maxValue) { int numBytes = getNumBytesForMax(maxValue); final ByteBuffer buffer = ByteBuffer.allocate((ints.size() * numBytes) + (4 - numBytes)); writeToBuffer(buffer, ints, numBytes, maxValue); return new VSizeColumnarInts(buffer.asReadOnlyBuffer(), numBytes); }
private void writeNumBytesForMax() throws IOException { if (!numBytesForMaxWritten) { final byte numBytesForMax = VSizeColumnarInts.getNumBytesForMax(maxId); valuesOut.write(new byte[4 - numBytesForMax]); numBytesForMaxWritten = true; } } }