public static CompressedVSizeColumnarIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ); }
@Test public void testLargeData() throws Exception { // more than one chunk for (int offsetChunk : OFFSET_CHUNK_FACTORS) { for (int maxValue : MAX_VALUES) { final int valueChunk = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals((rand.nextInt(2) + 1) * valueChunk + rand.nextInt(valueChunk), maxValue); checkSerializedSizeAndData(offsetChunk, valueChunk); } } }
@Test public void testMultiValueFileLargeData() throws Exception { for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals((rand.nextInt(5) + 5) * maxChunkSize + rand.nextInt(maxChunkSize), maxValue); checkV2SerializedSizeAndData(maxChunkSize); } }
@Test public void testLargeData() throws Exception { // more than one chunk for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals((rand.nextInt(5) + 5) * maxChunkSize + rand.nextInt(maxChunkSize), maxValue); checkSerializedSizeAndData(maxChunkSize); } }
@Test public void testMultiValueFileLargeData() throws Exception { // more than one chunk for (int offsetChunk : OFFSET_CHUNK_FACTORS) { for (int maxValue : MAX_VALUES) { final int valueChunk = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals((rand.nextInt(2) + 1) * valueChunk + rand.nextInt(valueChunk), maxValue); checkV2SerializedSizeAndData(offsetChunk, valueChunk); } } } }
@Test public void testSmallData() throws Exception { // less than one chunk for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals(rand.nextInt(maxChunkSize), maxValue); checkSerializedSizeAndData(maxChunkSize); } }
offsetList, offsetMax, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(offsetMax), byteOrder, compression, values, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), byteOrder, compression,
@Test public void testSmallData() throws Exception { // less than one chunk for (int offsetChunk : OFFSET_CHUNK_FACTORS) { for (int maxValue : MAX_VALUES) { final int valueChunk = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); generateVals(rand.nextInt(valueChunk), maxValue); checkSerializedSizeAndData(offsetChunk, valueChunk); } } }
public static V3CompressedVSizeColumnarMultiIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new V3CompressedVSizeColumnarMultiIntsSerializer( new CompressedColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, CompressedColumnarIntsSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ) ); }
values, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), byteOrder, compression,
@Test public void testChunkTooBig() throws Exception { for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); try { setupLargeChunks(maxChunkSize + 1, 10 * (maxChunkSize + 1), maxValue); Assert.fail(); } catch (IllegalArgumentException e) { Assert.assertTrue("chunk too big for maxValue " + maxValue, true); } } }
@Test public void testLargeChunks() throws Exception { for (int maxValue : MAX_VALUES) { final int maxChunkSize = CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue); setupLargeChunks(maxChunkSize, 10 * maxChunkSize, maxValue); Assert.assertEquals(10, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize, 10 * maxChunkSize + 1, maxValue); Assert.assertEquals(11, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(1, 0xFFFF, maxValue); Assert.assertEquals(0xFFFF, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); setupLargeChunks(maxChunkSize / 2, 10 * (maxChunkSize / 2) + 1, maxValue); Assert.assertEquals(11, supplier.getBaseBuffers().size()); assertIndexMatchesVals(); } }
public static CompressedVSizeColumnarIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ); }
offsetList, offsetMax, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(offsetMax), byteOrder, compression, values, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), byteOrder, compression,
public static V3CompressedVSizeColumnarMultiIntsSerializer create( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final CompressionStrategy compression ) { return new V3CompressedVSizeColumnarMultiIntsSerializer( new CompressedColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, CompressedColumnarIntsSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeColumnarIntsSerializer( segmentWriteOutMedium, filenameBase, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, compression ) ); }
values, maxValue, CompressedVSizeColumnarIntsSupplier.maxIntsInBufferForValue(maxValue), byteOrder, compression,