public CompactDoublesSketch compact() { return compact(null); }
@Test public void wrapEmptyCompactSketch() { final CompactDoublesSketch s1 = DoublesSketch.builder().build().compact(); final Memory mem = Memory.wrap(ByteBuffer.wrap(s1.toByteArray()).order(ByteOrder.nativeOrder())); final DoublesSketch s2 = DoublesSketch.wrap(mem); assertTrue(s2.isEmpty()); assertEquals(s2.getN(), 0); assertTrue(Double.isNaN(s2.getMinValue())); assertTrue(Double.isNaN(s2.getMaxValue())); }
@Test public void checkEmpty() { final int k = PreambleUtil.DEFAULT_K; final UpdateDoublesSketch qs1 = buildAndLoadQS(k, 0); final byte[] byteArr = qs1.compact().toByteArray(); final byte[] byteArr2 = qs1.toByteArray(true); final Memory mem = Memory.wrap(byteArr); final HeapCompactDoublesSketch qs2 = HeapCompactDoublesSketch.heapifyInstance(mem); assertTrue(qs2.isEmpty()); assertEquals(byteArr.length, qs1.getStorageBytes()); assertEquals(byteArr, byteArr2); assertTrue(Double.isNaN(qs2.getQuantile(0.0))); assertTrue(Double.isNaN(qs2.getQuantile(1.0))); assertTrue(Double.isNaN(qs2.getQuantile(0.5))); final double[] quantiles = qs2.getQuantiles(new double[] {0.0, 0.5, 1.0}); assertNull(quantiles); //println(qs1.toString(true, true)); }
@Test public void heapifyEmptyCompactSketch() { final CompactDoublesSketch s1 = DoublesSketch.builder().build().compact(); final Memory mem = Memory.wrap(s1.toByteArray()); DoublesSketch s2 = DoublesSketch.heapify(mem); Assert.assertTrue(s2.isEmpty()); }
@Test public void wrapEmptyCompactSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final Memory mem = Memory.wrap(s1.compact().toByteArray()); DoublesSketch s2 = DoublesSketch.wrap(mem); // compact, so this is ok Assert.assertTrue(s2.isEmpty()); }
@Test public void heapifyFromCompactSketch() { final int k = 8; final int n = 177; final UpdateDoublesSketch qs = buildAndLoadQS(k, n); // assuming reverse ordered inserts final byte[] qsBytes = qs.compact().toByteArray(); final Memory qsMem = Memory.wrap(qsBytes); final HeapCompactDoublesSketch compactQs = HeapCompactDoublesSketch.heapifyInstance(qsMem); DoublesSketchTest.testSketchEquality(qs, compactQs); }
@Test public void wrapCompactSketch() { UpdateDoublesSketch s1 = DoublesSketch.builder().build(); s1.update(1); s1.update(2); //Memory mem = Memory.wrap(ByteBuffer.wrap(s1.compact().toByteArray()) // .asReadOnlyBuffer().order(ByteOrder.nativeOrder()));); final Memory mem = Memory.wrap(s1.compact().toByteArray()); final DoublesSketch s2 = DoublesSketch.wrap(mem); // compact, so this is ok Assert.assertEquals(s2.getMinValue(), 1.0); Assert.assertEquals(s2.getMaxValue(), 2.0); Assert.assertEquals(s2.getN(), 2); }
public CompactDoublesSketch compact() { return compact(null); }
@Test public void doublesToItems() { final UpdateDoublesSketch sketch1 = DoublesSketch.builder().build(); //SerVer = 3 for (int i = 1; i <= 500; i++) { sketch1.update(i); } final CompactDoublesSketch cs = sketch1.compact(); DoublesSketchTest.testSketchEquality(sketch1, cs); //final byte[] bytes = sketch1.compact().toByteArray(); // must be compact final byte[] bytes = cs.toByteArray(); // must be compact //reconstruct with ItemsSketch final ItemsSketch<Double> sketch2 = ItemsSketch.getInstance(Memory.wrap(bytes), Comparator.naturalOrder(), serDe); for (int i = 501; i <= 1000; i++) { sketch2.update((double) i); } Assert.assertEquals(sketch2.getN(), 1000); Assert.assertTrue(sketch2.getRetainedItems() < 1000); Assert.assertEquals(sketch2.getMinValue(), 1.0); Assert.assertEquals(sketch2.getMaxValue(), 1000.0); // based on ~1.7% normalized rank error for this particular case Assert.assertEquals(sketch2.getQuantile(0.5), 500.0, 17); }
@Test public void checkUnion2() { final DoublesSketch qs1 = buildAndLoadQS(256, 1000).compact(); final DoublesSketch qs2 = buildAndLoadQS(128, 1000); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 assertEquals(union.getEffectiveK(), 256); union.update(qs1); final DoublesSketch res1 = union.getResult(); //println(res1.toString()); assertEquals(res1.getN(), 1000); assertEquals(res1.getK(), 256); union.update(qs2); final DoublesSketch res2 = union.getResult(); assertEquals(res2.getN(), 2000); assertEquals(res2.getK(), 128); assertEquals(union.getEffectiveK(), 128); println(union.toString()); }
static DirectCompactDoublesSketch buildAndLoadDCQS(final int k, final int n, final int startV) { final UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(); for (int i = 1; i <= n; i++) { qs.update(startV + i); } final byte[] byteArr = new byte[qs.getCompactStorageBytes()]; final WritableMemory mem = WritableMemory.wrap(byteArr); return (DirectCompactDoublesSketch) qs.compact(mem); }
@Test public void checkUnion1Direct() { DoublesSketch result; final DoublesSketch qs1; final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 final DoublesSketch qs0 = buildAndLoadDQS(256, 500); union.update(qs0); //me = null, that = valid, exact result = union.getResult(); assertEquals(result.getN(), 500); assertEquals(result.getK(), 256); union.reset(); qs1 = buildAndLoadDQS(256, 1000); //first 1000 union.update(qs1); //me = null, that = valid, OK //check copy me = null, that = valid result = union.getResult(); assertEquals(result.getN(), 1000); assertEquals(result.getK(), 256); //check merge me = valid, that = valid, both K's the same final DoublesSketch qs2 = buildAndLoadDQS(256, 1000, 1000).compact(); //add 1000 union.update(qs2); result = union.getResult(); assertEquals(result.getN(), 2000); assertEquals(result.getK(), 256); }
@Test public void checkUnionQuantiles() { final int k = 128; final int n1 = k * 13; final int n2 = (k * 8) + (k / 2); final int n = n1 + n2; final double errorTolerance = 0.0175 * n; // assuming k = 128 final UpdateDoublesSketch sketch1 = buildAndLoadQS(k, n1); final CompactDoublesSketch sketch2 = buildAndLoadQS(k, n2, n1).compact(); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketch2); union.update(sketch1); final Memory mem = Memory.wrap(union.getResult().toByteArray(true)); final DoublesSketch result = DoublesSketch.wrap(mem); assertEquals(result.getN(), n1 + n2); assertEquals(result.getK(), k); for (double fraction = 0.05; fraction < 1.0; fraction += 0.05) { assertEquals(result.getQuantile(fraction), fraction * n, errorTolerance); } }
@Test public void checkUnion4DirectCompact() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 5 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn0 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn0.update(i + 1); } final CompactDoublesSketch sketchIn1 = sketchIn0.compact(); final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final CompactDoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17).compact(); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Override protected void mergeUpdateSketch() { result = updateSketch.compact(); updateSketch.reset(); }