buildData.write(Double.toString(value)); buildData.newLine(); sketch.update(value); sequenceNumber++; sketchData.write(StringUtils.encodeBase64String(sketch.toByteArray(true))); sketchData.newLine();
public CompactDoublesSketch compact() { return compact(null); }
@Test public void testAggregate() { final DoublesUnionAggregator unionAggregator = new DoublesUnionAggregator(); DoublesUnion currentState = union1; assertEquals(3L, currentState.getResult().getN()); assertEquals(2.0D, currentState.getResult().getQuantile(0.5D), DELTA); currentState = unionAggregator.apply(currentState, union2); assertEquals(7L, currentState.getResult().getN()); assertEquals(4.0D, currentState.getResult().getQuantile(0.5D), DELTA); }
@Override public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { UpdateDoublesSketch sketch = sketches.get(oldBuffer).get(oldPosition); final WritableMemory oldRegion = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); if (sketch.isSameResource(oldRegion)) { // sketch was not relocated on heap final WritableMemory newRegion = getMemory(newBuffer).writableRegion(newPosition, maxIntermediateSize); sketch = UpdateDoublesSketch.wrap(newRegion); } putSketch(newBuffer, newPosition, sketch); final Int2ObjectMap<UpdateDoublesSketch> map = sketches.get(oldBuffer); map.remove(oldPosition); if (map.isEmpty()) { sketches.remove(oldBuffer); memCache.remove(oldBuffer); } }
final int tgtK = tgt.getK(); final long srcN = src.getN(); final long tgtN = tgt.getN(); tgt.update(srcSketchBuf.get(i)); final int tgtCombBufItemCap = tgt.getCombinedBufferItemCapacity(); if (spaceNeeded > tgtCombBufItemCap) { //copies base buffer plus current levels tgt.growCombinedBuffer(tgtCombBufItemCap, spaceNeeded); tgtK, tgtSketchBuf, tgt.getBitPattern() ); tgt.putBitPattern(newTgtBitPattern); if (tgt.isDirect() && (nFinal > 0)) { final WritableMemory mem = tgt.getMemory(); mem.clearBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); tgt.putN(nFinal); assert (tgt.getN() / (2L * tgtK)) == tgt.getBitPattern(); // internal consistency check double tgtMax = tgt.getMaxValue(); tgtMax = Double.isNaN(tgtMax) ? Double.NEGATIVE_INFINITY : tgtMax; double tgtMin = tgt.getMinValue(); tgtMin = Double.isNaN(tgtMin) ? Double.POSITIVE_INFINITY : tgtMin;
@Test public void heapifyAndUpdateSparseSketch() { UpdateDoublesSketch s1 = DoublesSketch.builder().build(); s1.update(1); s1.update(2); Memory mem = Memory.wrap(s1.toByteArray(false)); UpdateDoublesSketch s2 = (UpdateDoublesSketch) DoublesSketch.heapify(mem); s2.update(3); Assert.assertEquals(s2.getMinValue(), 1.0); Assert.assertEquals(s2.getMaxValue(), 3.0); }
@Test public void wrapEmptyUpdateSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final Memory mem = Memory.wrap(s1.toByteArray()); UpdateDoublesSketch s2 = (UpdateDoublesSketch) DoublesSketch.wrap(mem); Assert.assertTrue(s2.isEmpty()); s2.putMinValue(-1.0); fail(); } catch (final SketchesReadOnlyException e) { s2.putMaxValue(1.0); fail(); } catch (final SketchesReadOnlyException e) { s2.putN(1); fail(); } catch (final SketchesReadOnlyException e) { s2.putBitPattern(1); fail(); } catch (final SketchesReadOnlyException e) { s2.reset(); fail(); } catch (final SketchesReadOnlyException e) { s2.putBaseBufferCount(5); fail(); } catch (final SketchesReadOnlyException e) {
@Override public synchronized void aggregate() { sketch.update(valueSelector.getDouble()); }
@Test public void itemsToDoubles() { final ItemsSketch<Double> sketch1 = ItemsSketch.getInstance(Comparator.naturalOrder()); for (int i = 1; i <= 500; i++) { sketch1.update((double) i); } final byte[] bytes = sketch1.toByteArray(serDe); final UpdateDoublesSketch sketch2; sketch2 = UpdateDoublesSketch.heapify(Memory.wrap(bytes)); for (int i = 501; i <= 1000; i++) { sketch2.update(i); } Assert.assertEquals(sketch2.getN(), 1000); Assert.assertTrue(sketch2.getRetainedItems() < 1000); Assert.assertEquals(sketch2.getMinValue(), 1.0); Assert.assertEquals(sketch2.getMaxValue(), 1000.0); // based on ~1.7% normalized rank error for this particular case Assert.assertEquals(sketch2.getQuantile(0.5), 500.0, 17); }
final int k = sketch.getK(); final long n = sketch.getN(); assert hcds.bitPattern_ == sketch.getBitPattern(); hcds.minValue_ = sketch.getMinValue(); hcds.maxValue_ = sketch.getMaxValue(); hcds.baseBufferCount_ = computeBaseBufferItems(k, n); assert hcds.baseBufferCount_ == sketch.getBaseBufferCount();
@SuppressWarnings("null") static UpdateDoublesSketch updateLogic(final int myMaxK, final UpdateDoublesSketch myQS, final DoublesSketch other) { int sw1 = ((myQS == null) ? 0 : myQS.isEmpty() ? 4 : 8); sw1 |= ((other == null) ? 0 : other.isEmpty() ? 1 : 2); int outCase = 0; //0=null, 1=NOOP, 2=copy, 3=merge ret.update(otherAccessor.get(i)); ret.update(otherAccessor.get(i)); if (myQS.getK() <= other.getK()) { //I am smaller or equal, thus the target DoublesMergeImpl.mergeInto(other, myQS); ret = myQS; if (myQS.isEmpty()) { if (myQS.isDirect()) { final WritableMemory mem = myQS.getMemory(); //myQS is empty, ok to reconfigure other.putMemory(mem, false); // not compact, but BB ordered ret = DirectUpdateDoublesSketch.wrapInstance(mem); ret = (myQS.isDirect()) ? DoublesSketch.builder().setK(other.getK()).build(myQS.getMemory()) : DoublesSketch.builder().setK(other.getK()).build();
@Test public void checkMisc() { int k = PreambleUtil.DEFAULT_K; int n = 48; int cap = 32 + ((2 * k) << 3); WritableMemory mem = WritableMemory.wrap(new byte[cap]); UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(mem); mem = qs.getMemory(); assertEquals(mem.getCapacity(), cap); double[] combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, 2 * k); qs = buildAndLoadDQS(k, n); qs.update(Double.NaN); int n2 = (int)qs.getN(); assertEquals(n2, n); combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, ceilingPowerOf2(n)); // since n < k println(qs.toString(true, true)); qs.reset(); assertEquals(qs.getN(), 0); qs.putBaseBufferCount(0); }
@Test public void checkGrowFromWrappedEmptySketch() { final int k = 16; final int n = 0; final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); try (WritableDirectHandle memHandle = WritableMemory.allocateDirect(initBytes)) { WritableMemory mem = memHandle.get(); origSketchMem.copyTo(0, mem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(mem.isSameResource(usk2.getMemory())); assertEquals(mem.getCapacity(), initBytes); assertTrue(mem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); assertFalse(mem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); assertEquals(mem2.getCapacity(), expectedSize); } }
@Test public void wrapEmptyUpdateSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final WritableMemory mem = WritableMemory.wrap(ByteBuffer.wrap(s1.toByteArray()).order(ByteOrder.nativeOrder())); final UpdateDoublesSketch s2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(s2.isEmpty()); assertEquals(s2.getN(), 0); assertTrue(Double.isNaN(s2.getMinValue())); assertTrue(Double.isNaN(s2.getMaxValue())); s2.reset(); // empty: so should be a no-op assertEquals(s2.getN(), 0); }
@Test public void differentLargerK() { final DoublesUnion union = DoublesUnion.builder().setMaxK(128).build(); final UpdateDoublesSketch sketch1 = buildAndLoadQS(256, 0); union.update(sketch1); Assert.assertEquals(union.getResult().getK(), 128); sketch1.update(1.0); union.update(sketch1); Assert.assertEquals(union.getResult().getK(), 128); }
@Test public void checkToByteArray() { UpdateDoublesSketch ds = DoublesSketch.builder().build(); //k = 128 ds.update(1); ds.update(2); byte[] arr = ds.toByteArray(false); assertEquals(arr.length, ds.getUpdatableStorageBytes()); }
@Test public void checkUnion4() { //Union is direct, valid and with larger K than valid input final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn1 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn1.update(i + 1); } final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 //DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
final WritableMemory dstMem) { final long memCap = dstMem.getCapacity(); final int k = sketch.getK(); final long n = sketch.getN(); checkDirectMemCapacity(k, n, memCap); if (sketch.isEmpty()) { insertFlags(dstMem, flags | EMPTY_FLAG_MASK); } else { insertFlags(dstMem, flags); insertN(dstMem, n); insertMinDouble(dstMem, sketch.getMinValue()); insertMaxDouble(dstMem, sketch.getMaxValue());
@Test public void checkPuts() { long n1 = 1001; UpdateDoublesSketch qsk = buildAndLoadQS(32, (int)n1); long n2 = qsk.getN(); assertEquals(n2, n1); int bbCnt1 = qsk.getBaseBufferCount(); long pat1 = qsk.getBitPattern(); qsk.putBitPattern(pat1 + 1); //corrupt the pattern long pat2 = qsk.getBitPattern(); assertEquals(pat1 + 1, pat2); qsk.putBaseBufferCount(bbCnt1 + 1); //corrupt the bbCount int bbCnt2 = qsk.getBaseBufferCount(); assertEquals(bbCnt1 + 1, bbCnt2); qsk.putN(n1 + 1); //corrupt N long n3 = qsk.getN(); assertEquals(n1 + 1, n3); assertNull(qsk.getMemory()); }
@Test public void checkUnion4DirectCompact() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 5 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn0 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn0.update(i + 1); } final CompactDoublesSketch sketchIn1 = sketchIn0.compact(); final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final CompactDoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17).compact(); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }