@Override public Object combine(@Nullable final Object lhs, @Nullable final Object rhs) { final ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setNominalEntries(nominalEntries) .setNumberOfValues(numberOfValues).buildUnion(); if (lhs != null) { union.update((ArrayOfDoublesSketch) lhs); } if (rhs != null) { union.update((ArrayOfDoublesSketch) rhs); } return union.getResult(); }
@Override public int getMaxIntermediateSize() { return ArrayOfDoublesUnion.getMaxBytes(nominalEntries, numberOfValues); }
@Override public void reset(final ColumnValueSelector selector) { union.reset(); fold(selector); }
union1.update(sketch1); union1.update(sketch2); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(union1.toByteArray())); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union2.reset(); result = union2.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode());
union1.update(sketch1); union1.update(sketch2); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(WritableMemory.wrap(union1.toByteArray())); ArrayOfDoublesCompactSketch result = union2.getResult(WritableMemory.wrap(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union2.reset(); result = union2.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode());
@Test public void heapEstimationMode() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertTrue(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union.reset(); result = union.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getTheta(), 1.0); }
union.update(sketch1); union.update(sketch2); int maxBytes = ArrayOfDoublesUnion.getMaxBytes(union.nomEntries_, union.numValues_); Assert.assertEquals(maxBytes, 131104); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertEquals(result.getEstimate(), 3.0); double[][] values = result.getValues(); Assert.assertEquals(values[2][0], 3.0); WritableMemory wmem = WritableMemory.wrap(union.toByteArray()); ArrayOfDoublesUnion wrappedUnion = ArrayOfDoublesSketches.wrapUnion(wmem); result = wrappedUnion.getResult(); Assert.assertEquals(result.getEstimate(), 3.0); values = result.getValues(); union.reset(); result = union.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode());
@Test public void wrapAndTryUpdatingUnionV0_9_1() throws Exception { byte[] bytes = TestUtil.readBytesFromFile(getClass().getClassLoader() .getResource("ArrayOfDoublesUnion_v0.9.1.bin").getFile()); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(Memory.wrap(bytes)); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); boolean thrown = false; try { union2.reset(); } catch (SketchesReadOnlyException e) { thrown = true; } Assert.assertTrue(thrown); }
@Test public void heapDeserializeV0_9_1() throws Exception { byte[] bytes = TestUtil.readBytesFromFile(getClass().getClassLoader().getResource("ArrayOfDoublesUnion_v0.9.1.bin").getFile()); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(bytes)); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union2.reset(); result = union2.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getTheta(), 1.0); double[][] values = result.getValues(); for (int i = 0; i < values.length; i++) { Assert.assertEquals(values[i][0], 2.0); } }
@Override public ArrayOfDoublesSketch getObject() { return union.getResult(); }
@Test public void heapifyAndUpdateUnion() { int numUniques = 10000; int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch1.update(key++, new double[] {1}); } ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union1.update(sketch1); ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.heapifyUnion(Memory.wrap(union1.toByteArray())); ArrayOfDoublesSketch resultSketch = union2.getResult(); Assert.assertTrue(resultSketch.isEstimationMode()); Assert.assertEquals(resultSketch.getEstimate(), numUniques, numUniques * 0.04); // make sure union update actually needs to modify the union ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch2.update(key++, new double[] {1}); } union2.update(sketch2); }
@Override public void fold(final ColumnValueSelector selector) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) selector.getObject(); union.update(sketch); }
/** * Wrap the given Memory and seed as an ArrayOfDoublesUnion * @param mem the given Memory * @param seed the given seed * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final Memory mem, final long seed) { return ArrayOfDoublesUnion.wrap(mem, seed); }
/** * Heapify the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion heapify(final Memory mem) { return heapify(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given Memory and seed as an ArrayOfDoublesUnion * @param mem the given Memory * @param seed the given seed * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrap(final Memory mem, final long seed) { return wrapUnionImpl((WritableMemory) mem, seed, false); }
@Override void setThetaLong(final long theta) { super.setThetaLong(theta); mem_.putLong(THETA_LONG, theta); }
@Test public void heapSerializeDeserializeWithSeed() { long seed = 1; int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().setSeed(seed).buildUnion(); union1.update(sketch1); union1.update(sketch2); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(union1.toByteArray()), seed); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); }
@Test public void heapToDirect() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketch1.update(1, new double[] {1.0}); sketch1.update(1, new double[] {1.0}); sketch1.update(1, new double[] {1.0}); sketch1.update(2, new double[] {1.0}); ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketch2.update(2, new double[] {1.0}); sketch2.update(2, new double[] {1.0}); sketch2.update(3, new double[] {1.0}); sketch2.update(3, new double[] {1.0}); sketch2.update(3, new double[] {1.0}); ArrayOfDoublesUnion heapUnion = new ArrayOfDoublesSetOperationBuilder().buildUnion(); heapUnion.update(sketch1); ArrayOfDoublesUnion directUnion = ArrayOfDoublesUnion.wrap(WritableMemory.wrap(heapUnion.toByteArray())); directUnion.update(sketch2); ArrayOfDoublesCompactSketch result = directUnion.getResult(WritableMemory.wrap(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 3.0); double[][] values = result.getValues(); Assert.assertEquals(values.length, 3); Assert.assertEquals(values[0][0], 3.0); Assert.assertEquals(values[1][0], 3.0); Assert.assertEquals(values[2][0], 3.0); }
@Test public void directEstimationMode() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion(WritableMemory.wrap(new byte[1000000])); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(WritableMemory.wrap(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union.reset(); result = union.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getTheta(), 1.0); }
/** * This method uses synchronization because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public synchronized Object get() { return union.getResult(); }