public static double result(PercentileCounter counter) { return counter == null ? 0L : counter.getResultEstimate(); }
@Override public int compare(Object o1, Object o2) { if (o1 == null && o2 == null) return 0; if (o1 == null) // null is bigger to align with CubeCodeSystem return 1; if (o2 == null) // null is bigger to align with CubeCodeSystem return -1; // for the 'having clause', we only concern numbers and BigDecimal // we try to cache the o2, which should be a constant according to CompareTupleFilter.evaluate() double n1; if (o1 instanceof Number) { n1 = ((Number) o1).doubleValue(); } else if (o1 instanceof HLLCounter) { n1 = ((HLLCounter) o1).getCountEstimate(); } else if (o1 instanceof BitmapCounter) { n1 = ((BitmapCounter) o1).getCount(); } else if (o1 instanceof PercentileCounter) { n1 = ((PercentileCounter) o1).getResultEstimate(); } else { throw new RuntimeException("Unknown datatype: value=" + o1 + ", class=" + o1.getClass()); } double n2 = (o2Cache == o2) ? n2Cache : Double.parseDouble((String) o2); if (o2Cache == null) { o2Cache = o2; n2Cache = n2; } return Double.compare(n1, n2); }
@Test public void testBasic() { int times = 1; int compression = 100; for (int t = 0; t < times; t++) { PercentileCounter counter = new PercentileCounter(compression, 0.5); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); dataset.add(d); } Collections.sort(dataset); double actualResult = counter.getResultEstimate(); double expectedResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectedResult, actualResult, 0.001); } }
@Test public void testAggregate() { double compression = 100; int datasize = 10000; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); List<Double> dataset = Lists.newArrayListWithCapacity(datasize); for (int i = 0; i < datasize; i++) { double d = random.nextDouble(); dataset.add(d); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } Collections.sort(dataset); double actualResult = aggregator.getState().getResultEstimate(); double expectResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectResult, actualResult, 0.001); }
@Test public void testTDigest() { double compression = 100; double quantile = 0.5; PercentileCounter counter = new PercentileCounter(compression, quantile); TDigest tDigest = TDigest.createAvlTreeDigest(compression); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); tDigest.add(d); } double actualResult = counter.getResultEstimate(); Collections.sort(dataset); double expectedResult = tDigest.quantile(quantile); assertEquals(expectedResult, actualResult, 0); }
@Test public void testBasic() { PercentileSerializer serializer = new PercentileSerializer(DataType.getType("percentile(100)")); PercentileCounter counter = new PercentileCounter(100, 0.5); Random random = new Random(); for (int i = 0; i < 1000; i++) { counter.add(random.nextDouble()); } double markResult = counter.getResultEstimate(); ByteBuffer buffer = ByteBuffer.allocateDirect(serializer.getStorageBytesEstimate()); serializer.serialize(counter, buffer); buffer.flip(); counter = serializer.deserialize(buffer); PercentileCounter counter1 = new PercentileCounter(100, 0.5); counter1.merge(counter); assertEquals(markResult, counter1.getResultEstimate(), 0.01); }
@Test public void testSerialization() { Kryo kryo = new Kryo(); kryo.register(PercentileCounter.class, new PercentileCounterSerializer()); double compression = 100; double quantile = 0.8; PercentileCounter origin_counter = new PercentileCounter(compression, quantile); for (int i = 1; i < 10; i++) { origin_counter.add(i); } byte[] buffer = serialize(kryo, origin_counter); PercentileCounter deserialized_counter = deserialize(kryo, buffer, PercentileCounter.class); Assert.assertEquals("Compression Error", origin_counter.getCompression(), deserialized_counter.getCompression(), 0.00000001); Assert.assertEquals("QuantileRatio Error", origin_counter.getQuantileRatio(), deserialized_counter.getQuantileRatio(), 0.00000001); Assert.assertEquals("Estimation Error", origin_counter.getResultEstimate(), deserialized_counter.getResultEstimate(), 0.00000001); }
public static double result(PercentileCounter counter) { return counter == null ? 0L : counter.getResultEstimate(); }
@Override public int compare(Object o1, Object o2) { if (o1 == null && o2 == null) return 0; if (o1 == null) // null is bigger to align with CubeCodeSystem return 1; if (o2 == null) // null is bigger to align with CubeCodeSystem return -1; // for the 'having clause', we only concern numbers and BigDecimal // we try to cache the o2, which should be a constant according to CompareTupleFilter.evaluate() double n1; if (o1 instanceof Number) { n1 = ((Number) o1).doubleValue(); } else if (o1 instanceof HLLCounter) { n1 = ((HLLCounter) o1).getCountEstimate(); } else if (o1 instanceof BitmapCounter) { n1 = ((BitmapCounter) o1).getCount(); } else if (o1 instanceof PercentileCounter) { n1 = ((PercentileCounter) o1).getResultEstimate(); } else { throw new RuntimeException("Unknown datatype: value=" + o1 + ", class=" + o1.getClass()); } double n2 = (o2Cache == o2) ? n2Cache : Double.parseDouble((String) o2); if (o2Cache == null) { o2Cache = o2; n2Cache = n2; } return Double.compare(n1, n2); }