@Override public PercentileCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { PercentileCounter counter = current; counter.clear(); for (String v : values) { if (v != null) counter.add(Double.parseDouble(v)); } return counter; }
private PercentileAggregator createPercentileAggreator(int sumNums, Integer sqrtNum, Integer compression) { compression = compression == null ? DEFAULT_COMPRESSION : compression; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); for (int i = 0; i < sumNums; i++) { double d = 0; if (sqrtNum == null) d = random.nextInt(1000000000); else d = Math.sqrt(sqrtNum.intValue()) * random.nextGaussian(); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } return aggregator; }
@Test public void testBasic() { int times = 1; int compression = 100; for (int t = 0; t < times; t++) { PercentileCounter counter = new PercentileCounter(compression, 0.5); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); dataset.add(d); } Collections.sort(dataset); double actualResult = counter.getResultEstimate(); double expectedResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectedResult, actualResult, 0.001); } }
@Test public void testAggregate() { double compression = 100; int datasize = 10000; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); List<Double> dataset = Lists.newArrayListWithCapacity(datasize); for (int i = 0; i < datasize; i++) { double d = random.nextDouble(); dataset.add(d); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } Collections.sort(dataset); double actualResult = aggregator.getState().getResultEstimate(); double expectResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectResult, actualResult, 0.001); }
@Test public void testTDigest() { double compression = 100; double quantile = 0.5; PercentileCounter counter = new PercentileCounter(compression, quantile); TDigest tDigest = TDigest.createAvlTreeDigest(compression); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); tDigest.add(d); } double actualResult = counter.getResultEstimate(); Collections.sort(dataset); double expectedResult = tDigest.quantile(quantile); assertEquals(expectedResult, actualResult, 0); }
@Test public void testBasic() { PercentileSerializer serializer = new PercentileSerializer(DataType.getType("percentile(100)")); PercentileCounter counter = new PercentileCounter(100, 0.5); Random random = new Random(); for (int i = 0; i < 1000; i++) { counter.add(random.nextDouble()); } double markResult = counter.getResultEstimate(); ByteBuffer buffer = ByteBuffer.allocateDirect(serializer.getStorageBytesEstimate()); serializer.serialize(counter, buffer); buffer.flip(); counter = serializer.deserialize(buffer); PercentileCounter counter1 = new PercentileCounter(100, 0.5); counter1.merge(counter); assertEquals(markResult, counter1.getResultEstimate(), 0.01); }
@Test public void testSerialization() { Kryo kryo = new Kryo(); kryo.register(PercentileCounter.class, new PercentileCounterSerializer()); double compression = 100; double quantile = 0.8; PercentileCounter origin_counter = new PercentileCounter(compression, quantile); for (int i = 1; i < 10; i++) { origin_counter.add(i); } byte[] buffer = serialize(kryo, origin_counter); PercentileCounter deserialized_counter = deserialize(kryo, buffer, PercentileCounter.class); Assert.assertEquals("Compression Error", origin_counter.getCompression(), deserialized_counter.getCompression(), 0.00000001); Assert.assertEquals("QuantileRatio Error", origin_counter.getQuantileRatio(), deserialized_counter.getQuantileRatio(), 0.00000001); Assert.assertEquals("Estimation Error", origin_counter.getResultEstimate(), deserialized_counter.getResultEstimate(), 0.00000001); }
@Override public PercentileCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { PercentileCounter counter = current; counter.clear(); for (String v : values) { if (v != null) counter.add(Double.parseDouble(v)); } return counter; }