@Override public PercentileCounter aggregate(PercentileCounter value1, PercentileCounter value2) { if (value1 == null) { return new PercentileCounter(value2); } else if (value2 == null) { return new PercentileCounter(value1); } PercentileCounter merged = new PercentileCounter(value1); merged.merge(value2); return merged; }
private PercentileCounter current() { if (current == null) { current = new ThreadLocal<>(); } PercentileCounter counter = current.get(); if (counter == null) { counter = new PercentileCounter(compression); current.set(counter); } return counter; }
@Override public void aggregate(PercentileCounter value) { if (sum == null) sum = new PercentileCounter(value); else sum.merge(value); }
@Override public PercentileCounter deserialize(ByteBuffer in) { PercentileCounter counter = new PercentileCounter(compression); counter.readRegisters(in); return counter; } }
public static PercentileCounter add(PercentileCounter counter, Object v, Object r) { PercentileCounter c = (PercentileCounter) v; Number n = (Number) r; if (counter == null) { counter = new PercentileCounter(c.compression, n.doubleValue()); } counter.merge(c); return counter; }
@Override public void reset() { current = new PercentileCounter(dataType.getPrecision()); } };
@Override public MeasureIngester<PercentileCounter> newIngester() { return new MeasureIngester<PercentileCounter>() { PercentileCounter current = new PercentileCounter(dataType.getPrecision()); @Override public PercentileCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { PercentileCounter counter = current; counter.clear(); for (String v : values) { if (v != null) counter.add(Double.parseDouble(v)); } return counter; } @Override public void reset() { current = new PercentileCounter(dataType.getPrecision()); } }; }
private double getEstimateSize(int numBefore, int numAfter, int compression) { return new PercentileCounter(compression == 0 ? DEFAULT_COMPRESSION : compression) .getBytesEstimate(numBefore * 1.0 / numAfter); }
@Override public PercentileCounter read(Kryo kryo, Input input, Class type) { double compression = input.readDouble(); double quantileRatio = input.readDouble(); int length = input.readInt(); byte[] buffer = new byte[length]; input.read(buffer); PercentileCounter counter = new PercentileCounter(compression, quantileRatio); counter.readRegisters(ByteBuffer.wrap(buffer)); return counter; } }
private PercentileAggregator createPercentileAggreator(int sumNums, Integer sqrtNum, Integer compression) { compression = compression == null ? DEFAULT_COMPRESSION : compression; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); for (int i = 0; i < sumNums; i++) { double d = 0; if (sqrtNum == null) d = random.nextInt(1000000000); else d = Math.sqrt(sqrtNum.intValue()) * random.nextGaussian(); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } return aggregator; }
@Test public void testBasic() { int times = 1; int compression = 100; for (int t = 0; t < times; t++) { PercentileCounter counter = new PercentileCounter(compression, 0.5); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); dataset.add(d); } Collections.sort(dataset); double actualResult = counter.getResultEstimate(); double expectedResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectedResult, actualResult, 0.001); } }
@Test public void testAggregate() { double compression = 100; int datasize = 10000; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); List<Double> dataset = Lists.newArrayListWithCapacity(datasize); for (int i = 0; i < datasize; i++) { double d = random.nextDouble(); dataset.add(d); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } Collections.sort(dataset); double actualResult = aggregator.getState().getResultEstimate(); double expectResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectResult, actualResult, 0.001); }
@Test public void testTDigest() { double compression = 100; double quantile = 0.5; PercentileCounter counter = new PercentileCounter(compression, quantile); TDigest tDigest = TDigest.createAvlTreeDigest(compression); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); tDigest.add(d); } double actualResult = counter.getResultEstimate(); Collections.sort(dataset); double expectedResult = tDigest.quantile(quantile); assertEquals(expectedResult, actualResult, 0); }
@Test public void testBasic() { PercentileSerializer serializer = new PercentileSerializer(DataType.getType("percentile(100)")); PercentileCounter counter = new PercentileCounter(100, 0.5); Random random = new Random(); for (int i = 0; i < 1000; i++) { counter.add(random.nextDouble()); } double markResult = counter.getResultEstimate(); ByteBuffer buffer = ByteBuffer.allocateDirect(serializer.getStorageBytesEstimate()); serializer.serialize(counter, buffer); buffer.flip(); counter = serializer.deserialize(buffer); PercentileCounter counter1 = new PercentileCounter(100, 0.5); counter1.merge(counter); assertEquals(markResult, counter1.getResultEstimate(), 0.01); }
@Test public void testSerialization() { Kryo kryo = new Kryo(); kryo.register(PercentileCounter.class, new PercentileCounterSerializer()); double compression = 100; double quantile = 0.8; PercentileCounter origin_counter = new PercentileCounter(compression, quantile); for (int i = 1; i < 10; i++) { origin_counter.add(i); } byte[] buffer = serialize(kryo, origin_counter); PercentileCounter deserialized_counter = deserialize(kryo, buffer, PercentileCounter.class); Assert.assertEquals("Compression Error", origin_counter.getCompression(), deserialized_counter.getCompression(), 0.00000001); Assert.assertEquals("QuantileRatio Error", origin_counter.getQuantileRatio(), deserialized_counter.getQuantileRatio(), 0.00000001); Assert.assertEquals("Estimation Error", origin_counter.getResultEstimate(), deserialized_counter.getResultEstimate(), 0.00000001); }
@Override public PercentileCounter aggregate(PercentileCounter value1, PercentileCounter value2) { if (value1 == null) { return new PercentileCounter(value2); } else if (value2 == null) { return new PercentileCounter(value1); } PercentileCounter merged = new PercentileCounter(value1); merged.merge(value2); return merged; }
private PercentileCounter current() { if (current == null) { current = new ThreadLocal<>(); } PercentileCounter counter = current.get(); if (counter == null) { counter = new PercentileCounter(compression); current.set(counter); } return counter; }
@Override public void aggregate(PercentileCounter value) { if (sum == null) sum = new PercentileCounter(value); else sum.merge(value); }
public static PercentileCounter add(PercentileCounter counter, Object v, Object r) { PercentileCounter c = (PercentileCounter) v; Number n = (Number) r; if (counter == null) { counter = new PercentileCounter(c.compression, n.doubleValue()); } counter.merge(c); return counter; }
@Override public void reset() { current = new PercentileCounter(dataType.getPrecision()); } };