@Test public void testMergeSame() { NumericHistogram histogram = new NumericHistogram(10, 3); for (int i = 0; i < 1000; i++) { histogram.add(i); } Map<Double, Double> expected = Maps.transformValues(histogram.getBuckets(), value -> value * 2); histogram.mergeWith(histogram); assertEquals(histogram.getBuckets(), expected); }
public void add(double value, double weight) { if (nextIndex == values.length) { compact(); } values[nextIndex] = value; weights[nextIndex] = weight; nextIndex++; }
@Test public void testRoundtrip() { NumericHistogram histogram = new NumericHistogram(100, 20); for (int i = 0; i < 1000; i++) { histogram.add(i); } Slice serialized = histogram.serialize(); NumericHistogram deserialized = new NumericHistogram(serialized, 20); assertEquals(deserialized.getBuckets(), histogram.getBuckets()); }
@VisibleForTesting void compact() { nextIndex = mergeSameBuckets(values, weights, nextIndex); if (nextIndex <= maxBuckets) { return; } // entries are guaranteed to be sorted as a side-effect of the call to mergeSameBuckets store(mergeBuckets(values, weights, nextIndex, maxBuckets)); }
public void mergeWith(NumericHistogram other) { int count = nextIndex + other.nextIndex; double[] newValues = new double[count]; double[] newWeights = new double[count]; concat(newValues, this.values, this.nextIndex, other.values, other.nextIndex); concat(newWeights, this.weights, this.nextIndex, other.weights, other.nextIndex); count = mergeSameBuckets(newValues, newWeights, count); if (count <= maxBuckets) { // copy back into this.values/this.weights System.arraycopy(newValues, 0, this.values, 0, count); System.arraycopy(newWeights, 0, this.weights, 0, count); nextIndex = count; return; } sort(newValues, newWeights, count); store(mergeBuckets(newValues, newWeights, count, maxBuckets)); }
@Test public void testSameValues() { NumericHistogram histogram = new NumericHistogram(4); for (int i = 0; i < 100; ++i) { histogram.add(1.0, 3); histogram.add(2.0, 5); } Map<Double, Double> expected = ImmutableMap.<Double, Double>builder() .put(1.0, 300.0) .put(2.0, 500.0) .build(); assertEquals(histogram.getBuckets(), expected); }
@InputFunction public static void add(@AggregationState State state, @SqlType(BIGINT) long buckets, @SqlType(DOUBLE) double value, @SqlType(DOUBLE) double weight) { NumericHistogram histogram = state.get(); if (histogram == null) { checkCondition(buckets >= 2, INVALID_FUNCTION_ARGUMENT, "numeric_histogram bucket count must be greater than one"); histogram = new NumericHistogram(toIntExact(buckets), ENTRY_BUFFER_SIZE); state.set(histogram); } histogram.add(value, weight); }
@OutputFunction("map(real,real)") public static void output(@AggregationState DoubleHistogramAggregation.State state, BlockBuilder out) { if (state.get() == null) { out.appendNull(); } else { Map<Double, Double> value = state.get().getBuckets(); BlockBuilder entryBuilder = out.beginBlockEntry(); for (Map.Entry<Double, Double> entry : value.entrySet()) { REAL.writeLong(entryBuilder, floatToRawIntBits(entry.getKey().floatValue())); REAL.writeLong(entryBuilder, floatToRawIntBits(entry.getValue().floatValue())); } out.closeEntry(); } } }
@Override public void serialize(DoubleHistogramAggregation.State state, BlockBuilder out) { if (state.get() == null) { out.appendNull(); } else { VARBINARY.writeSlice(out, state.get().serialize()); } }
@CombineFunction public static void merge(@AggregationState State state, State other) { NumericHistogram input = other.get(); NumericHistogram previous = state.get(); if (previous == null) { state.set(input); } else { previous.mergeWith(input); } }
@Override public void deserialize(Block block, int index, DoubleHistogramAggregation.State state) { state.set(new NumericHistogram(VARBINARY.getSlice(block, index), DoubleHistogramAggregation.ENTRY_BUFFER_SIZE)); } }
public void add(double value) { add(value, 1); }
@Override public long getEstimatedSize() { if (histogram == null) { return 0; } return histogram.estimatedInMemorySize(); } }
PriorityQueue<Entry> queue = initializeQueue(values, weights, count);
@Test public void testSameValues() throws Exception { NumericHistogram histogram = new NumericHistogram(4); for (int i = 0; i < 100; ++i) { histogram.add(1.0, 3); histogram.add(2.0, 5); } Map<Double, Double> expected = ImmutableMap.<Double, Double>builder() .put(1.0, 300.0) .put(2.0, 500.0) .build(); assertEquals(histogram.getBuckets(), expected); }
public void mergeWith(NumericHistogram other) { int count = nextIndex + other.nextIndex; double[] newValues = new double[count]; double[] newWeights = new double[count]; concat(newValues, this.values, this.nextIndex, other.values, other.nextIndex); concat(newWeights, this.weights, this.nextIndex, other.weights, other.nextIndex); count = mergeSameBuckets(newValues, newWeights, count); if (count <= maxBuckets) { // copy back into this.values/this.weights System.arraycopy(newValues, 0, this.values, 0, count); System.arraycopy(newWeights, 0, this.weights, 0, count); nextIndex = count; return; } sort(newValues, newWeights, count); store(mergeBuckets(newValues, newWeights, count, maxBuckets)); }
@InputFunction public static void add(State state, @SqlType(BIGINT) long buckets, @SqlType(DOUBLE) double value, @SqlType(DOUBLE) double weight) { NumericHistogram histogram = state.get(); if (histogram == null) { checkCondition(buckets >= 2, INVALID_FUNCTION_ARGUMENT, "numeric_histogram bucket count must be greater than one"); histogram = new NumericHistogram(Ints.checkedCast(buckets), ENTRY_BUFFER_SIZE); state.set(histogram); } histogram.add(value, weight); }
@VisibleForTesting void compact() { nextIndex = mergeSameBuckets(values, weights, nextIndex); if (nextIndex <= maxBuckets) { return; } // entries are guaranteed to be sorted as a side-effect of the call to mergeSameBuckets store(mergeBuckets(values, weights, nextIndex, maxBuckets)); }
@OutputFunction("map(double,double)") public static void output(@AggregationState State state, BlockBuilder out) { if (state.get() == null) { out.appendNull(); } else { Map<Double, Double> value = state.get().getBuckets(); BlockBuilder entryBuilder = out.beginBlockEntry(); for (Map.Entry<Double, Double> entry : value.entrySet()) { DoubleType.DOUBLE.writeDouble(entryBuilder, entry.getKey()); DoubleType.DOUBLE.writeDouble(entryBuilder, entry.getValue()); } out.closeEntry(); } } }
@Override public void serialize(NumericHistogramAggregation.State state, BlockBuilder out) { if (state.get() == null) { out.appendNull(); } else { VARBINARY.writeSlice(out, state.get().serialize()); } }