/** * {@inheritDoc} */ @Override public synchronized double getStandardDeviation() { return super.getStandardDeviation(); }
/** * filter snapshots using statistically significant points only * * @param selectedLists list of snapshots */ private void statBasedSelectMetricHistory(final LinkedList<InMemoryHistoryNode> selectedLists) throws ClassCastException { logger.debug("selecting snapshots which are far away from mean value"); final DescriptiveStatistics descStats = getDescriptiveStatistics(selectedLists); final Double mean = descStats.getMean(); final Double std = descStats.getStandardDeviation(); final Iterator<InMemoryHistoryNode> ite = selectedLists.iterator(); while (ite.hasNext()) { final InMemoryHistoryNode currentNode = ite.next(); final double value = ((Number) currentNode.getValue()).doubleValue(); // remove all elements which lies in 95% value band if (value < mean + this.standardDeviationFactor * std && value > mean - this.standardDeviationFactor * std) { ite.remove(); } } }
/** * Produce results in printable CSV format, separated by separator. * @param fmt text formatter - destination of formatted results. * @param name benchmark name * @param stats benchmark data * @param separator field separator */ private void displayCSV(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats, @NotNull String separator) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%s%s%g%s%g%s%g%s%g%s%g%n", name, separator, mean / conv, separator, median(stats) / conv, separator, stats.getMin() / conv, separator, stats.getMax() / conv, separator, err); }
/** * Produce printable result * @param fmt text formatter - destination of formatted results. * @param name benchmark name * @param stats benchmark data */ private void displayStats(@NotNull Formatter fmt, @NotNull String name, @NotNull DescriptiveStatistics stats) { double mean = stats.getMean(); double err = stats.getStandardDeviation() / mean * 100; long conv = scale.toNanos(1); fmt.format("%-30s %-8.4g %-8.4g %-8.4g %-8.4g %-8.4g%n", name, mean / conv, median(stats) / conv, stats.getMin() / conv, stats.getMax() / conv, err); }
/** * Generates a text report displaying univariate statistics from values * that have been added. Each statistic is displayed on a separate * line. * * @return String with line feeds displaying statistics */ @Override public String toString() { StringBuilder outBuffer = new StringBuilder(); String endl = "\n"; outBuffer.append("DescriptiveStatistics:").append(endl); outBuffer.append("n: ").append(getN()).append(endl); outBuffer.append("min: ").append(getMin()).append(endl); outBuffer.append("max: ").append(getMax()).append(endl); outBuffer.append("mean: ").append(getMean()).append(endl); outBuffer.append("std dev: ").append(getStandardDeviation()) .append(endl); try { // No catch for MIAE because actual parameter is valid below outBuffer.append("median: ").append(getPercentile(50)).append(endl); } catch (MathIllegalStateException ex) { outBuffer.append("median: unavailable").append(endl); } outBuffer.append("skewness: ").append(getSkewness()).append(endl); outBuffer.append("kurtosis: ").append(getKurtosis()).append(endl); return outBuffer.toString(); }
static SimpleStats from(DescriptiveStatistics stats) { SimpleStats ss = new SimpleStats(); ss._min = stats.getMin(); ss._max = stats.getMax(); ss._mean = stats.getMean(); ss._std = stats.getStandardDeviation(); ss._median = stats.getPercentile(50); return ss; }
/** * Normalize (standardize) the sample, so it is has a mean of 0 and a standard deviation of 1. * * @param sample Sample to normalize. * @return normalized (standardized) sample. * @since 2.2 */ public static double[] normalize(final double[] sample) { DescriptiveStatistics stats = new DescriptiveStatistics(); // Add the data from the series to stats for (int i = 0; i < sample.length; i++) { stats.addValue(sample[i]); } // Compute mean and standard deviation double mean = stats.getMean(); double standardDeviation = stats.getStandardDeviation(); // initialize the standardizedSample, which has the same length as the sample double[] standardizedSample = new double[sample.length]; for (int i = 0; i < sample.length; i++) { // z = (x- mean)/standardDeviation standardizedSample[i] = (sample[i] - mean) / standardDeviation; } return standardizedSample; }
System.out.println("Compilation took " + stats.getMean() + "ms ± " + stats.getStandardDeviation() + "ms"); FileWriter wrt = new FileWriter(new File("target/compilation-stats.csv"), true); wrt.append(String.format("%s;%s;%s\n", GROOVY_VERSION, stats.getMean(), stats.getStandardDeviation())); wrt.close();
private void logStats(Set<?> set, double[] raw) { DescriptiveStatistics descStats = new DescriptiveStatistics(raw); Log.i( TAG, String.format( Locale.US, STAT_FORMAT, set.getClass().getSimpleName(), set.size(), descStats.getN(), Math.round(descStats.getMin()), Math.round(descStats.getMax()), Math.round(descStats.getMean()), descStats.getStandardDeviation(), descStats.getVariance() ) ); } }
@Test(dataProvider = "provideStandardErrors") public void testMultiplePositions(double maxStandardError) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < 500; ++i) { int uniques = ThreadLocalRandom.current().nextInt(getUniqueValuesCount()) + 1; List<Object> values = createRandomSample(uniques, (int) (uniques * 1.5)); long actual = estimateGroupByCount(values, maxStandardError); double error = (actual - uniques) * 1.0 / uniques; stats.addValue(error); } assertLessThan(stats.getMean(), 1.0e-2); assertLessThan(stats.getStandardDeviation(), 1.0e-2 + maxStandardError); }
@Override public double getStandardDeviation() { return descStats.getStandardDeviation(); }
@Override public void run() { DescriptiveStatistics stats = limiter.getRateStatsSinceLastReport(); if (stats != null) { log.info(String.format("Requests rate stats: count: %d, min: %f, max: %f, mean: %f, std: %f, sum: %f", stats.getN(), stats.getMin(), stats.getMax(), stats.getMean(), stats.getStandardDeviation(), stats.getSum())); } stats = limiter.getUnusedPermitsSinceLastReport(); if (stats != null) { log.info(String.format("Unused permits rate stats: count: %d, min: %f, max: %f, mean: %f, std: %f, sum: %f", stats.getN(), stats.getMin(), stats.getMax(), stats.getMean(), stats.getStandardDeviation(), stats.getSum())); } if (this.policy instanceof QPSPolicy) { QPSPolicy qpsPolicy = (QPSPolicy) this.policy; DynamicTokenBucket dynamicTokenBucket = qpsPolicy.getTokenBucket(); TokenBucket tokenBucket = dynamicTokenBucket.getTokenBucket(); log.info("Stored tokens: " + tokenBucket.getStoredTokens()); } } }
public String getStd() { return formatter.apply(stats.getStandardDeviation()); } }
public static String describe(DescriptiveStatistics stats, Double[] percentiles){ StringBuilder sb = new StringBuilder(); sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", (long)stats.getMean(), (long)stats.getStandardDeviation(), stats.getN() )); sb.append("\tMin - " + (long)stats.getMin() + "\n"); for(double pctile : percentiles) { sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); } sb.append("\tMax - " + (long)stats.getMax()); return sb.toString(); }
public void printResults() { printf("%-20s%20s%20s%20s\n", "Test Name", format.name + " Avg", "StdDev", "95% Interval"); for (int i = 0; i < numTests; i++) { double mean = stats[i].getMean() / format.divisor; double stddev = stats[i].getStandardDeviation() / format.divisor; double diff = 1.96 * stddev / MathUtils.sqrt(stats[i].getN()); printf("%-20s%20.3f%20.3f (%7.3f,%7.3f)\n", getTestName(i), mean, stddev, mean - diff, mean + diff); } }
@Test(groups = "slow") public void testError() throws Exception { DescriptiveStatistics stats = new DescriptiveStatistics(); int buckets = 2048; for (int i = 0; i < 10000; ++i) { HyperLogLog estimator = new HyperLogLog(buckets); Set<Long> randomSet = makeRandomSet(5 * buckets); for (Long value : randomSet) { estimator.add(value); } double error = (estimator.estimate() - randomSet.size()) * 1.0 / randomSet.size(); stats.addValue(error); } assertTrue(stats.getMean() < 1e-2); assertTrue(stats.getStandardDeviation() < 1.04 / Math.sqrt(buckets)); }
public static void summary(String title, DescriptiveStatistics statistics, PrintStream pw, boolean meanOnly) { if(meanOnly) { pw.println(title + ": " + "\n\tMean: " + statistics.getMean() ); } else { pw.println(title + ": " + "\n\tMean: " + statistics.getMean() + "\n\tMin: " + statistics.getMin() + "\n\t1th: " + statistics.getPercentile(1) + "\n\t5th: " + statistics.getPercentile(5) + "\n\t10th: " + statistics.getPercentile(10) + "\n\t25th: " + statistics.getPercentile(25) + "\n\t50th: " + statistics.getPercentile(50) + "\n\t90th: " + statistics.getPercentile(90) + "\n\t95th: " + statistics.getPercentile(95) + "\n\t99th: " + statistics.getPercentile(99) + "\n\tMax: " + statistics.getMax() + "\n\tStdDev: " + statistics.getStandardDeviation() ); } } public void printDepthSummary(int depth, boolean meanOnly) {
@Test public void testSummarize() { IntColumn c = IntColumn.indexColumn("t", 99, 1); IntColumn c2 = c.copy(); c2.appendCell(""); double c2Variance = c2.variance(); double cVariance = StatUtils.variance(c.asDoubleArray()); assertEquals(cVariance, c2Variance, 0.00001); assertEquals(StatUtils.sumLog(c.asDoubleArray()), c2.sumOfLogs(), 0.00001); assertEquals(StatUtils.sumSq(c.asDoubleArray()), c2.sumOfSquares(), 0.00001); assertEquals(StatUtils.geometricMean(c.asDoubleArray()), c2.geometricMean(), 0.00001); assertEquals(StatUtils.product(c.asDoubleArray()), c2.product(), 0.00001); assertEquals(StatUtils.populationVariance(c.asDoubleArray()), c2.populationVariance(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getQuadraticMean(), c2.quadraticMean(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getStandardDeviation(), c2.standardDeviation(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getKurtosis(), c2.kurtosis(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getSkewness(), c2.skewness(), 0.00001); assertEquals(StatUtils.variance(c.asDoubleArray()), c.variance(), 0.00001); assertEquals(StatUtils.sumLog(c.asDoubleArray()), c.sumOfLogs(), 0.00001); assertEquals(StatUtils.sumSq(c.asDoubleArray()), c.sumOfSquares(), 0.00001); assertEquals(StatUtils.geometricMean(c.asDoubleArray()), c.geometricMean(), 0.00001); assertEquals(StatUtils.product(c.asDoubleArray()), c.product(), 0.00001); assertEquals(StatUtils.populationVariance(c.asDoubleArray()), c.populationVariance(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getQuadraticMean(), c.quadraticMean(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getStandardDeviation(), c.standardDeviation(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getKurtosis(), c.kurtosis(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getSkewness(), c.skewness(), 0.00001); }
private double standardDeviation(double[][] x) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < x.length; i ++) for (int j = 0; j < x[i].length; j++) stats.addValue(x[i][j]); return stats.getStandardDeviation(); }
private double standardDeviation(double[][] x) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < x.length; i ++) for (int j = 0; j < x[i].length; j++) stats.addValue(x[i][j]); return stats.getStandardDeviation(); }