/** * Normalize (standardize) the sample, so it is has a mean of 0 and a standard deviation of 1. * * @param sample Sample to normalize. * @return normalized (standardized) sample. * @since 2.2 */ public static double[] normalize(final double[] sample) { DescriptiveStatistics stats = new DescriptiveStatistics(); // Add the data from the series to stats for (int i = 0; i < sample.length; i++) { stats.addValue(sample[i]); } // Compute mean and standard deviation double mean = stats.getMean(); double standardDeviation = stats.getStandardDeviation(); // initialize the standardizedSample, which has the same length as the sample double[] standardizedSample = new double[sample.length]; for (int i = 0; i < sample.length; i++) { // z = (x- mean)/standardDeviation standardizedSample[i] = (sample[i] - mean) / standardDeviation; } return standardizedSample; }
static SimpleStats from(DescriptiveStatistics stats) { SimpleStats ss = new SimpleStats(); ss._min = stats.getMin(); ss._max = stats.getMax(); ss._mean = stats.getMean(); ss._std = stats.getStandardDeviation(); ss._median = stats.getPercentile(50); return ss; }
private void logStats(Set<?> set, double[] raw) { DescriptiveStatistics descStats = new DescriptiveStatistics(raw); Log.i( TAG, String.format( Locale.US, STAT_FORMAT, set.getClass().getSimpleName(), set.size(), descStats.getN(), Math.round(descStats.getMin()), Math.round(descStats.getMax()), Math.round(descStats.getMean()), descStats.getStandardDeviation(), descStats.getVariance() ) ); } }
@Override public Double summarize(NumericColumn<?> column) { return new DescriptiveStatistics(removeMissing(column)).getQuadraticMean(); } };
/** * Returns a copy of this DescriptiveStatistics instance with the same internal state. * * @return a copy of this */ public DescriptiveStatistics copy() { DescriptiveStatistics result = new DescriptiveStatistics(); // No try-catch or advertised exception because parms are guaranteed valid copy(this, result); return result; }
public static double getMedianHighOpenSpread(String symbol, int days) throws Exception { YahooQuoteFetcher yahooFetcher = new YahooQuoteFetcher(); String quoteStr = yahooFetcher.fetchQuotes(symbol, days, 60*60*24); List<Quote> dailyQuoteList = yahooFetcher.parseQuotes(quoteStr, 60*60*24); DescriptiveStatistics stats = new DescriptiveStatistics(); for(Quote quote : dailyQuoteList) { stats.addValue(quote.getHigh().subtract(quote.getOpen()).doubleValue()); } return stats.getPercentile(50); }
private DescriptiveStatistics getDescriptiveStatistics( final LinkedList<InMemoryHistoryNode> selectedLists) throws ClassCastException { final DescriptiveStatistics descStats = new DescriptiveStatistics(); for (final InMemoryHistoryNode node : selectedLists) { descStats.addValue(((Number) node.getValue()).doubleValue()); } return descStats; }
/** * Generates a text report displaying univariate statistics from values * that have been added. Each statistic is displayed on a separate * line. * * @return String with line feeds displaying statistics */ @Override public String toString() { StringBuilder outBuffer = new StringBuilder(); String endl = "\n"; outBuffer.append("DescriptiveStatistics:").append(endl); outBuffer.append("n: ").append(getN()).append(endl); outBuffer.append("min: ").append(getMin()).append(endl); outBuffer.append("max: ").append(getMax()).append(endl); outBuffer.append("mean: ").append(getMean()).append(endl); outBuffer.append("std dev: ").append(getStandardDeviation()) .append(endl); try { // No catch for MIAE because actual parameter is valid below outBuffer.append("median: ").append(getPercentile(50)).append(endl); } catch (MathIllegalStateException ex) { outBuffer.append("median: unavailable").append(endl); } outBuffer.append("skewness: ").append(getSkewness()).append(endl); outBuffer.append("kurtosis: ").append(getKurtosis()).append(endl); return outBuffer.toString(); }
private double standardDeviation(double[][] x) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < x.length; i ++) for (int j = 0; j < x[i].length; j++) stats.addValue(x[i][j]); return stats.getStandardDeviation(); }
@Override public void run() { DescriptiveStatistics stats = limiter.getRateStatsSinceLastReport(); if (stats != null) { log.info(String.format("Requests rate stats: count: %d, min: %f, max: %f, mean: %f, std: %f, sum: %f", stats.getN(), stats.getMin(), stats.getMax(), stats.getMean(), stats.getStandardDeviation(), stats.getSum())); } stats = limiter.getUnusedPermitsSinceLastReport(); if (stats != null) { log.info(String.format("Unused permits rate stats: count: %d, min: %f, max: %f, mean: %f, std: %f, sum: %f", stats.getN(), stats.getMin(), stats.getMax(), stats.getMean(), stats.getStandardDeviation(), stats.getSum())); } if (this.policy instanceof QPSPolicy) { QPSPolicy qpsPolicy = (QPSPolicy) this.policy; DynamicTokenBucket dynamicTokenBucket = qpsPolicy.getTokenBucket(); TokenBucket tokenBucket = dynamicTokenBucket.getTokenBucket(); log.info("Stored tokens: " + tokenBucket.getStoredTokens()); } } }
private void writeResults() throws IOException { writeMessageToFile( "Experiment Finished Sucessfully. Data " + max_processes + ", " + stable_time_period + ", " + stats.getMin() + ", " + stats.getMax() + ", " + stats.getMean() + ", " + stats.getVariance() + ", " + stats.getStandardDeviation() + ", " + (stats.getStandardDeviation() / Math.sqrt(stats.getN()))); writeMessageToFile("DataPoints: " + stable_time_period + " " + Arrays.toString(stats.getValues())); }
public void printResults() { printf("%-20s%20s%20s%20s\n", "Test Name", format.name + " Avg", "StdDev", "95% Interval"); for (int i = 0; i < numTests; i++) { double mean = stats[i].getMean() / format.divisor; double stddev = stats[i].getStandardDeviation() / format.divisor; double diff = 1.96 * stddev / MathUtils.sqrt(stats[i].getN()); printf("%-20s%20.3f%20.3f (%7.3f,%7.3f)\n", getTestName(i), mean, stddev, mean - diff, mean + diff); } }
private History(int windowSize) { this.samples = new DescriptiveStatistics(windowSize); }
public double findFrequency(double[] voltage, double samplingInterval) { int voltageLength = voltage.length; double[] frequency; double[] amplitude; int index = 0; double max = 0; Complex[] complex; DescriptiveStatistics stats = new DescriptiveStatistics(); for (int i = 0; i < voltageLength; i++) stats.addValue(voltage[i]); double voltageMean = stats.getMean(); for (int i = 0; i < voltageLength; i++) voltage[i] = voltage[i] - voltageMean; // remove DC component frequency = Arrays.copyOfRange(fftFrequency(voltageLength, samplingInterval), 0, voltageLength / 2); // take only the +ive half of the frequncy array FastFourierTransformer fastFourierTransformer = new FastFourierTransformer(DftNormalization.STANDARD); complex = fastFourierTransformer.transform(voltage, TransformType.FORWARD); amplitude = new double[complex.length / 2]; for (int i = 0; i < complex.length / 2; i++) { // take only the +ive half of the fft result amplitude[i] = complex[i].abs() / voltageLength; if (amplitude[i] > max) { // search for the tallest peak, the fundamental max = amplitude[i]; index = i; } } return frequency[index]; }
/** * Completes an append request to the member. * * @param time The time in milliseconds for the append. */ public void completeAppend(long time) { appending--; timeStats.addValue(time); }
@Test public void testTableSampleBernoulli() { DescriptiveStatistics stats = new DescriptiveStatistics(); int total = computeExpected("SELECT orderkey FROM orders", ImmutableList.of(BIGINT)).getMaterializedRows().size(); for (int i = 0; i < 100; i++) { List<MaterializedRow> values = computeActual("SELECT orderkey FROM orders TABLESAMPLE BERNOULLI (50)").getMaterializedRows(); assertEquals(values.size(), ImmutableSet.copyOf(values).size(), "TABLESAMPLE produced duplicate rows"); stats.addValue(values.size() * 1.0 / total); } double mean = stats.getGeometricMean(); assertTrue(mean > 0.45 && mean < 0.55, format("Expected mean sampling rate to be ~0.5, but was %s", mean)); }
/** * Get new statistics that excludes values beyond mean +/- 2 * stdev * * @param data Source data * @return new {@link @DescriptiveStatistics objects with sanitized data} */ private static DescriptiveStatistics sanitize(@NotNull DescriptiveStatistics data) { double meanValue = data.getMean(); double delta = MARGIN * meanValue; double minVal = meanValue - delta; double maxVal = meanValue + delta; return new DescriptiveStatistics(Arrays.stream(data.getValues()) .filter(x -> x > minVal && x < maxVal) .toArray()); }
DescriptiveStatistics statistics = new DescriptiveStatistics(); statistics.addValue(count); statistics.addValue(count); statistics.addValue(count); log.info("data: " + Arrays.toString(statistics.getValues())); log.info(statistics.toString()); String specifiedPartitions = Joiner.on(",").join(partitionPoints); log.info("Calculated specified partitions: " + specifiedPartitions);
@Test public void testSummarize() { IntColumn c = IntColumn.indexColumn("t", 99, 1); IntColumn c2 = c.copy(); c2.appendCell(""); double c2Variance = c2.variance(); double cVariance = StatUtils.variance(c.asDoubleArray()); assertEquals(cVariance, c2Variance, 0.00001); assertEquals(StatUtils.sumLog(c.asDoubleArray()), c2.sumOfLogs(), 0.00001); assertEquals(StatUtils.sumSq(c.asDoubleArray()), c2.sumOfSquares(), 0.00001); assertEquals(StatUtils.geometricMean(c.asDoubleArray()), c2.geometricMean(), 0.00001); assertEquals(StatUtils.product(c.asDoubleArray()), c2.product(), 0.00001); assertEquals(StatUtils.populationVariance(c.asDoubleArray()), c2.populationVariance(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getQuadraticMean(), c2.quadraticMean(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getStandardDeviation(), c2.standardDeviation(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getKurtosis(), c2.kurtosis(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getSkewness(), c2.skewness(), 0.00001); assertEquals(StatUtils.variance(c.asDoubleArray()), c.variance(), 0.00001); assertEquals(StatUtils.sumLog(c.asDoubleArray()), c.sumOfLogs(), 0.00001); assertEquals(StatUtils.sumSq(c.asDoubleArray()), c.sumOfSquares(), 0.00001); assertEquals(StatUtils.geometricMean(c.asDoubleArray()), c.geometricMean(), 0.00001); assertEquals(StatUtils.product(c.asDoubleArray()), c.product(), 0.00001); assertEquals(StatUtils.populationVariance(c.asDoubleArray()), c.populationVariance(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getQuadraticMean(), c.quadraticMean(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getStandardDeviation(), c.standardDeviation(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getKurtosis(), c.kurtosis(), 0.00001); assertEquals(new DescriptiveStatistics(c.asDoubleArray()).getSkewness(), c.skewness(), 0.00001); }
/** * Computes the current timeout. */ private void recomputeTimeoutMillis() { double nextTimeout = maxReplyTime.getAndSet(0) * TIMEOUT_MULTIPLIER; timeoutHistory.addValue( Math.min(Math.max(nextTimeout, MIN_TIMEOUT_MILLIS), MAX_TIMEOUT_MILLIS)); if (timeoutHistory.getN() == WINDOW_SIZE) { this.currentTimeout = (long) timeoutHistory.getMax(); } } }