@Override protected double test(double[] sample1, double[] sample2) { return pairedTTest(sample1, sample2); } }
@Test public void downSampledDistributionIsRepresentativeForReasonableData() { MultisetStatistics stats = new MultisetStatistics(); for (double number : LARGE_SAMPLE) { int elements = (int) (10 * PerformanceResults.DOWNSAMPLE_MAXIMUM_SIZE / (Math.abs(PerformanceResults.DOWNSAMPLE_MAXIMUM_SIZE - number) + 1)); stats.addValue(number, elements); } Mockito.when(mockResult.getStatistics()).thenReturn(stats); List<Double> downSampledData = PerformanceResults.getData(mockRunResult); MultisetStatistics downSampledStats = new MultisetStatistics(); for (double number : downSampledData) { downSampledStats.addValue(number, 1); } // Hypothesis that means are the same cannot be rejected with confidence more than 0.5 assertThat(TestUtils.tTest(stats, downSampledStats, 1 - 0.5)).isEqualTo(false); // The typical p value is 0.05, but I went with 0.5 because I can assertThat(TestUtils.homoscedasticTTest(stats, downSampledStats)).isGreaterThan(0.5d); }
@Override protected double test(double[] sample1, double[] sample2) { return homoscedasticT(sample1, sample2); } }
p = TestUtils.tTest(baselineValues, testValues); } else { double[] baselineValues = new double[dimensions.size()]; p = TestUtils.pairedTTest(baselineValues, testValues); } else if ("wilcoxon".equals(method)) { p = new WilcoxonSignedRankTest().wilcoxonSignedRankTest(baselineValues, testValues, false);
@Override public double test(FeatureValues<String> values) { Set<String> nominalValuesSet = new HashSet<>(); Map<String, Map<String, Integer>> conditionsCounts = new HashMap<>(); for(String condition : values.getConditionNames()) { for(String value : values.getConditionValues(condition)) { nominalValuesSet.add(value); conditionsCounts.putIfAbsent(condition, new HashMap<>()); conditionsCounts.get(condition).put( value, conditionsCounts.get(condition).getOrDefault(value, 0) + 1 ); } } String[] nominalValues = nominalValuesSet .toArray(new String[nominalValuesSet.size()]); int conditionNamesLength = values.getConditionNames().length; long[][] counts = new long[nominalValues.length][conditionNamesLength]; for (int column = 0; column < conditionNamesLength; column++) { for (int row = 0; row < nominalValues.length; row++) { counts[row][column] = conditionsCounts .get(values.getConditionNames()[column]) .getOrDefault(nominalValues[row], 0); } } return chiSquareTest(counts); }
@Override protected double test(double[] sample1, double[] sample2) { return tTest(sample1, sample2); } }
@Override public double test(FeatureValues<Number> values) { List<double[]> data = new LinkedList<>(); for (String condition : values.getConditionNames()) { data.add(asDoubleArray(values.getConditionValues(condition))); } return oneWayAnovaPValue(data); }
boolean chiSquare(List<? extends Number> expectations, List<? extends Number> observations, double confidence) { final double chi = TestUtils.chiSquareTest(Doubles.toArray(expectations), Longs.toArray(observations)); return !(chi < confidence); }
/** * Perform a paired T-test on the F-Measures of two sets of values. * @param result1 The first value set. * @param result2 The second value set. * @param alpha The alpha value. * @return <code>true</code> iff the null hypothesis can be rejected with confidence <code>1 - alpha</code>. */ public static boolean performPairedTTest(double[] result1, double[] result2, double alpha) { // Null hypothesis - the hypothesis that there is no significant difference between specified populations, // any observed difference being due to sampling or experimental error. // Returns true iff the null hypothesis can be rejected with confidence 1 - alpha. return TestUtils.pairedTTest(result1, result2, alpha); }
/** * Checks whether the observations conform to a Poisson process with the * specified intensity. Uses a chi square test with the specified confidence. * The null hypothesis is that the observations are the result of a poisson * process. * @param observations * @param intensity * @param confidence * @return <code>true</code> if the observations */ static boolean isPoissonProcess(Frequency observations, double intensity, double length, double confidence) { final PoissonDistribution pd = new PoissonDistribution(length * intensity); final Iterator<?> it = observations.valuesIterator(); final long[] observed = new long[observations.getUniqueCount()]; final double[] expected = new double[observations.getUniqueCount()]; int index = 0; while (it.hasNext()) { final Long l = (Long) it.next(); observed[index] = observations.getCount(l); expected[index] = pd.probability(l.intValue()) * observations.getSumFreq(); if (expected[index] == 0) { return false; } index++; } final double chi = TestUtils.chiSquareTest(expected, observed); return !(chi < confidence); }
/** * @param base int of the baseline to compare over * @param imp int of the alternative results that are tested for significant improvement * @return p-value of paired Student's T-test, 1-tailed. */ public double sigOver( int base, int imp ) { try { return TestUtils.pairedTTest(get(base).queryresult, get(imp).queryresult )/2; } catch (Exception ex) { log.fatalexception(ex, "sigOver( %d, %d)", base, imp); } return -2; } }
/** * @param base int of the baseline to compare over * @param imp int of the alternative results that are tested for significant improvement * @return p-value of paired Student's T-test, 1-tailed. */ public double sigOver( int base, int imp ) { try { return TestUtils.pairedTTest(get(base).queryresult, get(imp).queryresult )/2; } catch (Exception ex) { log.fatalexception(ex, "sigOver( %d, %d)", base, imp); } return -2; } }
public double getDouble(long... coordinates) { try { long var1 = coordinates[ROW]; long var2 = coordinates[COLUMN]; double[] sample1 = new double[(int) getSource().getRowCount()]; double[] sample2 = new double[(int) getSource().getRowCount()]; for (int r = 0; r < getSource().getRowCount(); r++) { sample1[r] = getSource().getAsDouble(r, var1); sample2[r] = getSource().getAsDouble(r, var2); } double pValue = TestUtils.pairedTTest(sample1, sample2); return pValue; } catch (Exception e) { throw new RuntimeException(e); } }
public double getDouble(long... coordinates) { try { long var1 = coordinates[ROW]; long var2 = coordinates[COLUMN]; double[] sample1 = new double[(int) getSource().getRowCount()]; double[] sample2 = new double[(int) getSource().getRowCount()]; for (int r = 0; r < getSource().getRowCount(); r++) { sample1[r] = getSource().getAsDouble(r, var1); sample2[r] = getSource().getAsDouble(r, var2); } double pValue = TestUtils.pairedTTest(sample1, sample2); return pValue; } catch (Exception e) { throw new RuntimeException(e); } }