@Test public void doesNotDownsampleSmallSample() { MultisetStatistics stats = new MultisetStatistics(); stats.addValue(3.14, 1); Mockito.when(mockResult.getStatistics()).thenReturn(stats); assertThat(PerformanceResults.getData(mockRunResult)).containsExactlyElementsOf(ImmutableList.of(3.14)); }
@Test public void doesNotDownsampleSampleMaxsizeSample() { MultisetStatistics stats = new MultisetStatistics(); for (double number : SMALL_SAMPLE) { stats.addValue(number, 1); } Mockito.when(mockResult.getStatistics()).thenReturn(stats); assertThat(PerformanceResults.getData(mockRunResult)).containsExactlyElementsOf(SMALL_SAMPLE); }
@Test public void downSamplesCorrectlyWhenUniform() { MultisetStatistics stats = new MultisetStatistics(); for (int i = 0; i < 10; ++i) { for (double number : SMALL_SAMPLE) { stats.addValue(number, 2); } } Mockito.when(mockResult.getStatistics()).thenReturn(stats); assertThat(PerformanceResults.getData(mockRunResult)).containsExactlyElementsOf(SMALL_SAMPLE); }
@Test public void downSampledDistributionIsRepresentativeForReasonableData() { MultisetStatistics stats = new MultisetStatistics(); for (double number : LARGE_SAMPLE) { int elements = (int) (10 * PerformanceResults.DOWNSAMPLE_MAXIMUM_SIZE / (Math.abs(PerformanceResults.DOWNSAMPLE_MAXIMUM_SIZE - number) + 1)); stats.addValue(number, elements); } Mockito.when(mockResult.getStatistics()).thenReturn(stats); List<Double> downSampledData = PerformanceResults.getData(mockRunResult); MultisetStatistics downSampledStats = new MultisetStatistics(); for (double number : downSampledData) { downSampledStats.addValue(number, 1); } // Hypothesis that means are the same cannot be rejected with confidence more than 0.5 assertThat(TestUtils.tTest(stats, downSampledStats, 1 - 0.5)).isEqualTo(false); // The typical p value is 0.05, but I went with 0.5 because I can assertThat(TestUtils.homoscedasticTTest(stats, downSampledStats)).isGreaterThan(0.5d); }