public synchronized double std() { return Math.sqrt(this.var()); }
@Test public void testMutiEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(); statistics.add(17); statistics.add(29); Assert.assertEquals(2, statistics.count(), TOL); Assert.assertEquals(23.0, statistics.mean(), TOL); Assert.assertEquals(36.0, statistics.var(), TOL); Assert.assertEquals(6.0, statistics.std(), TOL); Assert.assertEquals(29.0, statistics.outlier(1.0f), TOL); }
@Test public void testUpdateStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17); statistics.add(29); Assert.assertEquals(2, statistics.count(), TOL); Assert.assertEquals(23.0, statistics.mean(), TOL); Assert.assertEquals(36.0, statistics.var(), TOL); statistics.updateStatistics(17, 29); Assert.assertEquals(2, statistics.count(), TOL); Assert.assertEquals(29.0, statistics.mean(), TOL); Assert.assertEquals(0.0, statistics.var(), TOL); } }
public synchronized double var() { // E(X^2) - E(X)^2 if (count <= 1) { return 0.0; } double mean = mean(); return Math.max((sumSquares/count) - mean * mean, 0.0d); }
@Override public void contextualize(Configuration conf, AppContext context) { this.context = context; Map<JobId, Job> allJobs = context.getAllJobs(); for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) { final Job job = entry.getValue(); mapperStatistics.put(job, new DataStatistics()); reducerStatistics.put(job, new DataStatistics()); slowTaskRelativeTresholds.put (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f)); } }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
statistics.add(duration);
@Test public void testSingleEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17.29); Assert.assertEquals(1, statistics.count(), TOL); Assert.assertEquals(17.29, statistics.mean(), TOL); Assert.assertEquals(0, statistics.var(), TOL); Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(17.29, statistics.outlier(1.0f), TOL); }
public synchronized double var() { // E(X^2) - E(X)^2 if (count <= 1) { return 0.0; } double mean = mean(); return Math.max((sumSquares/count) - mean * mean, 0.0d); }
@Override public void contextualize(Configuration conf, AppContext context) { this.context = context; Map<JobId, Job> allJobs = context.getAllJobs(); for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) { final Job job = entry.getValue(); mapperStatistics.put(job, new DataStatistics()); reducerStatistics.put(job, new DataStatistics()); slowTaskRelativeTresholds.put (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f)); } }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
statistics.add(duration);
@Test public void testEmptyDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(); Assert.assertEquals(0, statistics.count(), TOL); Assert.assertEquals(0, statistics.mean(), TOL); Assert.assertEquals(0, statistics.var(), TOL); Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(0, statistics.outlier(1.0f), TOL); }
public synchronized double var() { // E(X^2) - E(X)^2 if (count <= 1) { return 0.0; } double mean = mean(); return Math.max((sumSquares/count) - mean * mean, 0.0d); }
@Override public void contextualize(Configuration conf, AppContext context) { this.context = context; Map<JobId, Job> allJobs = context.getAllJobs(); for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) { final Job job = entry.getValue(); mapperStatistics.put(job, new DataStatistics()); reducerStatistics.put(job, new DataStatistics()); slowTaskRelativeTresholds.put (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f)); } }
public synchronized double std() { return Math.sqrt(this.var()); }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
statistics.add(duration);