@Override public double calculateObservedAgreement() { double result = 0.0; double denominator = 0.0; for (ICodingAnnotationItem item : study.getItems()) { int raterCount = item.getRaterCount(); if (raterCount > 1) { result += doCalculateItemAgreement(item); denominator += raterCount; } } return result / denominator; }