protected double calculateMaximumObservedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotations = annotationsPerCategory.get(category); int min = -1; for (int rater = 0; rater < study.getRaterCount(); rater++) if (annotations[rater] < min || min < 0) min = annotations[rater]; if (min > 0) result = result.add(new BigDecimal(min)); } result = result.divide(new BigDecimal(study.getItemCount()), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) result = result.add( new BigDecimal(annotationsPerCategory.get(category)).pow(2)); result = result.divide(new BigDecimal(4).multiply( new BigDecimal(study.getItemCount()).pow(2)), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes a * different probability distribution for all raters. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotations = annotationsPerCategory.get(category); BigDecimal prod = new BigDecimal(1); for (int rater = 0; rater < study.getRaterCount(); rater++) prod = prod.multiply(new BigDecimal(annotations[rater])); result = result.add(prod); } result = result.divide(new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>(); for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) { BigDecimal p = new BigDecimal(counts.getValue()).divide( new BigDecimal(item.getRaterCount()), MathContext.DECIMAL128); BigDecimal value = categoryProbability.get(counts.getKey()); if (value != null) p = p.add(value); categoryProbability.put(counts.getKey(), p); } } BigDecimal result = new BigDecimal(0); for (BigDecimal p : categoryProbability.values()) result = result.add(p.pow(2)); result = result.divide( new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotationCounts = annotationsPerCategory.get(category); for (int m = 0; m < study.getRaterCount(); m++) for (int n = m + 1; n < study.getRaterCount(); n++) result = result.add(new BigDecimal(annotationCounts[m]) .multiply(new BigDecimal(annotationCounts[n]))); } result = result.multiply(new BigDecimal(2)); result = result.divide(new BigDecimal(study.getRaterCount()) .multiply(new BigDecimal(study.getRaterCount() - 1)) .multiply(new BigDecimal(study.getItemCount()).pow(2)), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the observed inter-rater agreement for the annotation * study that was passed to the class constructor and the currently * assigned distance function. * @throws NullPointerException if the study is null. * @throws ArithmeticException if the study does not contain any item or * the number of raters is smaller than 2. */ public double calculateObservedDisagreement() { ensureDistanceFunction(); double result = 0.0; for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet()) for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) { if (category1.getValue() == null) continue; if (category2.getValue() == null) continue; result += category1.getValue() * category2.getValue() * distanceFunction.measureDistance(study, category1.getKey(), category2.getKey()); } } result /= (double) (study.getItemCount() * study.getRaterCount() * (study.getRaterCount() - 1)); return result; }
public static void dumpStudy(PrintStream aOut, ICodingAnnotationStudy aStudy) { try { aOut.printf("Category count: %d%n", aStudy.getCategoryCount()); } catch (Throwable e) { aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } try { aOut.printf("Item count: %d%n", aStudy.getItemCount()); } catch (Throwable e) { aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } for (ICodingAnnotationItem item : aStudy.getItems()) { StringBuilder sb = new StringBuilder(); for (IAnnotationUnit unit : item.getUnits()) { if (sb.length() > 0) { sb.append(" \t"); } sb.append(unit.getCategory()); } aOut.println(sb); } }
/** Calculates the expected inter-rater agreement using the defined * distance function to infer the assumed probability distribution. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items or raters in the * annotation study. */ public double calculateExpectedDisagreement() { ensureDistanceFunction(); double result = 0.0; Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study); for (Object category1 : study.getCategories()) for (Object category2 : study.getCategories()) result += annotationsPerCategory.get(category1) * annotationsPerCategory.get(category2) * distanceFunction.measureDistance(study, category1, category2); result /= (double) (study.getItemCount() * study.getRaterCount() * (study.getItemCount() * study.getRaterCount() - 1)); return result; }
public static AgreementResult getAgreement(ConcreteAgreementMeasure aMeasure, boolean aExcludeIncomplete, DiffResult aDiff, String aType, String aFeature, Map<String, List<JCas>> aCasMap) { if (aCasMap.size() != 2) { throw new IllegalArgumentException("CAS map must contain exactly two CASes"); } AgreementResult agreementResult = AgreementUtils.makeStudy(aDiff, aType, aFeature, aExcludeIncomplete, aCasMap); try { IAgreementMeasure agreement = aMeasure.make(agreementResult.study); if (agreementResult.study.getItemCount() > 0) { agreementResult.setAgreement(agreement.calculateAgreement()); } else { agreementResult.setAgreement(Double.NaN); } return agreementResult; } catch (RuntimeException e) { // FIXME AgreementUtils.dumpAgreementStudy(System.out, agreementResult); throw e; } }
for (int i = 0; i < study.getItemCount(); i++) out.print(DIVIDER + (i + 1)); out.print(DIVIDER + "Σ");
public double calculateCategoryAgreement(final Object category) { // N = # subjects = #items -> index i // n = # ratings/subject = #raters // k = # categories -> index j // n_ij = # raters that annotated item i as category j // // k_j = (P_j - p_j) / (1 - p_j) // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j ) // p_j = 1/Nn sum n_ij int N = study.getItemCount(); int n = study.getRaterCount(); int sum_nij = 0; int sum_nij_2 = 0; for (ICodingAnnotationItem item : study.getItems()) { int nij = 0; for (IAnnotationUnit annotation : item.getUnits()) if (annotation.getCategory().equals(category)) nij++; sum_nij += nij; sum_nij_2 += (nij * nij); } double pj = 1 / (double) (N * n) * sum_nij; double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj); double kappaj = (Pj - pj) / (double) (1 - pj); return kappaj; }
@Override public double calculateCategoryAgreement(final Object category) { // N = # subjects = #items -> index i // n = # ratings/subject = #raters // k = # categories -> index j // n_ij = # raters that annotated item i as category j // // k_j = (P_j - p_j) / (1 - p_j) // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j ) // p_j = 1/Nn sum n_ij int N = study.getItemCount(); int n = study.getRaterCount(); int sum_nij = 0; int sum_nij_2 = 0; for (ICodingAnnotationItem item : study.getItems()) { int nij = 0; for (IAnnotationUnit unit : item.getUnits()) if (unit.getCategory().equals(category)) nij++; sum_nij += nij; sum_nij_2 += (nij * nij); } double pj = 1 / (double) (N * n) * sum_nij; double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj); double kappaj = (Pj - pj) / (double) (1 - pj); return kappaj; }
public static void dumpAgreementStudy(PrintStream aOut, AgreementResult aAgreement) { try { aOut.printf("Category count: %d%n", aAgreement.getStudy().getCategoryCount()); } catch (Throwable e) { aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } try { aOut.printf("Item count: %d%n", aAgreement.getStudy().getItemCount()); } catch (Throwable e) { aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } aOut.printf("Relevant position count: %d%n", aAgreement.getRelevantSetCount()); aOut.printf("%n== Complete sets: %d ==%n", aAgreement.getCompleteSets().size()); dumpAgreementConfigurationSetsWithItems(aOut, aAgreement, aAgreement.getCompleteSets()); aOut.printf("%n== Incomplete sets (by position): %d == %n", aAgreement.getIncompleteSetsByPosition().size()); dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getIncompleteSetsByPosition()); aOut.printf("%n== Incomplete sets (by label): %d ==%n", aAgreement.getIncompleteSetsByLabel().size()); dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getIncompleteSetsByLabel()); aOut.printf("%n== Plurality sets: %d ==%n", aAgreement.getPluralitySets().size()); dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getPluralitySets()); }
result += catCount * (catCount - 1) + otherCatCount * (otherCatCount - 1); return result / (double) (study.getItemCount() * study.getRaterCount() * (study.getRaterCount() - 1));
nNullCategorySum += nNullCategory; observedDisagreement /= (double) study.getItemCount() * study.getRaterCount() * (study.getRaterCount() - 1); + nNullCategorySum * nKeepCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, category) + nNullCategorySum * nNullCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, NULL_CATEGORY); expectedDisagreement /= (double) study.getItemCount() * study.getRaterCount() * (study.getItemCount() * study.getRaterCount() - 1);