/** Print the coincidence matrix for the given annotation item. */ public void print(final PrintStream out, final ICodingAnnotationStudy study, final ICodingAnnotationItem item) { Map<Object, Map<Object, Double>> coincidence = CodingAnnotationStudy.countCategoryCoincidence(item); doPrint(out, study, coincidence); }
/** Print the coincidence matrix for the given coding study. */ public void print(final PrintStream out, final ICodingAnnotationStudy study) { Map<Object, Map<Object, Double>> coincidence = CodingAnnotationStudy.countCategoryCoincidence(study); doPrint(out, study, coincidence); }
@Override public double calculateObservedDisagreement() { ensureDistanceFunction(); if (coincidenceMatrix == null) coincidenceMatrix = CodingAnnotationStudy.countCategoryCoincidence(study); double n = 0.0; double result = 0.0; for (Entry<Object, Map<Object, Double>> cat1 : coincidenceMatrix.entrySet()) for (Entry<Object, Double> cat2 : cat1.getValue().entrySet()) { result += cat2.getValue() * distanceFunction.measureDistance(study, cat1.getKey(), cat2.getKey()); n += cat2.getValue(); } result /= n; return result; }
/** Returns a two dimensional map of category pairs and their co-occurrence * frequencies for the given annotation study. */ // Category x Category -> # public static Map<Object, Map<Object, Double>> countCategoryCoincidence(final ICodingAnnotationStudy study) { Map<Object, Map<Object, Double>> result = new HashMap<Object, Map<Object, Double>>(); for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Map<Object, Double>> itemMatrix = countCategoryCoincidence(item); for (Entry<Object, Map<Object, Double>> itemCat : itemMatrix.entrySet()) { Map<Object, Double> resultCat = result.get(itemCat.getKey()); if (resultCat == null) { resultCat = new HashMap<Object, Double>(); result.put(itemCat.getKey(), resultCat); } for (Entry<Object, Double> itemEntry : itemCat.getValue().entrySet()) { Double resultEntry = resultCat.get(itemEntry.getKey()); if (resultEntry == null) resultEntry = 0.0; resultCat.put(itemEntry.getKey(), resultEntry + itemEntry.getValue()); } } } return result; }
/** Calculates the expected inter-rater agreement using the defined * distance function to infer the assumed probability distribution. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items or raters in the * annotation study. */ public double calculateExpectedDisagreement() { ensureDistanceFunction(); if (coincidenceMatrix == null) coincidenceMatrix = CodingAnnotationStudy.countCategoryCoincidence(study); double n = 0.0; Map<Object, Double> marginals = new HashMap<Object, Double>(); for (Entry<Object, Map<Object, Double>> cat1 : coincidenceMatrix.entrySet()) { double n_c = 0.0; for (Entry<Object, Double> cat2 : cat1.getValue().entrySet()) n_c += cat2.getValue(); marginals.put(cat1.getKey(), n_c); n += n_c; } double result = 0.0; for (Entry<Object, Double> cat1 : marginals.entrySet()) for (Entry<Object, Double> cat2 : marginals.entrySet()) result += cat1.getValue() * cat2.getValue() * distanceFunction.measureDistance(study, cat1.getKey(), cat2.getKey()); result /= n * (n - 1.0); return result; }
public double calculateItemAgreement(final ICodingAnnotationItem item) { ensureDistanceFunction(); Map<Object, Map<Object, Double>> itemMatrix = CodingAnnotationStudy.countCategoryCoincidence(item); coincidenceMatrix = CodingAnnotationStudy.countCategoryCoincidence(study); n = 0.0; Map<Object, Double> marginals = new TreeMap<Object, Double>();