@Override public int getUnitCount() { int result = 0; for (ICodingAnnotationItem item : items) result += item.getRaterCount(); return result; //return items.size() * raters.size(); }
@Override public boolean hasMissingValues() { for (ICodingAnnotationItem item : items) if (item.getRaterCount() != raters.size()) return true; return false; }
protected double doCalculateItemAgreement(final ICodingAnnotationItem item) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); double result = 0.0; for (Integer count : annotationsPerCategory.values()) result += count * (count - 1); int raterCount = item.getRaterCount(); if (raterCount <= 1) return 0.0; else return result / (double) (raterCount - 1.0); }
/** Calculates the inter-rater agreement for the given annotation item. * This is the basic step that is performed for each item of an * annotation study, when calling {@link #calculateAgreement()}. * @throws NullPointerException if the given item is null. */ public double calculateItemAgreement(final ICodingAnnotationItem item) { return doCalculateItemAgreement(item) / item.getRaterCount(); }
@Override public double calculateObservedAgreement() { double result = 0.0; double denominator = 0.0; for (ICodingAnnotationItem item : study.getItems()) { int raterCount = item.getRaterCount(); if (raterCount > 1) { result += doCalculateItemAgreement(item); denominator += raterCount; } } return result / denominator; }
/** Returns a map of categories and their usage frequencies (i.e., * how often they are used in annotation units) within the given * annotation study. */ // Category -> # public static Map<Object, Integer> countTotalAnnotationsPerCategory( final ICodingAnnotationStudy study) { Map<Object, Integer> result = new HashMap<Object, Integer>(); for (ICodingAnnotationItem item : study.getItems()) { if (item.getRaterCount() <= 1) continue; for (IAnnotationUnit unit : item.getUnits()) { Object category = unit.getCategory(); if (category == null) continue; Integer count = result.get(category); if (count == null) result.put(category, 1); else result.put(category, count + 1); } } return result; }
int raterCount = item.getRaterCount(); for (Map<Object, Double> cat2 : result.values()) for (Entry<Object, Double> entry : cat2.entrySet())
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>(); for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) { BigDecimal p = new BigDecimal(counts.getValue()).divide( new BigDecimal(item.getRaterCount()), MathContext.DECIMAL128); BigDecimal value = categoryProbability.get(counts.getKey()); if (value != null) p = p.add(value); categoryProbability.put(counts.getKey(), p); } } BigDecimal result = new BigDecimal(0); for (BigDecimal p : categoryProbability.values()) result = result.add(p.pow(2)); result = result.divide( new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }