public int getNonNullCount(String aCasGroupId) { int i = 0; for (ICodingAnnotationItem item : study.getItems()) { if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) { i++; } } return i; }
/** Returns a map of categories and their usage frequencies (i.e., * how often they are used in annotation units) within the given * annotation item. */ // Category -> # public static Map<Object, Integer> countTotalAnnotationsPerCategory( final ICodingAnnotationItem item) { Map<Object, Integer> result = new HashMap<Object, Integer>(); for (IAnnotationUnit unit : item.getUnits()) { Object category = unit.getCategory(); if (category == null) continue; Integer count = result.get(category); if (count == null) result.put(category, 1); else result.put(category, count + 1); } return result; }
/** Returns a map of categories and their usage frequencies (i.e., * how often they are used in annotation units) within the given * annotation study. */ // Category -> # public static Map<Object, Integer> countTotalAnnotationsPerCategory( final ICodingAnnotationStudy study) { Map<Object, Integer> result = new HashMap<Object, Integer>(); for (ICodingAnnotationItem item : study.getItems()) { if (item.getRaterCount() <= 1) continue; for (IAnnotationUnit unit : item.getUnits()) { Object category = unit.getCategory(); if (category == null) continue; Integer count = result.get(category); if (count == null) result.put(category, 1); else result.put(category, count + 1); } } return result; }
out.print(r + 1); for (ICodingAnnotationItem item : study.getItems()) out.print(DIVIDER + item.getUnit(r).getCategory()); out.println(); for (ICodingAnnotationItem item : study.getItems()) { int catCount = 0; for (IAnnotationUnit unit : item.getUnits()) if (category.equals(unit.getCategory())) catCount++;
@Override public int getUnitCount() { int result = 0; for (ICodingAnnotationItem item : items) result += item.getRaterCount(); return result; //return items.size() * raters.size(); }
@Override public boolean hasMissingValues() { for (ICodingAnnotationItem item : items) if (item.getRaterCount() != raters.size()) return true; return false; }
public boolean isAllNull(String aCasGroupId) { for (ICodingAnnotationItem item : study.getItems()) { if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) { return false; } } return true; }
private static void dumpAgreementConfigurationSetsWithItems(PrintStream aOut, AgreementResult aAgreement, List<ConfigurationSet> aSets) { int i = 0; for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) { StringBuilder sb = new StringBuilder(); sb.append(aSets.get(i).getPosition()); for (IAnnotationUnit unit : item.getUnits()) { if (sb.length() > 0) { sb.append(" \t"); } sb.append(unit.getCategory()); } aOut.println(sb); i++; } }
Map<Object, Map<Object, Double>> result = new HashMap<Object, Map<Object, Double>>(); for (IAnnotationUnit unit1 : item.getUnits()) for (IAnnotationUnit unit2 : item.getUnits()) { if (unit1 == unit2) continue; int raterCount = item.getRaterCount(); for (Map<Object, Double> cat2 : result.values()) for (Entry<Object, Double> entry : cat2.entrySet())
protected double doCalculateItemAgreement(final ICodingAnnotationItem item) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); double result = 0.0; for (Integer count : annotationsPerCategory.values()) result += count * (count - 1); int raterCount = item.getRaterCount(); if (raterCount <= 1) return 0.0; else return result / (double) (raterCount - 1.0); }
/** Returns a two-dimensional map of categories and raters and the * corresponding usage frequencies in the given annotation study * (i.e., how often a certain rater used a certain category for * coding an annotation unit). */ // Category x Rater -> # public static Map<Object, int[]> countAnnotationsPerCategory( final ICodingAnnotationStudy study) { Map<Object, int[]> result = new HashMap<Object, int[]>(); for (ICodingAnnotationItem item : study.getItems()) for (IAnnotationUnit unit : item.getUnits()) { Object category = unit.getCategory(); if (category == null) continue; int[] counts = result.get(category); if (counts == null) counts = new int[study.getRaterCount()]; counts[unit.getRaterIdx()]++; result.put(category, counts); } return result; }
/** Calculates the inter-rater agreement for the given annotation item. * This is the basic step that is performed for each item of an * annotation study, when calling {@link #calculateAgreement()}. * @throws NullPointerException if the given item is null. */ public double calculateItemAgreement(final ICodingAnnotationItem item) { return doCalculateItemAgreement(item) / item.getRaterCount(); }
/** Returns a clone of the current annotation study which contains * only the annotation units of the raters with the given indexes. * All other units will be removed. This method is useful for * converting an annotation study with multiple raters into a * (pairwise) annotation study with two raters. */ public CodingAnnotationStudy extractRaters(final int... raters) { CodingAnnotationStudy result = new CodingAnnotationStudy(raters.length); for (ICodingAnnotationItem item : getItems()) { CodingAnnotationItem newItem = new CodingAnnotationItem(raters.length); for (int r = 0; r < raters.length; r++) { IAnnotationUnit unit = item.getUnit(raters[r]); newItem.addUnit(result.createUnit(result.items.size(), r, unit.getCategory())); } result.items.add(newItem); } return result; }
public static void dumpStudy(PrintStream aOut, ICodingAnnotationStudy aStudy) { try { aOut.printf("Category count: %d%n", aStudy.getCategoryCount()); } catch (Throwable e) { aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } try { aOut.printf("Item count: %d%n", aStudy.getItemCount()); } catch (Throwable e) { aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e)); } for (ICodingAnnotationItem item : aStudy.getItems()) { StringBuilder sb = new StringBuilder(); for (IAnnotationUnit unit : item.getUnits()) { if (sb.length() > 0) { sb.append(" \t"); } sb.append(unit.getCategory()); } aOut.println(sb); } }
@Override public double calculateObservedAgreement() { double result = 0.0; double denominator = 0.0; for (ICodingAnnotationItem item : study.getItems()) { int raterCount = item.getRaterCount(); if (raterCount > 1) { result += doCalculateItemAgreement(item); denominator += raterCount; } } return result / denominator; }
ICodingAnnotationItem item2 = agreement.getStudy().getItem(1); ICodingAnnotationItem item3 = agreement.getStudy().getItem(2); assertEquals("", item1.getUnit(0).getCategory()); assertEquals("", item1.getUnit(1).getCategory()); assertEquals("", item2.getUnit(0).getCategory()); assertEquals(null, item2.getUnit(1).getCategory()); assertEquals(null, item3.getUnit(0).getCategory()); assertEquals("", item3.getUnit(1).getCategory());
private static void configurationSetsWithItemsToCsv(CSVPrinter aOut, AgreementResult aAgreement, List<ConfigurationSet> aSets) throws IOException { List<String> headers = new ArrayList<>( asList("Type", "Collection", "Document", "Layer", "Feature", "Position")); headers.addAll(aAgreement.getCasGroupIds()); aOut.printRecord(headers); int i = 0; for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) { Position pos = aSets.get(i).getPosition(); List<String> values = new ArrayList<>(); values.add(pos.getClass().getSimpleName()); values.add(pos.getCollectionId()); values.add(pos.getDocumentId()); values.add(pos.getType()); values.add(aAgreement.getFeature()); values.add(aSets.get(i).getPosition().toMinimalString()); for (IAnnotationUnit unit : item.getUnits()) { values.add(String.valueOf(unit.getCategory())); } aOut.printRecord(values); i++; } }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>(); for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) { BigDecimal p = new BigDecimal(counts.getValue()).divide( new BigDecimal(item.getRaterCount()), MathContext.DECIMAL128); BigDecimal value = categoryProbability.get(counts.getKey()); if (value != null) p = p.add(value); categoryProbability.put(counts.getKey(), p); } } BigDecimal result = new BigDecimal(0); for (BigDecimal p : categoryProbability.values()) result = result.add(p.pow(2)); result = result.divide( new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }
@Override public double calculateCategoryAgreement(final Object category) { // N = # subjects = #items -> index i // n = # ratings/subject = #raters // k = # categories -> index j // n_ij = # raters that annotated item i as category j // // k_j = (P_j - p_j) / (1 - p_j) // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j ) // p_j = 1/Nn sum n_ij int N = study.getItemCount(); int n = study.getRaterCount(); int sum_nij = 0; int sum_nij_2 = 0; for (ICodingAnnotationItem item : study.getItems()) { int nij = 0; for (IAnnotationUnit unit : item.getUnits()) if (unit.getCategory().equals(category)) nij++; sum_nij += nij; sum_nij_2 += (nij * nij); } double pj = 1 / (double) (N * n) * sum_nij; double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj); double kappaj = (Pj - pj) / (double) (1 - pj); return kappaj; }