How to use

ICodingAnnotationItem

in

de.tudarmstadt.ukp.dkpro.statistics.agreement.coding

public int getNonNullCount(String aCasGroupId)
{
  int i = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) {
      i++;
    }
  }
  return i;
}

/** Returns a map of categories and their usage frequencies (i.e., 
 *  how often they are used in annotation units) within the given
 *  annotation item. */
// Category -> #
public static Map<Object, Integer> countTotalAnnotationsPerCategory(
    final ICodingAnnotationItem item) {
  Map<Object, Integer> result = new HashMap<Object, Integer>();
  for (IAnnotationUnit unit : item.getUnits()) {
    Object category = unit.getCategory();
    if (category == null)
      continue;
    
    Integer count = result.get(category);
    if (count == null)
      result.put(category, 1);
    else
      result.put(category, count + 1);
  }
  return result;
}

/** Returns a map of categories and their usage frequencies (i.e., 
 *  how often they are used in annotation units) within the given
 *  annotation study. */
// Category -> #
public static Map<Object, Integer> countTotalAnnotationsPerCategory(
    final ICodingAnnotationStudy study) {
  Map<Object, Integer> result = new HashMap<Object, Integer>();
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getRaterCount() <= 1)
      continue;
    
    for (IAnnotationUnit unit : item.getUnits()) {
      Object category = unit.getCategory();
      if (category == null)
        continue;
      
      Integer count = result.get(category);
      if (count == null)
        result.put(category, 1);
      else
        result.put(category, count + 1);
    }
  }
  return result;
}
 

out.print(r + 1);
for (ICodingAnnotationItem item : study.getItems())
  out.print(DIVIDER + item.getUnit(r).getCategory());
out.println();
for (ICodingAnnotationItem item : study.getItems()) {
  int catCount = 0;
  for (IAnnotationUnit unit : item.getUnits())
    if (category.equals(unit.getCategory()))
      catCount++;

@Override
public int getUnitCount() {
  int result = 0;
  for (ICodingAnnotationItem item : items)
    result += item.getRaterCount();
  return result;
  //return items.size() * raters.size();
}
 

@Override
public boolean hasMissingValues() {
  for (ICodingAnnotationItem item : items)
    if (item.getRaterCount() != raters.size())
      return true;
  
  return false;
}
 

public boolean isAllNull(String aCasGroupId)
{
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) {
      return false;
    }
  }
  return true;
}

private static void dumpAgreementConfigurationSetsWithItems(PrintStream aOut,
    AgreementResult aAgreement, List<ConfigurationSet> aSets)
{
  int i = 0;
  for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
    StringBuilder sb = new StringBuilder();
    sb.append(aSets.get(i).getPosition());
    for (IAnnotationUnit unit : item.getUnits()) {
      if (sb.length() > 0) {
        sb.append(" \t");
      }
      sb.append(unit.getCategory());
    }
    aOut.println(sb);
    i++;
  }
}

Map<Object, Map<Object, Double>> result = 
    new HashMap<Object, Map<Object, Double>>();
for (IAnnotationUnit unit1 : item.getUnits()) 
  for (IAnnotationUnit unit2 : item.getUnits()) {
    if (unit1 == unit2)
      continue;
int raterCount = item.getRaterCount();
for (Map<Object, Double> cat2 : result.values())
  for (Entry<Object, Double> entry : cat2.entrySet())

protected double doCalculateItemAgreement(final ICodingAnnotationItem item) {
  Map<Object, Integer> annotationsPerCategory 
      = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
  double result = 0.0;
  for (Integer count : annotationsPerCategory.values())
    result += count * (count - 1);
  int raterCount = item.getRaterCount();
  if (raterCount <= 1)
    return 0.0;
  else 
    return result / (double) (raterCount - 1.0);
}

int cat1 = categories.get(item.getUnit(0).getCategory());
int cat2 = categories.get(item.getUnit(1).getCategory());
frequencies[cat1][cat2]++;

/** Returns a two-dimensional map of categories and raters and the
 *  corresponding usage frequencies in the given annotation study 
 *  (i.e., how often a certain rater used a certain category for
 *  coding an annotation unit). */
// Category x Rater -> #
public static Map<Object, int[]> countAnnotationsPerCategory(
    final ICodingAnnotationStudy study) {
  Map<Object, int[]> result = new HashMap<Object, int[]>();
  for (ICodingAnnotationItem item : study.getItems())
    for (IAnnotationUnit unit : item.getUnits()) {
      Object category = unit.getCategory();
      if (category == null)
        continue;
      
      int[] counts = result.get(category);
      if (counts == null)
        counts = new int[study.getRaterCount()];
      counts[unit.getRaterIdx()]++;
      result.put(category, counts);
    }
  return result;
}

/** Calculates the inter-rater agreement for the given annotation item. 
 *  This is the basic step that is performed for each item of an 
 *  annotation study, when calling {@link #calculateAgreement()}. 
 *  @throws NullPointerException if the given item is null. */
public double calculateItemAgreement(final ICodingAnnotationItem item) {
  return doCalculateItemAgreement(item) / item.getRaterCount();
}

/** Returns a clone of the current annotation study which contains
 *  only the annotation units of the raters with the given indexes.
 *  All other units will be removed. This method is useful for
 *  converting an annotation study with multiple raters into a
 *  (pairwise) annotation study with two raters. */
public CodingAnnotationStudy extractRaters(final int... raters) {
  CodingAnnotationStudy result = new CodingAnnotationStudy(raters.length);
  for (ICodingAnnotationItem item : getItems()) {
    CodingAnnotationItem newItem = new CodingAnnotationItem(raters.length);
    for (int r = 0; r < raters.length; r++) {
      IAnnotationUnit unit = item.getUnit(raters[r]);
      newItem.addUnit(result.createUnit(result.items.size(), 
          r, unit.getCategory()));
    }
    result.items.add(newItem);
  }
  return result;
}

public static void dumpStudy(PrintStream aOut, ICodingAnnotationStudy aStudy)
{
  try {
    aOut.printf("Category count: %d%n", aStudy.getCategoryCount());
  }
  catch (Throwable e) {
    aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  try {
    aOut.printf("Item count: %d%n", aStudy.getItemCount());
  }
  catch (Throwable e) {
    aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  
  for (ICodingAnnotationItem item : aStudy.getItems()) {
    StringBuilder sb = new StringBuilder();
    for (IAnnotationUnit unit : item.getUnits()) {
      if (sb.length() > 0) {
        sb.append(" \t");
      }
      sb.append(unit.getCategory());
    }
    aOut.println(sb);
  }
}

@Override
public double calculateObservedAgreement() {
  double result = 0.0;
  double denominator = 0.0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int raterCount = item.getRaterCount();
    if (raterCount > 1) {
      result += doCalculateItemAgreement(item);
      denominator += raterCount;
    }
  }
  return result / denominator;
}

ICodingAnnotationItem item2 = agreement.getStudy().getItem(1);
ICodingAnnotationItem item3 = agreement.getStudy().getItem(2);
assertEquals("", item1.getUnit(0).getCategory());
assertEquals("", item1.getUnit(1).getCategory());
assertEquals("", item2.getUnit(0).getCategory());
assertEquals(null, item2.getUnit(1).getCategory());
assertEquals(null, item3.getUnit(0).getCategory());
assertEquals("", item3.getUnit(1).getCategory());

private static void configurationSetsWithItemsToCsv(CSVPrinter aOut,
    AgreementResult aAgreement, List<ConfigurationSet> aSets)
  throws IOException
{
  List<String> headers = new ArrayList<>(
      asList("Type", "Collection", "Document", "Layer", "Feature", "Position"));
  headers.addAll(aAgreement.getCasGroupIds());
  aOut.printRecord(headers);
  
  int i = 0;
  for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
    Position pos = aSets.get(i).getPosition();
    List<String> values = new ArrayList<>();
    values.add(pos.getClass().getSimpleName());
    values.add(pos.getCollectionId());
    values.add(pos.getDocumentId());
    values.add(pos.getType());
    values.add(aAgreement.getFeature());
    values.add(aSets.get(i).getPosition().toMinimalString());
    for (IAnnotationUnit unit : item.getUnits()) {
      values.add(String.valueOf(unit.getCategory()));
    }
    aOut.printRecord(values);
    i++;
  }
}    

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>();
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
    for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) {
      BigDecimal p = new BigDecimal(counts.getValue()).divide(
          new BigDecimal(item.getRaterCount()), 
          MathContext.DECIMAL128);
      BigDecimal value = categoryProbability.get(counts.getKey());
      if (value != null)
        p = p.add(value);
      categoryProbability.put(counts.getKey(), p);
    }
  }
  BigDecimal result = new BigDecimal(0);
  for (BigDecimal p : categoryProbability.values())
    result = result.add(p.pow(2));
  result = result.divide(
      new BigDecimal(study.getItemCount()).pow(2), 
      MathContext.DECIMAL128);
  return result.doubleValue();
}
 

@Override
public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit unit : item.getUnits())
      if (unit.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}