How to use

getItemCount

method

in

de.tudarmstadt.ukp.dkpro.statistics.agreement.coding.ICodingAnnotationStudy

protected double calculateMaximumObservedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotations = annotationsPerCategory.get(category);
    int min = -1;
    for (int rater = 0; rater < study.getRaterCount(); rater++)
      if (annotations[rater] < min || min < 0)
        min = annotations[rater];
    if (min > 0)
      result = result.add(new BigDecimal(min));
  }
  result = result.divide(new BigDecimal(study.getItemCount()), MathContext.DECIMAL128);
  return result.doubleValue();
}

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, Integer> annotationsPerCategory 
      = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study);
  
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories())
    result = result.add(
        new BigDecimal(annotationsPerCategory.get(category)).pow(2));
  result = result.divide(new BigDecimal(4).multiply(
      new BigDecimal(study.getItemCount()).pow(2)), MathContext.DECIMAL128);
  return result.doubleValue();
}

/** Calculates the expected inter-rater agreement that assumes a 
 *  different probability distribution for all raters. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotations = annotationsPerCategory.get(category);
    BigDecimal prod = new BigDecimal(1);
    for (int rater = 0; rater < study.getRaterCount(); rater++)
      prod = prod.multiply(new BigDecimal(annotations[rater]));
    result = result.add(prod);
  }
  result = result.divide(new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128);
  return result.doubleValue();
}
 

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>();
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
    for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) {
      BigDecimal p = new BigDecimal(counts.getValue()).divide(
          new BigDecimal(item.getRaterCount()), 
          MathContext.DECIMAL128);
      BigDecimal value = categoryProbability.get(counts.getKey());
      if (value != null)
        p = p.add(value);
      categoryProbability.put(counts.getKey(), p);
    }
  }
  BigDecimal result = new BigDecimal(0);
  for (BigDecimal p : categoryProbability.values())
    result = result.add(p.pow(2));
  result = result.divide(
      new BigDecimal(study.getItemCount()).pow(2), 
      MathContext.DECIMAL128);
  return result.doubleValue();
}
 

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotationCounts = annotationsPerCategory.get(category);
    for (int m = 0; m < study.getRaterCount(); m++)
      for (int n = m + 1; n < study.getRaterCount(); n++)
        result = result.add(new BigDecimal(annotationCounts[m])
            .multiply(new BigDecimal(annotationCounts[n])));
  }
  result = result.multiply(new BigDecimal(2));
  result = result.divide(new BigDecimal(study.getRaterCount())
      .multiply(new BigDecimal(study.getRaterCount() - 1))
      .multiply(new BigDecimal(study.getItemCount()).pow(2)), 
      MathContext.DECIMAL128);
  return result.doubleValue();
}

/** Calculates the observed inter-rater agreement for the annotation 
 *  study that was passed to the class constructor and the currently
 *  assigned distance function.
 *  @throws NullPointerException if the study is null.
 *  @throws ArithmeticException if the study does not contain any item or
 *      the number of raters is smaller than 2. */
public double calculateObservedDisagreement() {
  ensureDistanceFunction();
  
  double result = 0.0;
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
          
    for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet())
      for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) {
        if (category1.getValue() == null)
          continue;
        if (category2.getValue() == null)
          continue;
        
        result += category1.getValue() * category2.getValue()
            * distanceFunction.measureDistance(study, 
                category1.getKey(), category2.getKey());
      }
  }
  result /= (double) (study.getItemCount() * study.getRaterCount() 
      * (study.getRaterCount() - 1));
  return result;
}

public static void dumpStudy(PrintStream aOut, ICodingAnnotationStudy aStudy)
{
  try {
    aOut.printf("Category count: %d%n", aStudy.getCategoryCount());
  }
  catch (Throwable e) {
    aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  try {
    aOut.printf("Item count: %d%n", aStudy.getItemCount());
  }
  catch (Throwable e) {
    aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  
  for (ICodingAnnotationItem item : aStudy.getItems()) {
    StringBuilder sb = new StringBuilder();
    for (IAnnotationUnit unit : item.getUnits()) {
      if (sb.length() > 0) {
        sb.append(" \t");
      }
      sb.append(unit.getCategory());
    }
    aOut.println(sb);
  }
}

/** Calculates the expected inter-rater agreement using the defined 
 *  distance function to infer the assumed probability distribution. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items or raters in the 
 *      annotation study. */
public double calculateExpectedDisagreement() {
  ensureDistanceFunction();
  
  double result = 0.0;
  Map<Object, Integer> annotationsPerCategory 
      = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study);
  
  for (Object category1 : study.getCategories())
    for (Object category2 : study.getCategories())
      result += annotationsPerCategory.get(category1)
          * annotationsPerCategory.get(category2)
          * distanceFunction.measureDistance(study, category1, category2);
    
  result /= (double) (study.getItemCount() * study.getRaterCount() 
      * (study.getItemCount() * study.getRaterCount() - 1));
  return result;
}

public static AgreementResult getAgreement(ConcreteAgreementMeasure aMeasure,
    boolean aExcludeIncomplete, DiffResult aDiff, String aType, String aFeature,
    Map<String, List<JCas>> aCasMap)
{
  if (aCasMap.size() != 2) {
    throw new IllegalArgumentException("CAS map must contain exactly two CASes");
  }
  
  AgreementResult agreementResult = AgreementUtils.makeStudy(aDiff, aType, aFeature,
      aExcludeIncomplete, aCasMap);
  try {
    IAgreementMeasure agreement = aMeasure.make(agreementResult.study);
    
    if (agreementResult.study.getItemCount() > 0) {
      agreementResult.setAgreement(agreement.calculateAgreement());
    }
    else {
      agreementResult.setAgreement(Double.NaN);
    }
    return agreementResult;
    
  }
  catch (RuntimeException e) {
    // FIXME
    AgreementUtils.dumpAgreementStudy(System.out, agreementResult);
    throw e;
  }
}

for (int i = 0; i < study.getItemCount(); i++)
  out.print(DIVIDER + (i + 1));
out.print(DIVIDER + "Σ");

public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit annotation : item.getUnits())
      if (annotation.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

@Override
public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit unit : item.getUnits())
      if (unit.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

public static void dumpAgreementStudy(PrintStream aOut, AgreementResult aAgreement)
{
  try {
    aOut.printf("Category count: %d%n", aAgreement.getStudy().getCategoryCount());
  }
  catch (Throwable e) {
    aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  try {
    aOut.printf("Item count: %d%n", aAgreement.getStudy().getItemCount());
  }
  catch (Throwable e) {
    aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  aOut.printf("Relevant position count: %d%n", aAgreement.getRelevantSetCount());
  aOut.printf("%n== Complete sets: %d ==%n", aAgreement.getCompleteSets().size());
  dumpAgreementConfigurationSetsWithItems(aOut, aAgreement, aAgreement.getCompleteSets());
  
  aOut.printf("%n== Incomplete sets (by position): %d == %n",
      aAgreement.getIncompleteSetsByPosition().size());
  dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getIncompleteSetsByPosition());
  aOut.printf("%n== Incomplete sets (by label): %d ==%n",
      aAgreement.getIncompleteSetsByLabel().size());
  dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getIncompleteSetsByLabel());
  aOut.printf("%n== Plurality sets: %d ==%n", aAgreement.getPluralitySets().size());
  dumpAgreementConfigurationSets(aOut, aAgreement, aAgreement.getPluralitySets());
}

  result += catCount * (catCount - 1) + otherCatCount * (otherCatCount - 1);
return result / (double) (study.getItemCount() 
    * study.getRaterCount() * (study.getRaterCount() - 1));

  nNullCategorySum += nNullCategory;
observedDisagreement /= (double) study.getItemCount() 
    * study.getRaterCount() * (study.getRaterCount() - 1);
    + nNullCategorySum * nKeepCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, category)
    + nNullCategorySum * nNullCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, NULL_CATEGORY);
expectedDisagreement /= (double) study.getItemCount() * study.getRaterCount() 
    * (study.getItemCount() * study.getRaterCount() - 1);

.getItemCount()));