How to use

getTokens

method

in

de.tudarmstadt.ukp.dkpro.teaching.core.Sentence

@Override
public String toString() {
  return StringUtils.join(getTokens(), ' ');
}
 

public String getFormattedString() {
  return "[" + StringUtils.join(getTokens(), ' ') + "]";
}

@Ignore
@Test
public void tazTest() throws Exception {
  TazCorpus corpus = new TazCorpus();
  for (Sentence s : corpus.getSentences()) {
    for (int i=0; i<s.getTokens().size(); i++) {
      System.out.print(s.getTokens().get(i));
    }
  }
}

public BrownProvider(int minN, int maxN) throws Exception {
  BrownCorpus brownCorpus = new BrownCorpus();
  
  cfd = new ConditionalFrequencyDistribution<Integer,String>();
  
  if (minN > maxN) {
    throw new IllegalArgumentException("minN > maxN");
  }
  
  for (int i=minN; i<=maxN; i++) {
    for (Sentence s : brownCorpus.getSentences()) {
      cfd.addSamples(
          i,
          new NGramIterable(s.getTokens(), i, i)
      );
    }
  }
}
 

  public static void main(String[] args) throws Exception
  {
    Web1TProvider web1t = new Web1TProvider(new Locale("de"), 1);
    BrownCorpus brown = new BrownCorpus();
    
    StopWatch watch = new StopWatch();
    watch.start();
    watch.suspend();

    for (Text text : brown.getTexts()) {
      for (Sentence s : text.getSentences()) {
        for (String t : s.getTokens()) {
          watch.resume();
          web1t.getFrequency(t);
          watch.suspend();
        }
      }
    }
    
    double time = (double) watch.getTime() / 1000;
    System.out.println(time + "s");
  }
}