@Override public boolean incrementToken() throws IOException { Token token = tagger.next(); if (token == null) { return false; } else { clearAttributes(); final Morpheme m = token.getMorpheme(); // note, unlike the previous implementation, we set the surface form termAtt.setEmpty().append(token.getSurface()); final int cost = token.getCost(); if (token.isSentenceStart()) { accumulatedCost = 0; sentenceAtt.setSentenceStart(true); } costAtt.setCost(cost - accumulatedCost); accumulatedCost = cost; basicFormAtt.setMorpheme(m); conjugationAtt.setMorpheme(m); partOfSpeechAtt.setMorpheme(m); pronunciationsAtt.setMorpheme(m); readingsAtt.setMorpheme(m); offsetAtt.setOffset(correctOffset(token.getStart()), correctOffset(token.end())); return true; } }