if (curGramSize <= curCodePointCount) { // if the remaining input is too short, we can't generate any n-grams clearAttributes(); offsetAtt.setOffset(tokStart, tokEnd);
if (curGramSize <= curCodePointCount) { // if the remaining input is too short, we can't generate any n-grams clearAttributes(); offsetAtt.setOffset(tokStart, tokEnd);