protected Token process(Token t) throws IOException { Token tok = read(); while (tok != null && tok.getPositionIncrement()==0) { if (null != t) { write(t); t = null; } boolean dup=false; for (Token outTok : output()) { int tokLen = tok.termLength(); if (outTok.termLength() == tokLen && ArraysUtils.equals(outTok.termBuffer(), 0, tok.termBuffer(), 0, tokLen)) { dup=true; //continue;; } } if (!dup){ write(tok); } tok = read(); } if (tok != null) { pushBack(tok); } return t; } }