@Override public boolean incrementToken() throws IOException { boolean result = false; if (input.incrementToken()) { String docid = termAtt.term(); termAtt.setTermBuffer(LsiIndex.PAYLOAD_TERM_TEXT); payAtt.setPayload(new Payload(PayloadEncoder.encodePayloadId(docid))); return true; } return result; } }
/** * Fills Lucene token with the current token text. */ final void getText(TermAttribute t) { t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); }
@Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; CharSequence text = CharBuffer.wrap(termAtt.termBuffer(), 0, termAtt.termLength()); Matcher m = p.matcher(text); if (all) { termAtt.setTermBuffer(m.replaceAll(replacement)); } else { termAtt.setTermBuffer(m.replaceFirst(replacement)); } return true; }
@Override public boolean incrementToken() throws IOException { if (consumed) { return false; } clearAttributes(); termAttribute.setTermBuffer(value); payloadAttribute.setPayload(payload); consumed = true; return true; }
@Override public boolean incrementToken() throws IOException { if (consumed) { return false; } clearAttributes(); termAttribute.setTermBuffer(value); payloadAttribute.setPayload(payload); consumed = true; return true; }
termAttr.setTermBuffer(terms[pos++]); if (pos == terms.length) { terms = null;
final String match = matcher.group(group); if (match.length() == 0) continue; termAtt.setTermBuffer(match); index = matcher.start(group); offsetAtt.setOffset(correctOffset(index), correctOffset(matcher.end(group))); if (matcher.start() - index > 0) { termAtt.setTermBuffer(str, index, matcher.start() - index); offsetAtt.setOffset(correctOffset(index), correctOffset(matcher.start())); index = matcher.end(); termAtt.setTermBuffer(str, index, str.length() - index); offsetAtt.setOffset(correctOffset(index), correctOffset(str.length()));
@Override public boolean incrementToken() throws IOException { if (i.hasNext()) { clearAttributes(); T a = i.next(); termAtt.setTermBuffer(a.getCoveredText().toLowerCase()); offsetAtt.setOffset(a.getBegin(), a.getEnd()); return true; } else { return false; } } }
@Override public boolean incrementToken() throws IOException { clearAttributes(); int n = input.read(cbuf,0,maxChars); if (n<=0) return false; String s = toInternal(new String(cbuf,0,n)); termAtt.setTermBuffer(s); offsetAtt.setOffset(correctOffset(0),correctOffset(n)); return true; } };
@Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; char[] termBuffer = termAtt.termBuffer(); int len = termAtt.termLength(); // if protected, don't stem. use this to avoid stemming collisions. if (protWords != null && protWords.contains(termBuffer, 0, len)) { return true; } stemmer.setCurrent(new String(termBuffer, 0, len));//ugh, wish the Stemmer took a char array stemmer.stem(); String newstr = stemmer.getCurrent(); termAtt.setTermBuffer(newstr.toCharArray(), 0, newstr.length()); return true; } }
termAtt.setTermBuffer(backup, 0, termBufferLength);
termAtt.setTermBuffer(primaryPhoneticValue); saveState = true; termAtt.setTermBuffer(alternatePhoneticValue); saveState = true;
termAtt.setTermBuffer(phonetic); return true; termAtt.setTermBuffer(phonetic); return true;
termAtt.setTermBuffer(s); termAtt.setTermBuffer(number.getFullStrongsNumber());
termAtt.setTermBuffer(s.toCharArray(), 0, s.length()); offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length())); flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
termAtt.setTermBuffer(s.toCharArray(), 0, s.length()); offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length())); flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
private void applyToken(Token token) { termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength()); posAtt.setPositionIncrement(token.getPositionIncrement()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); }
termAtt.setTermBuffer(token.term()); offsetAtt.setOffset(correctOffset(token.startOffset()), correctOffset(token.endOffset()));
termAtt.setTermBuffer(termBuffer, start, (end - start)); } else { termAtt.setTermLength(0);
@Override public boolean incrementToken() throws IOException { if( save != null ) { // clearAttributes(); // not currently necessary restoreState(save); save = null; return true; } if (!input.incrementToken()) return false; // pass through zero-length terms int oldLen = termAtt.termLength(); if (oldLen ==0) return true; int origOffset = posAtt.getPositionIncrement(); if (withOriginal == true){ posAtt.setPositionIncrement(0); save = captureState(); } char [] buffer = termAtt.resizeTermBuffer(oldLen + 1); buffer[oldLen] = markerChar; //String reversed = reverseAndMark(value, markerChar); ReverseStringFilter.reverse(buffer, oldLen + 1); posAtt.setPositionIncrement(origOffset); termAtt.setTermBuffer(buffer, 0, oldLen +1); return true; }