@Override public void copyTo(AttributeImpl target) { PositionIncrementAttribute t = (PositionIncrementAttribute) target; t.setPositionIncrement(positionIncrement); }
@Override public boolean incrementToken() throws IOException { if (offset < end) { clearAttributes(); int id = ids.ints[offset]; termAtt.setBytesRef(idToTerm.get(id)); int incr = 1; if (idToInc.containsKey(id)) { incr = idToInc.get(id); } posIncAtt.setPositionIncrement(incr); offset++; return true; } return false; } }
/** * {@inheritDoc} */ @Override public void end() throws IOException { super.end(); posIncrAttribute.setPositionIncrement(posIncrAttribute.getPositionIncrement() + skipCounter); }
@Override public void end() throws IOException { super.end(); posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); } }
void setTerm(String term, int startOffset, int endOffset, int position) { if (config.lowercase) { term = term.toLowerCase(); } if (config.trimWhitespace) { term = term.trim(); } //ignore empty term if(term.length()==0){ return; } termAtt.setEmpty(); termAtt.append(term); if (startOffset < 0) { startOffset = 0; } if (endOffset < startOffset) { endOffset = startOffset + term.length(); } int offset = position - lastIncrementPosition; if (offset < 0) { offset = 0; } positionAttr.setPositionIncrement(offset); lastIncrementPosition = position; }
@Override public final boolean incrementToken() throws IOException { skippedPositions = 0; while (input.incrementToken()) { if (accept()) { if (skippedPositions != 0) { posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); } return true; } skippedPositions += posIncrAtt.getPositionIncrement(); } // reached EOS -- return false return false; }
@Override public final void end() throws IOException { super.end(); // set final offset int finalOffset = correctOffset(this.endPosition); offsetAtt.setOffset(finalOffset, finalOffset); posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); } }
@Override public final boolean incrementToken() throws IOException { if (enablePositionIncrements) { int skippedPositions = 0; while (input.incrementToken()) { if (accept()) { if (skippedPositions != 0) { posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); } return true; } skippedPositions += posIncrAtt.getPositionIncrement(); } } else { while (input.incrementToken()) { if (accept()) { return true; } } } // reached EOS -- return false return false; }
private void setAttribs(PendingToken tok) { clearAttributes(); this.posIncrAtt.setPositionIncrement(tok.nonpos ? 0 : 1); this.termAtt.setEmpty(); this.termAtt.append(tok.str); this.offsetAtt.setOffset(tok.start, tok.end); }
} else { if (skipCounter != 0) { posIncrAttribute.setPositionIncrement(posIncrAttribute.getPositionIncrement() + skipCounter);
void setTerm(String term, int startOffset, int endOffset, int position) { if (config.lowercase) { term = term.toLowerCase(); } if (config.trimWhitespace) { term = term.trim(); } //ignore empty term if(term.length()==0){ return; } termAtt.setEmpty(); termAtt.append(term); if (startOffset < 0) { startOffset = 0; } if (endOffset < startOffset) { endOffset = startOffset + term.length(); } if(!config.ignorePinyinOffset){ offsetAtt.setOffset(correctOffset(startOffset), correctOffset(endOffset)); } int offset = position - lastIncrementPosition; if (offset < 0) { offset = 0; } positionAttr.setPositionIncrement(offset); lastIncrementPosition = position; }
/** * Clears, and then resets the instances attributes per the specified * arguments. * @param str the matched symbol * @param start the match start position * @param end the match end position */ protected void setAttribs(String str, int start, int end) { clearAttributes(); //FIXME increasing below by one(default) might be tricky, need more analysis // after lucene upgrade to 3.5 below is most probably not even needed this.posIncrAtt.setPositionIncrement(1); this.termAtt.setEmpty(); this.termAtt.append(str); this.offsetAtt.setOffset(start, end); } }
@Override public final void end() throws IOException { super.end(); // set final offset int finalOffset = correctOffset(scanner.yychar() + scanner.yylength()); offsetAtt.setOffset(finalOffset, finalOffset); // adjust any skipped tokens posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+skippedPositions); }
@Override public void copyTo(AttributeImpl target) { if (target instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target; to.copyBuffer(buffer(), 0, length()); to.positionIncrement = positionIncrement; to.positionLength = positionLength; to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; to.termFrequency = termFrequency; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); ((TermFrequencyAttribute) target).setTermFrequency(termFrequency); } }
@Override public final boolean incrementToken() throws IOException { clearAttributes(); skippedPositions = 0; while(true) { int tokenType = scanner.getNextToken(); if (tokenType == StandardTokenizerImpl.YYEOF) { return false; } if (scanner.yylength() <= maxTokenLength) { posIncrAtt.setPositionIncrement(skippedPositions+1); scanner.getText(termAtt); final int start = scanner.yychar(); offsetAtt.setOffset(correctOffset(start), correctOffset(start+termAtt.length())); typeAtt.setType(StandardTokenizer.TOKEN_TYPES[tokenType]); return true; } else // When we skip a too-long term, we still increment the // position increment skippedPositions++; } }
@Override public boolean incrementToken() throws IOException { //清除所有的词元属性 clearAttributes(); skippedPositions = 0; Lexeme nextLexeme = _IKImplement.next(); if(nextLexeme != null){ posIncrAtt.setPositionIncrement(skippedPositions +1 ); //将Lexeme转成Attributes //设置词元文本 termAtt.append(nextLexeme.getLexemeText()); //设置词元长度 termAtt.setLength(nextLexeme.getLength()); //设置词元位移 offsetAtt.setOffset(correctOffset(nextLexeme.getBeginPosition()), correctOffset(nextLexeme.getEndPosition())); //记录分词的最后位置 endPosition = nextLexeme.getEndPosition(); //记录词元分类 typeAtt.setType(nextLexeme.getLexemeTypeString()); //返会true告知还有下个词元 return true; } //返会false告知词元输出完毕 return false; }
@Override public void end() throws IOException { super.end(); posIncrAtt.setPositionIncrement(curPosIncr); } }
@Override public void end() throws IOException { super.end(); posIncrAtt.setPositionIncrement(curPosIncr); } }
@Override public void end() throws IOException { if (inputStreamExhausted == false) { finishInnerStream(); } clearAttributes(); this.offsetAtt.setOffset(0, endToken.endOffset()); this.incAtt.setPositionIncrement(endToken.posInc()); }
@Override public final void end() throws IOException { super.end(); // set final offset int finalOffset = correctOffset(scanner.yychar() + scanner.yylength()); offsetAtt.setOffset(finalOffset, finalOffset); // adjust any skipped tokens posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+skippedPositions); }