Refine search
@Override public void copyTo(AttributeImpl target) { if (target instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target; to.copyBuffer(buffer(), 0, length()); to.positionIncrement = positionIncrement; to.positionLength = positionLength; to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; to.termFrequency = termFrequency; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); ((TermFrequencyAttribute) target).setTermFrequency(termFrequency); } }
int maxOffset = 0; while (in.incrementToken()) { int posInc = posIncAtt.getPositionIncrement(); if (preservePositionIncrements == false && posInc > 1) { posInc = 1; final int endPos = pos + posLengthAtt.getPositionLength(); maxOffset = Math.max(maxOffset, offsetAtt.endOffset()); int endPosInc = posIncAtt.getPositionIncrement(); if (endPosInc == 0 && finalOffsetGapAsHole && offsetAtt.endOffset() > maxOffset) { endPosInc = 1; } else if (endPosInc > 0 && preservePositionIncrements==false) {
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
/** * Constructs a compound token. */ private void gramToken() { buffer.append(termAttribute.buffer(), 0, termAttribute.length()); int endOffset = offsetAttribute.endOffset(); clearAttributes(); int length = buffer.length(); char termText[] = termAttribute.buffer(); if (length > termText.length) { termText = termAttribute.resizeBuffer(length); } buffer.getChars(0, length, termText, 0); termAttribute.setLength(length); posIncAttribute.setPositionIncrement(0); posLenAttribute.setPositionLength(2); // bigram offsetAttribute.setOffset(lastStartOffset, endOffset); typeAttribute.setType(GRAM_TYPE); buffer.setLength(0); } }
curCodePointCount = charUtils.codePointCount(termAtt); curGramSize = minGram; tokStart = offsetAtt.startOffset(); tokEnd = offsetAtt.endOffset(); savePosIncr += posIncrAtt.getPositionIncrement(); savePosLen = posLenAtt.getPositionLength(); offsetAtt.setOffset(tokStart, tokEnd); posIncrAtt.setPositionIncrement(savePosIncr); savePosIncr = 0; } else { posIncrAtt.setPositionIncrement(0); posLenAtt.setPositionLength(savePosLen); final int charLength = charUtils.offsetByCodePoints(curTermBuffer, 0, curTermLength, 0, curGramSize); termAtt.copyBuffer(curTermBuffer, 0, charLength);
char[] termBuffer = termAttribute.buffer(); accumPosInc += posIncAttribute.getPositionIncrement(); posIncAttribute.setPositionIncrement(accumPosInc); accumPosInc = 0; return true; endOffset = Math.max(endOffset, lastStartOffset); offsetAttribute.setOffset(startOffset, endOffset); lastStartOffset = startOffset; posIncAttribute.setPositionIncrement(accumPosInc + startPos - wordPos); accumPosInc = 0; posLenAttribute.setPositionLength(endPos - startPos); wordPos = startPos; return true;
posIncAtt.setPositionIncrement(outputFrom - lastOutputFrom); int toInputNodeID = inputNode.node + posLenAtt.getPositionLength(); InputNode toInputNode = inputNodes.get(toInputNodeID); posLenAtt.setPositionLength(toInputNode.outputNode - outputFrom); lastOutputFrom = outputFrom; inputNode.nextOut++; offsetAtt.setOffset(startOffset, endOffset); lastStartOffset = startOffset;
int state = -1; while (in.incrementToken()) { int currentIncr = posIncAtt.getPositionIncrement(); if (pos == -1 && currentIncr < 1) { throw new IllegalStateException("Malformed TokenStream, start token can't have increment less than 1"); int endPos = pos + posLengthAtt.getPositionLength(); while (state < endPos) { state = builder.createState();
posIncAttribute.setPositionIncrement(1); posLengthAttribute.setPositionLength(1); posIncAttribute.setPositionIncrement(1); posLengthAttribute.setPositionLength(1);
end = start + length; termAtt.copyBuffer(buffer[current].toCharArray(), 0, length); offsetAtt.setOffset(correctOffset(start), correctOffset(end)); posArr.setPositionLength(current + 1); typeAtt.setType("word"); start = end;
@Override public final boolean incrementToken() throws IOException { OutputTuple outputTuple; while (true) { if ((outputTuple = output.poll()) != null) { // return buffered output state restoreTuple(outputTuple); return true; } else if (!input.incrementToken()) { // no more tokens available from source return false; } State baseState = captureState(); int synonymCount = generateSynonyms(); restoreState(baseState); if (synonymCount > 0 && synonymPositionsEnabled) { int currentLength = posLenAtt.getPositionLength(); posLenAtt.setPositionLength(Math.max(synonymCount, currentLength)); } return true; } }
@Override public void copyTo(AttributeImpl target) { PositionLengthAttribute t = (PositionLengthAttribute) target; t.setPositionLength(positionLength); }
public int getPositionLength() { return posLenAtt.getPositionLength(); }
int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null)); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement();
posIncrAtt.setPositionIncrement(isOutputHere ? 0 : 1); termAtt.setEmpty().append(gramBuilder); if (gramSize.getValue() > 1) { noShingleOutput = false; offsetAtt.setOffset(offsetAtt.startOffset(), nextToken.offsetAtt.endOffset()); if (outputUnigrams) { posLenAtt.setPositionLength(builtGramSize); } else { posLenAtt.setPositionLength(Math.max(1, (builtGramSize - minShingleSize) + 1));
curCodePointCount = charUtils.codePointCount(termAtt); curGramSize = minGram; tokStart = offsetAtt.startOffset(); tokEnd = offsetAtt.endOffset(); savePosIncr += posIncrAtt.getPositionIncrement(); savePosLen = posLenAtt.getPositionLength(); offsetAtt.setOffset(tokStart, tokEnd); posIncrAtt.setPositionIncrement(savePosIncr); savePosIncr = 0; } else { posIncrAtt.setPositionIncrement(0); posLenAtt.setPositionLength(savePosLen); final int charLength = charUtils.offsetByCodePoints(curTermBuffer, 0, curTermLength, 0, curGramSize); termAtt.copyBuffer(curTermBuffer, 0, charLength);
while (stream.incrementToken()) { numTokens++; int positionIncrement = posIncAtt.getPositionIncrement(); if (positionIncrement != 0) { positionCount += positionIncrement; int positionLength = posLenAtt.getPositionLength(); if (enableGraphQueries && positionLength > 1) { isGraph = true;
private void setAttribute(Morpheme morpheme) throws IOException { if (aUnitSize != 0) { posLengthAtt.setPositionLength(aUnitSize); aUnitSize = 0; } else if (oovSize != 0) { posLengthAtt.setPositionLength(oovSize); oovSize = 0; } else { posLengthAtt.setPositionLength(1); } posIncAtt.setPositionIncrement(1); setMorphemeAttributes(morpheme); }
@Override public void copyTo(AttributeImpl target) { PositionLengthAttribute t = (PositionLengthAttribute) target; t.setPositionLength(positionLength); }
inputFrom += posIncAtt.getPositionIncrement(); int startOffset = offsetAtt.startOffset(); int endOffset = offsetAtt.endOffset(); int inputTo = inputFrom + posLenAtt.getPositionLength(); finalPosInc = posIncAtt.getPositionIncrement(); finalOffset = offsetAtt.endOffset(); done = true;