int endPos = pos + posLengthAtt.getPositionLength(); while (state < endPos) { state = builder.createState();
int positionLength = posLenAtt.getPositionLength(); if (enableGraphQueries && positionLength > 1) { isGraph = true;
final int endPos = pos + posLengthAtt.getPositionLength();
int toInputNodeID = inputNode.node + posLenAtt.getPositionLength(); InputNode toInputNode = inputNodes.get(toInputNodeID);
int inputTo = inputFrom + posLenAtt.getPositionLength();
nextNodeOut = lastNodeOut + posLenAtt.getPositionLength();
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
public int getPositionLength() { return posLenAtt.getPositionLength(); }
lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
@Override public final boolean incrementToken() throws IOException { OutputTuple outputTuple; while (true) { if ((outputTuple = output.poll()) != null) { // return buffered output state restoreTuple(outputTuple); return true; } else if (!input.incrementToken()) { // no more tokens available from source return false; } State baseState = captureState(); int synonymCount = generateSynonyms(); restoreState(baseState); if (synonymCount > 0 && synonymPositionsEnabled) { int currentLength = posLenAtt.getPositionLength(); posLenAtt.setPositionLength(Math.max(synonymCount, currentLength)); } return true; } }
TermDefinition(final CharTermAttribute charTermAttr, final FlagsAttribute flagsAttr, final OffsetAttribute offsetAttr, final PositionIncrementAttribute posIncAttr, final PositionLengthAttribute posLengthAttr, final TypeAttribute typeAttr, final KeywordAttribute keywordAttr) { char_term = charTermAttr == null ? null : charTermAttr.toString(); if (offsetAttr != null) { start_offset = offsetAttr.startOffset(); end_offset = offsetAttr.endOffset(); } else { start_offset = null; end_offset = null; } flags = flagsAttr == null ? null : flagsAttr.getFlags(); position_increment = posIncAttr == null ? null : posIncAttr.getPositionIncrement(); position_length = posLengthAttr == null ? null : posLengthAttr.getPositionLength(); type = typeAttr == null ? null : typeAttr.type(); is_keyword = keywordAttr == null ? null : keywordAttr.isKeyword(); }
int endPos = pos + posLengthAtt.getPositionLength(); while (state < endPos) { state = builder.createState();
lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
tokEnd = offsetAtt.endOffset(); savePosIncr += posIncrAtt.getPositionIncrement(); savePosLen = posLenAtt.getPositionLength();
private ArrayList<Data> analyze(Analyzer analyzer1) throws IOException { ArrayList<Data> results = new ArrayList<>(50); TokenStream ts = analyzer1.tokenStream("foo", text); ts.reset(); while (ts.incrementToken()) { Data data = new Data(); OffsetAttribute offsetAttribute = ts.getAttribute(OffsetAttribute.class); data.startOffset = offsetAttribute.startOffset(); data.endOffset = offsetAttribute.endOffset(); data.positionLength = ts.getAttribute(PositionLengthAttribute.class).getPositionLength(); data.positionIncGap = ts.getAttribute(PositionIncrementAttribute.class).getPositionIncrement(); data.tokenType = ts.getAttribute(HebrewTokenTypeAttribute.class).getType().toString(); data.term = ts.getAttribute(CharTermAttribute.class).toString(); if (ts.getAttribute(KeywordAttribute.class) != null) data.isKeyword = ts.getAttribute(KeywordAttribute.class).isKeyword(); // System.out.println(data.term + " " + data.tokenType); results.add(data); } ts.close(); return results; } }