@Override public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { if (analyzer == wrapper) { final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName); return wrappedAnalyzer.getReuseStrategy().getReusableComponents(wrappedAnalyzer, fieldName); } else { return fallbackStrategy.getReusableComponents(analyzer, fieldName); } }
@Override public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) { if (analyzer == wrapper) { final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName); wrappedAnalyzer.getReuseStrategy().setReusableComponents(wrappedAnalyzer, fieldName, components); } else { fallbackStrategy.setReusableComponents(analyzer, fieldName, components); } } };
@Override protected TokenStream normalize(String fieldName, TokenStream in) { switch (fieldName) { case QueryBuilder.DEFS: case QueryBuilder.REFS: return in; default: return new LowerCaseFilter(in); } } }
TokenStream tokenStream = analyzer.tokenStream(fieldName, reader); OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class); CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); tokenStream.reset(); while (tokenStream.incrementToken()) { int startOffset = offsetAttribute.startOffset(); int endOffset = offsetAttribute.endOffset(); String term = charTermAttribute.toString(); }
/** * 重载Analyzer接口,构造分词组件 */ @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer _IKTokenizer = new IKTokenizer(configuration); return new TokenStreamComponents(_IKTokenizer); }
@Override public void reset() throws IOException { super.reset(); dot = false; charsRead = 0; startPosition = 0; }
@Override public void reset() throws IOException { super.reset(); tokenIter = null; } }
/** * Reads lines from a Reader and adds every line as an entry to a CharArraySet (omitting * leading and trailing whitespace). Every line of the Reader should contain only * one word. The words need to be in lowercase if you make use of an * Analyzer which uses LowerCaseFilter (like StandardAnalyzer). * * @param reader Reader containing the wordlist * @return A {@link CharArraySet} with the reader's words */ public static CharArraySet getWordSet(Reader reader) throws IOException { return getWordSet(reader, new CharArraySet(INITIAL_CAPACITY, false)); }
@Override public int getPositionIncrementGap(String fieldName) { return getWrappedAnalyzer(fieldName).getPositionIncrementGap(fieldName); }
@Override public final void close() throws IOException { super.close(); } }
/** * Builds an analyzer which removes words in ENGLISH_STOP_WORDS. */ public HistoryAnalyzer() { super(Analyzer.PER_FIELD_REUSE_STRATEGY); stopWords = StopFilter.makeStopSet(ENGLISH_STOP_WORDS); }
@Override protected final TokenStreamComponents createComponents(String fieldName) { return wrapComponents(fieldName, getWrappedAnalyzer(fieldName).createComponents(fieldName)); }
@Override public final Reader initReader(String fieldName, Reader reader) { return getWrappedAnalyzer(fieldName).initReader(fieldName, wrapReader(fieldName, reader)); }
@Override protected final Reader initReaderForNormalization(String fieldName, Reader reader) { return getWrappedAnalyzer(fieldName).initReaderForNormalization(fieldName, wrapReaderForNormalization(fieldName, reader)); }
@Override protected final AttributeFactory attributeFactory(String fieldName) { return getWrappedAnalyzer(fieldName).attributeFactory(fieldName); } }
/** * Constructor. * @param fallbackStrategy is the strategy to use if delegation is not possible * This is to support the common pattern: * {@code new OtherWrapper(thisWrapper.getReuseStrategy())} */ protected DelegatingAnalyzerWrapper(ReuseStrategy fallbackStrategy) { super(new DelegatingReuseStrategy(fallbackStrategy)); // häckidy-hick-hack, because we cannot call super() with a reference to "this": ((DelegatingReuseStrategy) getReuseStrategy()).wrapper = this; }
@Override public void reset() throws IOException { super.reset(); tokenStart = tokenEnd = 0; }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { return new LowerCaseFilter(in); } }
@Override public void reset() throws IOException { super.reset(); _IKImplement.reset(input); skippedPositions = 0; }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { switch (fieldName) { case QueryBuilder.DEFS: case QueryBuilder.REFS: return in; default: return new LowerCaseFilter(in); } } }