/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from a {@link StandardTokenizer} filtered with * {@link LowerCaseFilter}, {@link StopFilter} * , {@link SetKeywordMarkerFilter} if a stem exclusion set is * provided, and {@link SnowballFilter} */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new StopFilter(result, stopwords); if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(result, stemExclusionSet); result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer()); return new TokenStreamComponents(source, result); }
public RussianSnowballStemmer() { super(new RussianStemmer()); } public Language getLanguage() { return Language.RUSSIAN; }
@Override public TokenStream apply(final TokenStream input) { return new SnowballFilter(input, new RussianStemmer()); } };
static public TokenStream russian(TokenStream result) { result = new LowerCaseFilter(result); result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer()); return result; }
@Override public TokenStream getTokenStream(Tokenizer tokenizer, CharArraySet stemExclusionSet) { TokenStream stream = new StandardFilter(matchVersion, tokenizer); if (caseInsensitive) stream = new LowerCaseFilter(matchVersion, stream); if (useStopWords) stream = new StopFilter(matchVersion, stream, RussianAnalyzer.getDefaultStopSet()); if (useStem) { if (!stemExclusionSet.isEmpty()) stream = new SetKeywordMarkerFilter(stream, stemExclusionSet); stream = new SnowballFilter(stream, new RussianStemmer()); } return stream; } }
if (!stemExclusionSet.isEmpty()) result = new KeywordMarkerFilter( result, stemExclusionSet); result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer()); return new TokenStreamComponents(source, result); } else {
return new SnowballFilter(tokenStream, new RussianStemmer()); } else if ("light_russian".equalsIgnoreCase(language) || "lightRussian".equalsIgnoreCase(language)) { return new RussianLightStemFilter(tokenStream);
return new SnowballFilter(tokenStream, new RussianStemmer()); } else if ("light_russian".equalsIgnoreCase(language) || "lightRussian".equalsIgnoreCase(language)) { return new RussianLightStemFilter(tokenStream);
public static SnowballProgram getStemmer(String lang) { switch (lang.toUpperCase()) { case "EN": return new EnglishStemmer(); case "PT": return new PortugueseStemmer(); case "ES": return new SpanishStemmer(); case "DE": return new GermanStemmer(); case "FR": return new FrenchStemmer(); case "SV": return new SwedishStemmer(); case "IT": return new ItalianStemmer(); case "NL": return new DutchStemmer(); case "RU": return new RussianStemmer(); case "AR": case "FA": case "ZH": case "KO": return null; } return null; } }