@SuppressWarnings("resource")
@Override
protected TokenStream getTokenStream(String strOrig, boolean stemsAllowed, boolean stopWordsAllowed)
throws IOException {
if (stemsAllowed) {
strOrig = blankOutTags(strOrig);
CharArraySet stopWords = stopWordsAllowed ? JapaneseAnalyzer.getDefaultStopSet() : CharArraySet.EMPTY_SET;
Set<String> stopTags = stopWordsAllowed ? JapaneseAnalyzer.getDefaultStopTags() : Collections.emptySet();
return new JapaneseAnalyzer(null, Mode.SEARCH, stopWords, stopTags).tokenStream("",
new StringReader(strOrig));
} else {
JapaneseTokenizer tokenizer = new JapaneseTokenizer(null, false, Mode.NORMAL);
tokenizer.setReader(new StringReader(strOrig));
return new TagJoiningFilter(tokenizer);
}
}