@Override public Tokenizer create(AttributeFactory factory) { return new EdgeNGramTokenizer(factory, minGramSize, maxGramSize); } }
@Override protected Tokenizer create(Version version) { return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE); } },
public EdgeNGramTokenizer create(Reader input) { return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize); } }
@Override public Tokenizer create() { if (matcher == null) { return new EdgeNGramTokenizer(minGram, maxGram); } else { return new EdgeNGramTokenizer(minGram, maxGram) { @Override protected boolean isTokenChar(int chr) { return matcher.isTokenChar(chr); } }; } } }
@Override public Tokenizer create() { if (matcher == null) { return new EdgeNGramTokenizer(minGram, maxGram); } else { return new EdgeNGramTokenizer(minGram, maxGram) { @Override protected boolean isTokenChar(int chr) { return matcher.isTokenChar(chr); } }; } } }
@Override public Tokenizer create() { if (matcher == null) { return new EdgeNGramTokenizer(minGram, maxGram); } else { return new EdgeNGramTokenizer(minGram, maxGram) { @Override protected boolean isTokenChar(int chr) { return matcher.isTokenChar(chr); } }; } } }
@Override public Tokenizer create(AttributeFactory factory) { if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) { return new EdgeNGramTokenizer(factory, minGramSize, maxGramSize); } return new Lucene43NGramTokenizer(factory, minGramSize, maxGramSize); } }
tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new, null)); tokenizers.add(PreConfiguredTokenizer.singleton("edge_ngram", () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE), null)); tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1), null)); tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new, null)); () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE), null)); tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new, null));