/** * Tokenizes the given text into sentences. */ public List<String> sentenceTokenize(String text) { return language.getSentenceTokenizer().tokenize(text); }
/** * Returns true if sentences ends with a paragraph break. * @since 4.3 */ public boolean hasParagraphEndMark(Language lang) { return StringTools.isParagraphEnd(getText(), lang.getSentenceTokenizer().singleLineBreaksMarksPara()); }
public static void testSplit(String[] sentences, SentenceTokenizer sTokenizer) { StringBuilder inputString = new StringBuilder(); List<String> input = new ArrayList<>(); Collections.addAll(input, sentences); for (String s : input) { inputString.append(s); } assertEquals(input, sTokenizer.tokenize(inputString.toString())); }
if (lineBreakPos == 0) { columnCount = sentence.length(); if (!language.getSentenceTokenizer().singleLineBreaksMarksPara()) { columnCount--;
throws IOException { StringBuilder outputStr = new StringBuilder(); List<String> sentences = sentenceTokenizer.tokenize(input); for (String sentence : sentences) { List<String> tokens = tokenizer.tokenize(sentence);
/** * Returns true if sentences ends with a paragraph break. * @since 4.3 */ public boolean hasParagraphEndMark(Language lang) { return StringTools.isParagraphEnd(getText(), lang.getSentenceTokenizer().singleLineBreaksMarksPara()); }
/** * Tokenizes the given text into sentences. */ public List<String> sentenceTokenize(String text) { return language.getSentenceTokenizer().tokenize(text); }
if (lineBreakPos == 0) { columnCount = sentence.length(); if (!language.getSentenceTokenizer().singleLineBreaksMarksPara()) { columnCount--;
List<String> sentences = lang.getSentenceTokenizer().tokenize(aText);