How to use

getTokenizingCharacters

method

in

org.languagetool.tokenizers.WordTokenizer

@Override
public List<String> tokenize(String text) {
 List<String> l = new ArrayList<>();
 StringTokenizer st = new StringTokenizer(text, getTokenizingCharacters(), true);
 while (st.hasMoreElements()) {
  l.add(st.nextToken());
 }
 return joinEMailsAndUrls(l);
}

 @Override
 public String getTokenizingCharacters() {
  return super.getTokenizingCharacters() + "-";
 } 
}

public PolishWordTokenizer() {
 plTokenizing = super.getTokenizingCharacters() + "–";   // n-dash
}

public DutchWordTokenizer() {
 //remove the apostrophe etc. from the standard tokenizing characters:
 String chars = super.getTokenizingCharacters();
 for (String quote : QUOTES) {
  chars = chars.replace(quote, "");
 }
 nlTokenizingChars = chars;
}

@Override
public List<String> tokenize(String text) {
 List<String> l = new ArrayList<>();
 StringTokenizer st = new StringTokenizer(text, getTokenizingCharacters(), true);
 while (st.hasMoreElements()) {
  l.add(st.nextToken());
 }
 return joinEMailsAndUrls(l);
}