/** Test routine */ public static void main(String[] argv) { NGramTokenizer tokenizer = NGramTokenizer.DEFAULT_TOKENIZER; //NGramTokenizer tokenizer = new NGramTokenizer(1,1,false,SimpleTokenizer.DEFAULT_TOKENIZER); int n = 0; for (int i=0; i<argv.length; i++) { System.out.println("argument "+i+": '"+argv[i]+"'"); Token[] tokens = tokenizer.tokenize(argv[i]); for (int j=0; j<tokens.length; j++) { System.out.println("token "+(++n)+":" +" id="+tokens[j].getIndex() +" value: '"+tokens[j].getValue()+"'"); } } } }