/** * Split the text on spaces and convert each word into * a Token */ protected static List<Token> baseTokenize(String text) { String[] words = text.split(" "); List<Token> tokens = new LinkedList<Token>(); for (String word : words) { tokens.add(new Token(word)); } return tokens; }
Token distoken = new Token("disambiguator"); distoken.tag(new IntegerRepeaterDayPortion(Integer.valueOf(options.getAmbiguousTimeRange()))); ttokens.add(distoken);