protected Stack<Pair<Token, String>> lex(String string) { int offset = 0; Matcher wsMatcher = WS.matcher(string); List<Pair<Token, Matcher>> matcher = Lists.newArrayList(); for (Token t : Token.values()) matcher.add(Tuples.create(t, t.pattern.matcher(string))); Stack<Pair<Token, String>> result = new Stack<Pair<Token, String>>(); ROOT: while (offset < string.length()) { wsMatcher.region(offset, string.length()); if (wsMatcher.find()) { offset = wsMatcher.end(); continue ROOT; } for (Pair<Token, Matcher> m : matcher) { m.getSecond().region(offset, string.length()); if (m.getSecond().find()) { result.add(Tuples.create(m.getFirst(), m.getSecond().group(m.getSecond().groupCount() == 1 ? 1 : 0))); offset = m.getSecond().end(); continue ROOT; } } throw new RuntimeException("No valid token found at '" + string.substring(offset) + "'"); } Collections.reverse(result); return result; }
protected Stack<Pair<Token, String>> lex(String string) { int offset = 0; Matcher wsMatcher = WS.matcher(string); List<Pair<Token, Matcher>> matcher = Lists.newArrayList(); for (Token t : Token.values()) matcher.add(Tuples.create(t, t.pattern.matcher(string))); Stack<Pair<Token, String>> result = new Stack<Pair<Token, String>>(); ROOT: while (offset < string.length()) { wsMatcher.region(offset, string.length()); if (wsMatcher.find()) { offset = wsMatcher.end(); continue ROOT; } for (Pair<Token, Matcher> m : matcher) { m.getSecond().region(offset, string.length()); if (m.getSecond().find()) { result.add(Tuples.create(m.getFirst(), m.getSecond().group(m.getSecond().groupCount() == 1 ? 1 : 0))); offset = m.getSecond().end(); continue ROOT; } } throw new RuntimeException("No valid token found at '" + string.substring(offset) + "'"); } Collections.reverse(result); return result; }