@Override public boolean hasNext() { return !isEndOfInput(); }
public List<Token> readAll() { List<Token> tokens = Lists.newArrayList(); while (!isEndOfInput()) { tokens.add(next()); } return tokens; }
private Token readQuotedToken(TokenType type, char closingQuote) throws FormulaSyntaxException { while(true) { if (isEndOfInput()) { throw new FormulaSyntaxException("End of input reached while looking for closing '" + closingQuote + ""); } if (nextChar() == closingQuote) { return finishToken(type, string.substring(currentTokenStart+1, currentCharIndex-1)); } } }
private Token readOperator(char c) { while (!isEndOfInput() && isOperator(peekChar())) { consumeChar(); } return finishToken(TokenType.OPERATOR); }
private Token readNumber() { while (!isEndOfInput() && isNumberPart(peekChar())) { consumeChar(); } return finishToken(TokenType.NUMBER); }
private Token readWhitespace() { while (!isEndOfInput() && isWhitespace(peekChar())) { consumeChar(); } return finishToken(TokenType.WHITESPACE); }
private Token readSymbol(TokenType tokenType) { while (!isEndOfInput() && isSymbolChar(peekChar())) { consumeChar(); } return finishToken(tokenType); }
private void expect(String string, Token... tokens) { System.out.println("Tokenizing [" + string + "]"); FormulaLexer tokenizer = new FormulaLexer(string); int expectedIndex = 0; while (!tokenizer.isEndOfInput()) { String expectedString; Token expected; if(expectedIndex < tokens.length) { expected = tokens[expectedIndex]; expectedString = expected.toString(); } else { expected = null; expectedString = "<NOTHING>"; } Token actual = tokenizer.next(); System.out.println(String.format("Expected: %15s, got %s", expectedString, actual.toString())); if(expected != null) { assertEquals("tokenStart", expected.getTokenStartColumn(), actual.getTokenStartColumn()); assertEquals("text", expected.getString(), actual.getString()); assertEquals("type", expected.getType(), actual.getType()); if (!expected.equals(actual)) { System.err.println("Unexpected result!"); throw new AssertionError(); } } expectedIndex ++; } }