private Token finishToken(TokenType type, String text) { int length = currentCharIndex - currentTokenStart; Token token = new Token(type, currentTokenStartPos, length, text); currentTokenStart = currentCharIndex; currentTokenStartPos = new SourcePos(currentLineIndex, currentColumnIndex); return token; }
private void expect(String string, Token... tokens) { System.out.println("Tokenizing [" + string + "]"); FormulaLexer tokenizer = new FormulaLexer(string); int expectedIndex = 0; while (!tokenizer.isEndOfInput()) { String expectedString; Token expected; if(expectedIndex < tokens.length) { expected = tokens[expectedIndex]; expectedString = expected.toString(); } else { expected = null; expectedString = "<NOTHING>"; } Token actual = tokenizer.next(); System.out.println(String.format("Expected: %15s, got %s", expectedString, actual.toString())); if(expected != null) { assertEquals("tokenStart", expected.getTokenStartColumn(), actual.getTokenStartColumn()); assertEquals("text", expected.getString(), actual.getString()); assertEquals("type", expected.getType(), actual.getType()); if (!expected.equals(actual)) { System.err.println("Unexpected result!"); throw new AssertionError(); } } expectedIndex ++; } }
private FormulaNode unary() { // <unary> ::= + <unary> | - <unary> | <unary2> if(!lexer.hasNext()) { throw new FormulaSyntaxException("Unexpected end of formula"); } Token token = lexer.peek(); if(token.getType() == TokenType.OPERATOR) { if(token.getString().equals("-") || token.getString().equals("+")) { Token opToken = lexer.next(); FormulaFunction op = function(opToken); FormulaNode operand = unary(); SourceRange sourceRange = new SourceRange(opToken.getStart(), operand.getSourceRange().getEnd()); return new FunctionCallNode(op, singletonList(operand), sourceRange); } } return unary2(); }
private SymbolNode symbol(Token token) { assert token.getType() == TokenType.SYMBOL; return new SymbolNode(token); }
public ConstantNode(Token token, SourceRange range) { // Enum constant from parser this(new EnumValue(ResourceId.valueOf(token.getString()))); this.sourceRange = range; }
private FormulaNode term() { // <term> ::= <factor> | <term> + <factor> | <term> - <factor> FormulaNode left = factor(); while (lexer.hasNext() && lexer.peek().isAdditiveOperator()) { FormulaFunction op = function(); FormulaNode right = factor(); left = binaryInfixCall(op, left, right); } return left; }
private FormulaNode conjunction() { FormulaNode left = equality(); while(lexer.hasNext() && lexer.peek().isAndOperator()) { FormulaFunction op = function(); FormulaNode right = equality(); left = binaryInfixCall(op, left, right); } return left; }
private FormulaNode unary2() { if(!lexer.hasNext()) { throw new FormulaSyntaxException("Unexpected end of formula"); } Token token = lexer.peek(); if(token.getType() == TokenType.OPERATOR) { if(token.getString().equals("!")) { Token opToken = lexer.next(); FormulaFunction op = function(opToken); FormulaNode operand = primary(); SourceRange sourceRange = new SourceRange(opToken.getStart(), operand.getSourceRange().getEnd()); return new FunctionCallNode(op, singletonList(operand), sourceRange); } } return primary(); }
public FormulaParser(Iterator<Token> tokens) { this.lexer = peekingIterator(filter(tokens, token -> token.getType() != TokenType.WHITESPACE)); }
@Test public void functionTokenizing() { expect("containsAll({f1},{v1})", new Token(TokenType.SYMBOL, 0, "containsAll"), new Token(TokenType.PAREN_START, 11, "("), new Token(TokenType.SYMBOL, 12, "f1"), new Token(TokenType.COMMA, 16, ","), new Token(TokenType.SYMBOL, 17, "v1"), new Token(TokenType.PAREN_END, 21, ")") ); }
private FormulaNode number() { Token token = lexer.next(); double value; try { value = Double.parseDouble(token.getString()); } catch (NumberFormatException e) { throw new FormulaSyntaxException(new SourceRange(token), "Invalid number '" + token.getString() + "': " + e.getMessage()); } return new ConstantNode(value, new SourceRange(token)); }
@Test public void booleanTokenizing() { expect("true", new Token(TokenType.BOOLEAN_LITERAL, 0, "true")); expect("false", new Token(TokenType.BOOLEAN_LITERAL, 0, "false")); expect("true&&false", new Token(TokenType.BOOLEAN_LITERAL, 0, "true"), new Token(TokenType.OPERATOR, 4, "&&"), new Token(TokenType.BOOLEAN_LITERAL, 6, "false") ); expect("(true||false)&&(false&&true)", new Token(TokenType.PAREN_START, 0, "("), new Token(TokenType.BOOLEAN_LITERAL, 1, "true"), new Token(TokenType.OPERATOR, 5, "||"), new Token(TokenType.BOOLEAN_LITERAL, 7, "false"), new Token(TokenType.PAREN_END, 12, ")"), new Token(TokenType.OPERATOR, 13, "&&"), new Token(TokenType.PAREN_START, 15, "("), new Token(TokenType.BOOLEAN_LITERAL, 16, "false"), new Token(TokenType.OPERATOR, 21, "&&"), new Token(TokenType.BOOLEAN_LITERAL, 23, "true"), new Token(TokenType.PAREN_END, 27, ")") ); }
/** * Consume and return the next token if it is of type {@code tokenType}, or * throw an {@link FormulaSyntaxException} if there are no remaining tokens or * the next token is of the wrong type. * @param tokenType * @return */ private Token expect(TokenType tokenType) { if(lexer.hasNext()) { Token token = lexer.next(); if (token.getType() == tokenType) { return token; } } throw new FormulaSyntaxException("Expected " + tokenType); }