public String getText() { StringBuilder buf = new StringBuilder(); boolean paste = false; for (Token tok : tokens) { if (tok.getType() == Token.M_PASTE) { assert paste == false : "Two sequential pastes."; paste = true; continue; } else { buf.append(tok.getText()); } if (paste) { buf.append(" #" + "# "); paste = false; } // buf.append(tokens.get(i)); } return buf.toString(); }
private int expr_char(Token token) { Object value = token.getValue(); if (value instanceof Character) return ((Character) value).charValue(); String text = String.valueOf(value); if (text.length() == 0) return 0; return text.charAt(0); }
private Token stringify(Token pos, Argument arg) { StringBuilder buf = new StringBuilder(); concat(buf, arg); // System.out.println("Concat: " + arg + " -> " + buf); StringBuilder str = new StringBuilder("\""); escape(str, buf); str.append("\""); // System.out.println("Escape: " + buf + " -> " + str); return new Token(STRING, pos.getLine(), pos.getColumn(), str.toString(), buf.toString()); }
if (!tokens.hasNext()) { error(ptok.getLine(), ptok.getColumn(), "Paste at end of expansion"); buf.append(' ').append(ptok.getText()); break; switch (tok.getType()) { case M_PASTE: break; case M_ARG: int idx = ((Integer)tok.getValue()).intValue(); concat(buf, args.get(idx)); break; break; default: buf.append(tok.getText()); break;
switch (tok.getType()) { case '(': lhs = expr(0); tok = expr_token(); if (tok.getType() != ')') { expr_untoken(tok); error(tok, "missing ) in expression"); case '-': lhs = -expr(11); break; case INTEGER: lhs = ((Number)tok.getValue()).longValue(); break; case CHARACTER: lhs = (long)((Character)tok.getValue()).charValue(); break; case IDENTIFIER: if (warnings.contains(Warning.UNDEF)) warning(tok, "Undefined token '" + tok.getText() + "' encountered in conditional."); lhs = 0; expr_untoken(tok); error(tok, "Bad token in expression: " + tok.getText()); return 0; switch (op.getType()) { case '/': if (rhs == 0) {
private boolean isWhite(Token tok) { int type = tok.getType(); return (type == WHITESPACE) || (type == CCOMMENT) || (type == CPPCOMMENT); }
Token tok = arg.next(); assert tok.getType() != M_PASTE : "Unexpected paste token"; return tok; return new Token(EOF, -1, -1, ""); /* End of macro. */ Token tok = tokens.next(); int idx; switch (tok.getType()) { case M_STRING: idx = ((Integer)tok.getValue()).intValue(); return stringify(tok, args.get(idx)); case M_ARG: idx = ((Integer)tok.getValue()).intValue();
private void concat(@Nonnull StringBuilder buf, @Nonnull Argument arg) { for (Token tok : arg) { buf.append(tok.getText()); } }
bol = true; if (include) { tok = new Token(NL, _l, _c, "\n"); tok = new Token(NL, _l, _c, new String(text)); tok = new Token(HASH); else tok = cond('#', PASTE, '#'); d = read(); if (d == '+') tok = new Token(INC); else if (d == '=') tok = new Token(PLUS_EQ); else unread(d); d = read(); if (d == '-') tok = new Token(DEC); else if (d == '=') tok = new Token(SUB_EQ); else if (d == '>') tok = new Token(ARROW); else unread(d); tok = cppcomment();
private void testNumericValue(String in, double out) throws IOException, LexerException { System.out.println("Testing '" + in + "' -> " + out); Token tok = testNumericValue(in); assertEquals(in, tok.getText()); NumericValue value = (NumericValue) tok.getValue(); assertEquals("Double mismatch", out, value.doubleValue(), 0.01d); assertEquals("Float mismatch", (float) out, value.floatValue(), 0.01f); assertEquals("Long mismatch", (long) out, value.longValue()); assertEquals("Integer mismatch", (int) out, value.intValue()); }
private Token line_token(int line, String name, String extra) { /*StringBuilder buf = new StringBuilder(); buf.append("#line ").append(line) .append(" \""); /* XXX This call to escape(name) is correct but ugly. * / MacroTokenSource.escape(buf, name); buf.append("\"").append(extra).append("\n"); return new Token(P_LINE, line, 0, buf.toString(), null); */ //* return new Token(P_LINE, line, 0, name == null ? "" : "\n#line " + line + " \"" + (properStringTokensInLinePragmas ? String.valueOf(name).replace("\\", "\\\\") : name) + "\"" + extra + "\n", null ); //*/ }
/** * Returns a description of this token, for debugging purposes. */ public String toString() { StringBuilder buf = new StringBuilder(); buf.append('[').append(getTokenName(type)); if (line != -1) { buf.append('@').append(line); if (column != -1) buf.append(',').append(column); } buf.append("]:"); if (text != null) buf.append('"').append(text).append('"'); else if (type > 3 && type < 256) buf.append( (char)type ); else buf.append('<').append(type).append('>'); if (value != null) buf.append('=').append(value); return buf.toString(); }
if (!tokens.hasNext()) { error(ptok.getLine(), ptok.getColumn(), "Paste at end of expansion"); buf.append(' ').append(ptok.getText()); break; switch (tok.getType()) { case M_PASTE: break; case M_ARG: int idx = ((Integer)tok.getValue()).intValue(); concat(buf, args.get(idx)); break; break; default: buf.append(tok.getText()); break;
switch (tok.getType()) { case '(': lhs = expr(0); tok = expr_token(); if (tok.getType() != ')') { expr_untoken(tok); error(tok, "missing ) in expression"); case '-': lhs = -expr(11); break; case INTEGER: lhs = ((Number)tok.getValue()).longValue(); break; case CHARACTER: lhs = (long)((Character)tok.getValue()).charValue(); break; case IDENTIFIER: if (warnings.contains(Warning.UNDEF)) warning(tok, "Undefined token '" + tok.getText() + "' encountered in conditional."); lhs = 0; expr_untoken(tok); error(tok, "Bad token in expression: " + tok.getText()); return 0; switch (op.getType()) { case '/': if (rhs == 0) {
private boolean isWhite(Token tok) { int type = tok.getType(); return (type == WHITESPACE) || (type == CCOMMENT) || (type == CPPCOMMENT); }
Token tok = arg.next(); assert tok.getType() != M_PASTE : "Unexpected paste token"; return tok; return new Token(EOF, -1, -1, ""); /* End of macro. */ Token tok = tokens.next(); int idx; switch (tok.getType()) { case M_STRING: idx = ((Integer)tok.getValue()).intValue(); return stringify(tok, args.get(idx)); case M_ARG: idx = ((Integer)tok.getValue()).intValue();
private void concat(StringBuilder buf, Argument arg) { Iterator<Token> it = arg.iterator(); while (it.hasNext()) { Token tok = it.next(); buf.append(tok.getText()); } }