private boolean isWhite(Token tok) { int type = tok.getType(); return (type == WHITESPACE) || (type == CCOMMENT) || (type == CPPCOMMENT); }
private boolean isWhite(Token tok) { int type = tok.getType(); return (type == WHITESPACE) || (type == CCOMMENT) || (type == CPPCOMMENT); }
private boolean isWhite(Token tok) { int type = tok.getType(); return (type == WHITESPACE) || (type == CCOMMENT) || (type == CPPCOMMENT); }
/** * Returns true if the enclosed Source has more tokens. * * The EOF token is never returned by the iterator. * @throws IllegalStateException if the Source * throws a LexerException or IOException */ public boolean hasNext() { advance(); return tok.getType() != EOF; }
/** * Returns true if the enclosed Source has more tokens. * * The EOF token is never returned by the iterator. * @throws IllegalStateException if the Source * throws a LexerException or IOException */ public boolean hasNext() { advance(); return tok.getType() != EOF; }
/** * Returns true if the enclosed Source has more tokens. * * The EOF token is never returned by the iterator. * @throws IllegalStateException if the Source * throws a LexerException or IOException */ @Override public boolean hasNext() { advance(); return tok.getType() != EOF; }
public Token token() throws IOException, LexerException { Token tok = getParent().token(); if (tok.getType() != EOF) target.add(tok); return tok; }
public Token token() throws IOException, LexerException { Token tok = getParent().token(); if (tok.getType() != EOF) target.add(tok); return tok; }
public Token token() throws IOException, LexerException { Token tok = getParent().token(); if (tok.getType() != EOF) target.add(tok); return tok; }
private Token undef() throws IOException, LexerException { Token tok = source_token_nonwhite(); if (tok.getType() != IDENTIFIER) { error(tok, "Expected identifier, not " + tok.getText()); if (tok.getType() == NL || tok.getType() == EOF) return tok; } else { Macro m = macros.get(tok.getText()); if (m != null) { /* XXX error if predefined */ macros.remove(m.getName()); } } return source_skipline(true); }
private Token undef() throws IOException, LexerException { Token tok = source_token_nonwhite(); if (tok.getType() != IDENTIFIER) { error(tok, "Expected identifier, not " + tok.getText()); if (tok.getType() == NL || tok.getType() == EOF) return tok; } else { Macro m = macros.get(tok.getText()); if (m != null) { /* XXX error if predefined */ macros.remove(m.getName()); } } return source_skipline(true); }
private Token expanded_token() throws IOException, LexerException { for (;;) { Token tok = source_token(); // System.out.println("Source token is " + tok); if (tok.getType() == IDENTIFIER) { Macro m = macros.get(tok.getText()); if (m == null) return tok; if (source.isExpanding(m)) return tok; if (macro(m, tok)) continue; } return tok; } }
private Token expanded_token() throws IOException, LexerException { for (;;) { Token tok = source_token(); // System.out.println("Source token is " + tok); if (tok.getType() == IDENTIFIER) { Macro m = macros.get(tok.getText()); if (m == null) return tok; if (source.isExpanding(m)) return tok; if (macro(m, tok)) continue; } return tok; } }
private void testLexerSource(String in, int... out) throws Exception { System.out.println("Testing '" + in + "' => " + Arrays.toString(out)); StringLexerSource s = new StringLexerSource(in); for (int i = 0; i < out.length; i++) { Token tok = s.token(); System.out.println("Token is " + tok); assertEquals(out[i], tok.getType()); } assertEquals(EOF, s.token().getType()); }
@Nonnull private Token expanded_token() throws IOException, LexerException { for (;;) { Token tok = source_token(); // System.out.println("Source token is " + tok); if (tok.getType() == IDENTIFIER) { Macro m = getMacro(tok.getText()); if (m == null) return tok; if (source.isExpanding(m)) return tok; if (macro(m, tok)) continue; } return tok; } }
@Test public void testPreprocessorUnterminated() throws Exception { testInput("#ifndef X\na\n#else\nb\n"); // Bug #16 writer.close(); Token t; do { t = p.token(); LOG.warn("Remaining token " + t); } while (t.getType() != EOF); }
private Token testNumericValue(String in) throws IOException, LexerException { StringLexerSource s = new StringLexerSource(in); Token tok = s.token(); System.out.println("Token is " + tok); assertEquals(NUMBER, tok.getType()); Token eof = s.token(); assertEquals("Didn't get EOF, but " + tok, EOF, eof.getType()); return tok; }