/** * Creates CSSToken of given type with cloning lexer state * automatically, allows to set text boundaries in input stream * @param type Type of token * @param state State of lexer, which will be copied * @param start Start position in stream * @param stop End position in stream */ public CSSToken(int type, CSSLexerState state, int start, int stop, Class<? extends Lexer> lexerClass) { this(null, type, Token.DEFAULT_CHANNEL, start, stop, lexerClass); this.ls = new CSSLexerState(state); }
ls}); }while(!(ls.isBalanced() && follow.member(c)));
/** * Consumes token until lexer state is balanced and * token from follow is matched. Matched token is also consumed */ private void consumeUntilGreedy(TokenStream input, BitSet follow) { CSSToken t = null; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ log.trace("Skipped greedy: {} follow: {}", t, follow); // consume token even if it will match input.consume(); }while(!(t.getLexerState().isBalanced() && follow.member(t.getType()))); }
/** * Consumes token until lexer state is function-balanced and * token from follow is matched. Matched token is also consumed */ private void consumeUntilGreedy(TokenStream input, BitSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) { CSSToken t = null; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ log.trace("Skipped greedy: {}", t); // consume token even if it will match input.consume(); }while(!(t.getLexerState().isBalanced(mode, ls, t) && follow.member(t.getType()))); }
/** * This function must be called to initialize lexer's state. * Because we can't change directly generated constructors. */ public void init() { this.log = org.slf4j.LoggerFactory.getLogger(getClass()); this.ls = new cz.vutbr.web.csskit.antlr.CSSLexerState(); this.tf = new cz.vutbr.web.csskit.antlr.CSSTokenFactory(input, state, ls, getClass()); this.tr = new cz.vutbr.web.csskit.antlr.CSSTokenRecovery(this, input, state, ls, log); this.er = new cz.vutbr.web.csskit.antlr.CSSExpressionReader(input, log); }
/** * Consumes token until lexer state is function-balanced and * token from follow is matched. */ private void consumeUntil(TokenStream input, BitSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) { CSSToken t = null; boolean finish = false; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ // consume token if does not match finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.member(t.getType())); if (!finish) { log.trace("Skipped: {}", t); input.consume(); } }while(!finish); } }
public CSSToken make() { CSSToken t = new CSSToken(input, state.type, state.channel, state.tokenStartCharIndex, input.index()-1, lexerClass); t.setLine(state.tokenStartLine); t.setText(state.text); t.setCharPositionInLine(state.tokenStartCharPositionInLine); t.setBase(input.getBase()); // clone lexer state t.setLexerState(new CSSLexerState(ls)); return t; } }
if ( input.LA(1)==CharStream.EOF ) { if (!ls.isBalanced()) return generateEOFRecover(); return lexer.getEOFToken();