public CSSToken make() { CSSToken t = new CSSToken(input, state.type, state.channel, state.tokenStartCharIndex, input.index()-1, lexerClass); t.setLine(state.tokenStartLine); t.setText(state.text); t.setCharPositionInLine(state.tokenStartCharPositionInLine); t.setBase(input.getBase()); // clone lexer state t.setLexerState(new CSSLexerState(ls)); return t; } }
/** * Returns common text stored in token. Content is not modified. * @return Model view of text in token */ @Override public String getText() { // sets text from input if not text directly available text = super.getText(); switch(type) { case CSSLexer.FUNCTION: return text.substring(0, text.length()-1); case CSSLexer.URI: return extractURI(text); case CSSLexer.STRING: return extractSTRING(text); case CSSLexer.CLASSKEYWORD: return extractCLASSKEYWORD(text); case CSSLexer.HASH: return extractHASH(text); default: return text; } }
private cz.vutbr.web.css.Declaration.Source extractSource(CommonTree token) { cz.vutbr.web.csskit.antlr.CSSToken ct = (cz.vutbr.web.csskit.antlr.CSSToken) token.getToken(); cz.vutbr.web.css.Declaration.Source src = new cz.vutbr.web.css.Declaration.Source(ct.getBase(), ct.getLine(), ct.getCharPositionInLine()); return src; }
/** * Consumes token until lexer state is balanced and * token from follow is matched. Matched token is also consumed */ private void consumeUntilGreedy(TokenStream input, BitSet follow) { CSSToken t = null; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ log.trace("Skipped greedy: {} follow: {}", t, follow); // consume token even if it will match input.consume(); }while(!(t.getLexerState().isBalanced() && follow.member(t.getType()))); }
fileName = CSSToken.extractSTRING(fileName); else fileName = CSSToken.extractURI(fileName); imports.push(stream); CSSToken t = new CSSToken(IMPORT, ls); t.setText(media.toString());
/** * Obtains the current lexer state from current token */ private cz.vutbr.web.csskit.antlr.CSSLexerState getCurrentLexerState(Token t) { if (t instanceof cz.vutbr.web.csskit.antlr.CSSToken) return ((cz.vutbr.web.csskit.antlr.CSSToken) t).getLexerState(); else return null; }
private java.net.URL extractBase(CommonTree token) { cz.vutbr.web.csskit.antlr.CSSToken ct = (cz.vutbr.web.csskit.antlr.CSSToken) token.getToken(); return ct.getBase(); }
String enc = CSSToken.extractSTRING(s.getText());
else if (mode == RecoveryMode.DECL) if (t.getType() == CSSLexer.RCURLY) //if '}' is processed the curlyNest has been already decreased return aposOpen==false && quotOpen==false && parenNest==0 && curlyNest==state.curlyNest-1; else
/** * Consumes token until lexer state is function-balanced and * token from follow is matched. Matched token is also consumed */ private void consumeUntilGreedy(TokenStream input, BitSet follow, LexerState.RecoveryMode mode, LexerState ls) { CSSToken t = null; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ log.trace("Skipped greedy: {}", t); // consume token even if it will match input.consume(); }while(!(t.getLexerState().isBalanced(mode, ls) && follow.member(t.getType()))); }
t = new CSSToken(typeMapper.get(APOS), ls, lexer.getClass()); t.setText("'"); t = new CSSToken(typeMapper.get(QUOT), ls, lexer.getClass()); t.setText("\""); t = new CSSToken(typeMapper.get(RPAREN), ls, lexer.getClass()); t.setText(")"); t = new CSSToken(typeMapper.get(RCURLY), ls, lexer.getClass()); t.setText("}");
/** * Obtains the current lexer state from current token */ private LexerState getCurrentLexerState(Token t) { if (t instanceof CSSToken) return ((CSSToken) t).getLexerState(); else return null; }
private URL extractBase(CommonTree token) { CSSToken ct = (CSSToken) token.getToken(); return ct.getBase(); }
ls.aposOpen = false; state.token = (Token) new CSSToken(typeMapper.get(INVALID_STRING), ls, lexer.getClass()); state.token.setText("INVALID_STRING"); ls.quotOpen = false; ls.aposOpen = false; state.token = (Token) new CSSToken(typeMapper.get(STRING), ls, state.tokenStartCharIndex, input.index() - 1, lexer.getClass());
String enc = cz.vutbr.web.csskit.antlr.CSSToken.extractSTRING(s.getText());
/** * Adds contextual information about n { esting into token. */ @Override public Token emit() { CSSToken t = new CSSToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); t.setLine(state.tokenStartLine); t.setText(state.text); t.setCharPositionInLine(state.tokenStartCharPositionInLine); t.setBase(((CSSInputStream) input).getBase()); // clone lexer state t.setLexerState(new LexerState(ls)); emit(t); return t; }
return text.substring(0, text.length()-1); case URI: return extractURI(text); case STRING: return extractSTRING(text); case CLASSKEYWORD: return extractCLASSKEYWORD(text); case HASH: return extractHASH(text); default: return text;
/** * Consumes token until lexer state is balanced and * token from follow is matched. Matched token is also consumed */ private void consumeUntilGreedy(TokenStream input, BitSet follow) { CSSToken t = null; do{ Token next = input.LT(1); if (next instanceof CSSToken) t= (CSSToken) input.LT(1); else break; /* not a CSSToken, probably EOF */ log.trace("Skipped greedy: {} follow: {}", t, follow); // consume token even if it will match input.consume(); }while(!(t.getLexerState().isBalanced() && follow.member(t.getType()))); }