public boolean isHidden() { return token.getChannel() == Lexer.HIDDEN; } }
/** * Return the next token from the character stream and records this last * token in case it resides on the default channel. This recorded token * is used to determine when the lexer could possibly match a regex * literal. * * @return the next token from the character stream. */ @Override public Token nextToken() { // Get the next token. Token next = super.nextToken(); if (next.getChannel() == Token.DEFAULT_CHANNEL) { // Keep track of the last token on the default channel. this.lastToken = next; } return next; }
public static boolean isEmptyStatement(String sql) { TokenSource tokens = getLexer(sql, ImmutableSet.of()); while (true) { Token token = tokens.nextToken(); if (token.getType() == Token.EOF) { return true; } if (token.getChannel() != Token.HIDDEN_CHANNEL) { return false; } } }
private void extractComments() { for (Token token : tokenStream.getTokens()) { if (token.getChannel() != Token.HIDDEN_CHANNEL) { continue; } if (ListenerUtil.isSingleLineComment(token)) { singleLineComments.add(token); } if (ListenerUtil.isMultilineComment(token)) { multilineComments.add(token); } } } }
if (ahead.getChannel() != Lexer.HIDDEN) {
boolean sameChannel = last.getChannel() == stop.getChannel(); if (notEOF && lastGreaterThanDocument && sameChannel) { throw bailStrategy.mkMoreTokensException(last);
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule)); } // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
protected List<Token> filterForChannel(int from, int to, int channel) { List<Token> hidden = new ArrayList<Token>(); for (int i=from; i<=to; i++) { Token t = tokens.get(i); if ( channel==-1 ) { if ( t.getChannel()!= Lexer.DEFAULT_TOKEN_CHANNEL ) hidden.add(t); } else { if ( t.getChannel()==channel ) hidden.add(t); } } if ( hidden.size()==0 ) return null; return hidden; }
protected String tokenText(TemplateOptions templateOptions, Token token) { String text = token.getText(); if (token.getChannel() == Java7Lexer.CHANNEL_COMMENT) { text = processComment(text, templateOptions); } return text; } }
/** Count EOF just once. */ public int getNumberOfOnChannelTokens() { int n = 0; fill(); for (int i = 0; i < tokens.size(); i++) { Token t = tokens.get(i); if ( t.getChannel()==channel ) n++; if ( t.getType()==Token.EOF ) break; } return n; } }
/** * Given a starting index, return the index of the previous token on * channel. Return {@code i} if {@code tokens[i]} is on channel. Return -1 * if there are no tokens on channel between {@code i} and 0. * * <p> * If {@code i} specifies an index at or after the EOF token, the EOF token * index is returned. This is due to the fact that the EOF token is treated * as though it were on every channel.</p> */ protected int previousTokenOnChannel(int i, int channel) { sync(i); if (i >= size()) { // the EOF token is on every channel return size() - 1; } while (i >= 0) { Token token = tokens.get(i); if (token.getType() == Token.EOF || token.getChannel() == channel) { return i; } i--; } return i; }
/** * Given a starting index, return the index of the next token on channel. * Return {@code i} if {@code tokens[i]} is on channel. Return the index of * the EOF token if there are no tokens on channel between {@code i} and * EOF. */ protected int nextTokenOnChannel(int i, int channel) { sync(i); if (i >= size()) { return size() - 1; } Token token = tokens.get(i); while ( token.getChannel()!=channel ) { if ( token.getType()==Token.EOF ) { return i; } i++; sync(i); token = tokens.get(i); } return i; }
/** * Constructs a new {@link CommonToken} as a copy of another {@link Token}. * * <p> * If {@code oldToken} is also a {@link CommonToken} instance, the newly * constructed token will share a reference to the {@link #text} field and * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will * be assigned the result of calling {@link #getText}, and {@link #source} * will be constructed from the result of {@link Token#getTokenSource} and * {@link Token#getInputStream}.</p> * * @param oldToken The token to copy. */ public CommonToken(Token oldToken) { type = oldToken.getType(); line = oldToken.getLine(); index = oldToken.getTokenIndex(); charPositionInLine = oldToken.getCharPositionInLine(); channel = oldToken.getChannel(); start = oldToken.getStartIndex(); stop = oldToken.getStopIndex(); if (oldToken instanceof CommonToken) { text = ((CommonToken)oldToken).text; source = ((CommonToken)oldToken).source; } else { text = oldToken.getText(); source = new Pair<TokenSource, CharStream>(oldToken.getTokenSource(), oldToken.getInputStream()); } }
protected String tokenText(TemplateOptions templateOptions, Token token) { String text = token.getText(); if (token.getChannel() == Java7Lexer.CHANNEL_COMMENT) { text = processComment(text, templateOptions); } return text; } }
/** Count EOF just once. */ public int getNumberOfOnChannelTokens() { int n = 0; fill(); for (int i = 0; i < tokens.size(); i++) { Token t = tokens.get(i); if ( t.getChannel()==channel ) n++; if ( t.getType()==Token.EOF ) break; } return n; } }
private static Token getTokenBefore(int i, CommonTokenStream tokens) { int position = i - 1; while (position >= 0) { Token t = tokens.get(position); if (t.getChannel() != 99 && t.getType() != EsperEPL2GrammarLexer.WS) { return t; } position--; } return null; }
@Override public void emit(Token token) { super.emit(token); if (token.getChannel() != HIDDEN) { lastTokenType = token.getType(); } }
@Override public void emit(Token token) { super.emit(token); if (token.getChannel() != HIDDEN) { lastTokenType = token.getType(); } }
public static boolean isEmptyStatement(String sql) { TokenSource tokens = getLexer(sql, ImmutableSet.of()); while (true) { Token token = tokens.nextToken(); if (token.getType() == Token.EOF) { return true; } if (token.getChannel() != Token.HIDDEN_CHANNEL) { return false; } } }
@Override public Token nextToken() { Token token; do { token = source().nextToken(); } while (token.getChannel() != channel()); return token; }