public CodeBuffTokenStream(CommonTokenStream stream) { super(stream.getTokenSource()); this.fetchedEOF = false; for (Token t : stream.getTokens()) { tokens.add(new CommonToken(t)); } reset(); }
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream(); String input = tokens.getTokenSource().getInputStream().toString(); Token token = (Token) offendingSymbol; String[] lines = StringUtils.splitPreserveAllTokens(input, '\n'); String errorLine = lines[line - 1]; String simpleMessage = "syntax error at or near \"" + token.getText() + "\""; throw new SQLParseError(token, line, charPositionInLine, simpleMessage, errorLine); } }
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream(); String input = tokens.getTokenSource().getInputStream().toString(); Token token = (Token) offendingSymbol; String[] lines = StringUtils.splitPreserveAllTokens(input, '\n'); String errorLine = lines[line - 1]; String simpleMessage = "syntax error at or near \"" + token.getText() + "\""; throw new SQLParseError(token, line, charPositionInLine, simpleMessage, errorLine); } }
@Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { String offendingInput; if (Parser.class.isAssignableFrom(recognizer.getClass())) { List<String> stack = ((Parser) recognizer).getRuleInvocationStack(); Collections.reverse(stack); offendingInput = ((CommonTokenStream) recognizer.getInputStream()).getTokenSource().getInputStream().toString(); } else { offendingInput = recognizer.getInputStream().toString(); } if (e != null) { throw new SightlyCompilerException(msg, offendingInput, line, charPositionInLine, e); } throw new SightlyCompilerException(msg, offendingInput, line, charPositionInLine); } }
public static Token getTokenUnderCursor(CommonTokenStream tokens, int offset) { Comparator<Token> cmp = new Comparator<Token>() { @Override public int compare(Token a, Token b) { if ( a.getStopIndex() < b.getStartIndex() ) return -1; if ( a.getStartIndex() > b.getStopIndex() ) return 1; return 0; } }; if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null; CommonToken key = new CommonToken(Token.INVALID_TYPE, ""); key.setStartIndex(offset); key.setStopIndex(offset); List<Token> tokenList = tokens.getTokens(); Token tokenUnderCursor = null; int i = Collections.binarySearch(tokenList, key, cmp); if ( i>=0 ) tokenUnderCursor = tokenList.get(i); return tokenUnderCursor; }
CFSCRIPTParser parser = new CFSCRIPTParser(tokens); parser.removeErrorListeners(); if (tokens.getTokenSource() instanceof CFSCRIPTLexer) { ((CFSCRIPTLexer) tokens.getTokenSource()).addErrorListener(errorReporter); ((CFSCRIPTLexer) tokens.getTokenSource()).removeErrorListeners();
public static Token getSkippedTokenUnderCursor(CommonTokenStream tokens, int offset) { if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null; Token prevToken = null; Token tokenUnderCursor = null; if ( (prevToken==null || offset > prevToken.getStopIndex()) && offset < begin ) { TokenSource tokenSource = tokens.getTokenSource(); CharStream inputStream = null; if ( tokenSource!=null ) {
public static Token nextRealToken(CommonTokenStream tokens, int i) { int n = tokens.size(); i++; // search after current i token if ( i>=n || i<0 ) return null; Token t = tokens.get(i); while ( t.getChannel()==Token.HIDDEN_CHANNEL ) { if ( t.getType()==Token.EOF ) { TokenSource tokenSource = tokens.getTokenSource(); if ( tokenSource==null ) { return new CommonToken(Token.EOF, "EOF"); } TokenFactory<?> tokenFactory = tokenSource.getTokenFactory(); if ( tokenFactory==null ) { return new CommonToken(Token.EOF, "EOF"); } return tokenFactory.create(Token.EOF, "EOF"); } i++; if ( i>=n ) return null; // just in case no EOF t = tokens.get(i); } return t; }