private void extractComments() { for (Token token : tokenStream.getTokens()) { if (token.getChannel() != Token.HIDDEN_CHANNEL) { continue; } if (ListenerUtil.isSingleLineComment(token)) { singleLineComments.add(token); } if (ListenerUtil.isMultilineComment(token)) { multilineComments.add(token); } } } }
List<Token> allTokens = tokens.getTokens(); if (stop != null && allTokens != null && !allTokens.isEmpty()) { Token last = allTokens.get(allTokens.size() - 1);
List<Token> tokens = tokenStream.getTokens();
@Override public boolean hasNext() { if (direction < 0) return tokens != null && tokenIndex >= 0; else return tokens != null && tokenIndex < tokens.getTokens().size(); }
@Override public boolean hasNext() { if (direction < 0) return tokens != null && tokenIndex >= 0; else return tokens != null && tokenIndex < tokens.getTokens().size(); }
/** * Dumps all tokens to console. */ public void printLexerTokens() { for (Token token : this.tokens.getTokens()) { System.out.print("'" + token.getText() + "<" + token.getType() + ">' "); } }
@Override public Token next() { if (!hasNext()) { throw new NoSuchElementException(); } if (tokens != null && tokenIndex >= 0) { Token retval = tokens.getTokens().get(tokenIndex); tokenIndex += direction; return retval; } return null; }
@Override public Token next() { if (!hasNext()) { throw new NoSuchElementException(); } if (tokens != null && tokenIndex >= 0) { Token retval = tokens.getTokens().get(tokenIndex); tokenIndex += direction; return retval; } return null; }
/** * 在语法解析器可以定位到错误单词的基础下获取错误单词 */ private String getOffendingSymbolWithHint(Recognizer<?, ?> recognizer, Object offendingSymbol) { Token token = (Token)offendingSymbol; String tokenText = token.getText(); if (tokenText.equals(SYMBOL_EOF)) { List<Token> allTokens = ((org.antlr.v4.runtime.CommonTokenStream)recognizer.getInputStream()).getTokens(); int tokensCount = allTokens.size(); return (tokensCount < MIN_SIZE_FOR_TOKENS) ? "" : allTokens.get(tokensCount - MIN_SIZE_FOR_TOKENS) .getText(); } return tokenText; } /**
public static List<CommonToken> copy(CommonTokenStream tokens) { List<CommonToken> copy = new ArrayList<>(); tokens.fill(); for (Token t : tokens.getTokens()) { copy.add(new CommonToken(t)); } return copy; }
public static void printLex(InputStream in) throws Exception { Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in)); CommonTokenStream tokens = new CommonTokenStream(lex); tokens.fill(); for (Token token : tokens.getTokens()) { System.out.println(token.getType() + " " + token.getText()); } }
public static void printLex(InputStream in) throws Exception { Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in)); CommonTokenStream tokens = new CommonTokenStream(lex); tokens.fill(); for (Token token : tokens.getTokens()) { System.out.println(token.getType() + " " + token.getText()); } }
protected Queue<Token> getQueue(String input) { DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get(); lexer.setInputStream(new ANTLRInputStream(input)); CommonTokenStream commonTokenStream = new CommonTokenStream(lexer); commonTokenStream.fill(); List<Token> tokens = commonTokenStream.getTokens(); if (tokens.isEmpty()) { throw new RuntimeException("Could not parse: " + input); } // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so // means we need to be creating the AST and instead we could just use the lexer. in any case, // we don't expect the graphite format to change anytime soon. // filter all EOF tokens first. Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect( Collectors.toCollection(ArrayDeque::new)); return queue; }
/** * Print the token stream to the logger. * * @param tokens to print */ public static void printTokens(CommonTokenStream tokens) { if (log.isDebugEnabled()) { List tokenList = tokens.getTokens(); StringWriter writer = new StringWriter(); PrintWriter printer = new PrintWriter(writer); for (int i = 0; i < tokens.size(); i++) { Token t = (Token) tokenList.get(i); String text = t.getText(); if (text.trim().length() == 0) { printer.print("'" + text + "'"); } else { printer.print(text); } printer.print('['); printer.print(t.getType()); printer.print(']'); printer.print(" "); } printer.println(); log.debug("Tokens: " + writer.toString()); } }
protected Queue<Token> getQueue(String input) { DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get(); lexer.setInputStream(new ANTLRInputStream(input)); CommonTokenStream commonTokenStream = new CommonTokenStream(lexer); commonTokenStream.fill(); List<Token> tokens = commonTokenStream.getTokens(); if (tokens.isEmpty()) { throw new RuntimeException("Could not parse: " + input); } // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so // means we need to be creating the AST and instead we could just use the lexer. in any case, // we don't expect the graphite format to change anytime soon. // filter all EOF tokens first. Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect( Collectors.toCollection(ArrayDeque::new)); return queue; }
public CodeBuffTokenStream(CommonTokenStream stream) { super(stream.getTokenSource()); this.fetchedEOF = false; for (Token t : stream.getTokens()) { tokens.add(new CommonToken(t)); } reset(); }
public float getWSEditDistance() throws Exception { List<Token> wsTokens = filter(originalTokens.getTokens(), t -> t.getText().matches("\\s+")); // only count whitespace String originalWS = tokenText(wsTokens); String formattedOutput = getOutput(); CommonTokenStream formatted_tokens = tokenize(formattedOutput, corpus.language.lexerClass); wsTokens = filter(formatted_tokens.getTokens(), t -> t.getText().matches("\\s+")); String formattedWS = tokenText(wsTokens); float editDistance = normalizedLevenshteinDistance(originalWS, formattedWS); return editDistance; }
public static Token getTokenUnderCursor(CommonTokenStream tokens, int offset) { Comparator<Token> cmp = new Comparator<Token>() { @Override public int compare(Token a, Token b) { if ( a.getStopIndex() < b.getStartIndex() ) return -1; if ( a.getStartIndex() > b.getStopIndex() ) return 1; return 0; } }; if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null; CommonToken key = new CommonToken(Token.INVALID_TYPE, ""); key.setStartIndex(offset); key.setStopIndex(offset); List<Token> tokenList = tokens.getTokens(); Token tokenUnderCursor = null; int i = Collections.binarySearch(tokenList, key, cmp); if ( i>=0 ) tokenUnderCursor = tokenList.get(i); return tokenUnderCursor; }
@Test public void testLexer() throws IOException{ this.createTokenStream().getTokens(); }
@Test public void test(){ KalangCompiler kc = new KalangCompiler(){ @Override public CodeGenerator createCodeGenerator(CompilationUnit compilationUnit) { return new Ast2JavaStub(); } }; kc.addSource("Test", "class{ }","Test.kl"); kc.compile(); CompilationUnit unit = kc.getCompilationUnit("Test"); assert unit != null; CommonTokenStream ts = unit.getTokenStream(); //the tokens contains tokens in all channels List<Token> tokens = ts.getTokens(); assertEquals(5, ts.size()); testTokenNavigator(tokens.toArray(new Token[0]),unit.getAstBuilder().getParseTree()); }