Tabnine Logo
CommonTokenStream.getTokens
Code IndexAdd Tabnine to your IDE (free)

How to use
getTokens
method
in
org.antlr.v4.runtime.CommonTokenStream

Best Java code snippets using org.antlr.v4.runtime.CommonTokenStream.getTokens (Showing top 20 results out of 315)

origin: sleekbyte/tailor

  private void extractComments() {
    for (Token token : tokenStream.getTokens()) {
      if (token.getChannel() != Token.HIDDEN_CHANNEL) {
        continue;
      }
      if (ListenerUtil.isSingleLineComment(token)) {
        singleLineComments.add(token);
      }
      if (ListenerUtil.isMultilineComment(token)) {
        multilineComments.add(token);
      }
    }
  }
}
origin: graphql-java/graphql-java

List<Token> allTokens = tokens.getTokens();
if (stop != null && allTokens != null && !allTokens.isEmpty()) {
  Token last = allTokens.get(allTokens.size() - 1);
origin: org.antlr/antlr4-runtime

List<Token> tokens = tokenStream.getTokens();
origin: cflint/CFLint

@Override
public boolean hasNext() {
  if (direction < 0)
    return tokens != null && tokenIndex >= 0;
  else
    return tokens != null && tokenIndex < tokens.getTokens().size();
}
origin: cflint/CFLint

@Override
public boolean hasNext() {
  if (direction < 0)
    return tokens != null && tokenIndex >= 0;
  else
    return tokens != null && tokenIndex < tokens.getTokens().size();
}
origin: org.bitbucket.goalhub.grammar/languageTools

/**
 * Dumps all tokens to console.
 */
public void printLexerTokens() {
  for (Token token : this.tokens.getTokens()) {
    System.out.print("'" + token.getText() + "<" + token.getType() + ">' ");
  }
}
origin: cflint/CFLint

@Override
public Token next() {
  if (!hasNext()) {
    throw new NoSuchElementException();
  }
  if (tokens != null && tokenIndex >= 0) {
    Token retval = tokens.getTokens().get(tokenIndex);
    tokenIndex += direction;
    return retval;
  }
  return null;
}
origin: cflint/CFLint

@Override
public Token next() {
  if (!hasNext()) {
    throw new NoSuchElementException();
  }
  if (tokens != null && tokenIndex >= 0) {
    Token retval = tokens.getTokens().get(tokenIndex);
    tokenIndex += direction;
    return retval;
  }
  return null;
}
origin: HuaweiBigData/StreamCQL

/**
 * 在语法解析器可以定位到错误单词的基础下获取错误单词
 */
private String getOffendingSymbolWithHint(Recognizer<?, ?> recognizer, Object offendingSymbol)
{
  Token token = (Token)offendingSymbol;
  String tokenText = token.getText();
  if (tokenText.equals(SYMBOL_EOF))
  {
    List<Token> allTokens = ((org.antlr.v4.runtime.CommonTokenStream)recognizer.getInputStream()).getTokens();
    int tokensCount = allTokens.size();
    return (tokensCount < MIN_SIZE_FOR_TOKENS) ? "" : allTokens.get(tokensCount - MIN_SIZE_FOR_TOKENS)
      .getText();
  }
  return tokenText;
}
/**
origin: antlr/codebuff

public static List<CommonToken> copy(CommonTokenStream tokens) {
  List<CommonToken> copy = new ArrayList<>();
  tokens.fill();
  for (Token t : tokens.getTokens()) {
    copy.add(new CommonToken(t));
  }
  return copy;
}
origin: antlrjavaparser/antlr-java-parser

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}
origin: com.github.antlrjavaparser/antlr-java-parser

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}
origin: wavefrontHQ/java

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}
origin: espertechinc/esper

/**
 * Print the token stream to the logger.
 *
 * @param tokens to print
 */
public static void printTokens(CommonTokenStream tokens) {
  if (log.isDebugEnabled()) {
    List tokenList = tokens.getTokens();
    StringWriter writer = new StringWriter();
    PrintWriter printer = new PrintWriter(writer);
    for (int i = 0; i < tokens.size(); i++) {
      Token t = (Token) tokenList.get(i);
      String text = t.getText();
      if (text.trim().length() == 0) {
        printer.print("'" + text + "'");
      } else {
        printer.print(text);
      }
      printer.print('[');
      printer.print(t.getType());
      printer.print(']');
      printer.print(" ");
    }
    printer.println();
    log.debug("Tokens: " + writer.toString());
  }
}
origin: com.wavefront/java-lib

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}
origin: antlr/codebuff

public CodeBuffTokenStream(CommonTokenStream stream) {
  super(stream.getTokenSource());
  this.fetchedEOF = false;
  for (Token t : stream.getTokens()) {
    tokens.add(new CommonToken(t));
  }
  reset();
}
origin: antlr/codebuff

public float getWSEditDistance() throws Exception {
  List<Token> wsTokens = filter(originalTokens.getTokens(),
                 t -> t.getText().matches("\\s+")); // only count whitespace
  String originalWS = tokenText(wsTokens);
  String formattedOutput = getOutput();
  CommonTokenStream formatted_tokens = tokenize(formattedOutput, corpus.language.lexerClass);
  wsTokens = filter(formatted_tokens.getTokens(),
           t -> t.getText().matches("\\s+"));
  String formattedWS = tokenText(wsTokens);
  float editDistance = normalizedLevenshteinDistance(originalWS, formattedWS);
  return editDistance;
}
origin: antlr/intellij-plugin-v4

public static Token getTokenUnderCursor(CommonTokenStream tokens, int offset) {
  Comparator<Token> cmp = new Comparator<Token>() {
    @Override
    public int compare(Token a, Token b) {
      if ( a.getStopIndex() < b.getStartIndex() ) return -1;
      if ( a.getStartIndex() > b.getStopIndex() ) return 1;
      return 0;
    }
  };
  if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null;
  CommonToken key = new CommonToken(Token.INVALID_TYPE, "");
  key.setStartIndex(offset);
  key.setStopIndex(offset);
  List<Token> tokenList = tokens.getTokens();
  Token tokenUnderCursor = null;
  int i = Collections.binarySearch(tokenList, key, cmp);
  if ( i>=0 ) tokenUnderCursor = tokenList.get(i);
  return tokenUnderCursor;
}
origin: kasonyang/kalang

@Test
public void testLexer() throws IOException{
  this.createTokenStream().getTokens();
}
 
origin: kasonyang/kalang

@Test
public void test(){
  KalangCompiler kc = new KalangCompiler(){
    @Override
    public CodeGenerator createCodeGenerator(CompilationUnit compilationUnit) {
      return new Ast2JavaStub();
    }
  };
  kc.addSource("Test", "class{  }","Test.kl");
  kc.compile();
  CompilationUnit unit = kc.getCompilationUnit("Test");
  assert unit != null;
  CommonTokenStream ts = unit.getTokenStream();
  //the tokens contains tokens in all channels
  List<Token> tokens = ts.getTokens();
  assertEquals(5, ts.size());
  testTokenNavigator(tokens.toArray(new Token[0]),unit.getAstBuilder().getParseTree());
}
 
org.antlr.v4.runtimeCommonTokenStreamgetTokens

Popular methods of CommonTokenStream

  • <init>
    Constructs a new CommonTokenStream using the specified token source and filtering tokens to the spec
  • fill
  • get
  • getHiddenTokensToLeft
  • size
  • LA
  • getHiddenTokensToRight
  • reset
  • seek
  • LB
  • getText
  • getTokenSource
  • getText,
  • getTokenSource,
  • lazyInit,
  • nextTokenOnChannel,
  • previousTokenOnChannel,
  • sync,
  • LT,
  • consume,
  • index

Popular in Java

  • Finding current android device location
  • getOriginalFilename (MultipartFile)
    Return the original filename in the client's filesystem.This may contain path information depending
  • getExternalFilesDir (Context)
  • onRequestPermissionsResult (Fragment)
  • InputStreamReader (java.io)
    A class for turning a byte stream into a character stream. Data read from the source input stream is
  • OutputStream (java.io)
    A writable sink for bytes.Most clients will use output streams that write data to the file system (
  • URI (java.net)
    A Uniform Resource Identifier that identifies an abstract or physical resource, as specified by RFC
  • DataSource (javax.sql)
    An interface for the creation of Connection objects which represent a connection to a database. This
  • BoxLayout (javax.swing)
  • JTable (javax.swing)
  • Best plugins for Eclipse
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now