congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
CommonTokenStream.fill
Code IndexAdd Tabnine to your IDE (free)

How to use
fill
method
in
org.antlr.v4.runtime.CommonTokenStream

Best Java code snippets using org.antlr.v4.runtime.CommonTokenStream.fill (Showing top 20 results out of 315)

origin: org.antlr/antlr4-runtime

  /** Count EOF just once. */
  public int getNumberOfOnChannelTokens() {
    int n = 0;
    fill();
    for (int i = 0; i < tokens.size(); i++) {
      Token t = tokens.get(i);
      if ( t.getChannel()==channel ) n++;
      if ( t.getType()==Token.EOF ) break;
    }
    return n;
  }
}
origin: org.antlr/antlr4-runtime

CommonTokenStream tokenStream = new CommonTokenStream(lexer);
try {
  tokenStream.fill();
origin: com.tunnelvisionlabs/antlr4-runtime

  /** Count EOF just once. */
  public int getNumberOfOnChannelTokens() {
    int n = 0;
    fill();
    for (int i = 0; i < tokens.size(); i++) {
      Token t = tokens.get(i);
      if ( t.getChannel()==channel ) n++;
      if ( t.getType()==Token.EOF ) break;
    }
    return n;
  }
}
origin: antlr/codebuff

public static List<CommonToken> copy(CommonTokenStream tokens) {
  List<CommonToken> copy = new ArrayList<>();
  tokens.fill();
  for (Token t : tokens.getTokens()) {
    copy.add(new CommonToken(t));
  }
  return copy;
}
origin: uk.co.nichesolutions/antlr4-runtime

  /** Count EOF just once. */
  public int getNumberOfOnChannelTokens() {
    int n = 0;
    fill();
    for (int i = 0; i < tokens.size(); i++) {
      Token t = tokens.get(i);
      if ( t.getChannel()==channel ) n++;
      if ( t.getType()==Token.EOF ) break;
    }
    return n;
  }
}
origin: io.virtdata/virtdata-lib-realer

  /** Count EOF just once. */
  public int getNumberOfOnChannelTokens() {
    int n = 0;
    fill();
    for (int i = 0; i < tokens.size(); i++) {
      Token t = tokens.get(i);
      if ( t.getChannel()==channel ) n++;
      if ( t.getType()==Token.EOF ) break;
    }
    return n;
  }
}
origin: antlrjavaparser/antlr-java-parser

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}
origin: com.github.antlrjavaparser/antlr-java-parser

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}
origin: com.impetus.fabric/fabric-jdbc-driver-shaded

  /** Count EOF just once. */
  public int getNumberOfOnChannelTokens() {
    int n = 0;
    fill();
    for (int i = 0; i < tokens.size(); i++) {
      Token t = tokens.get(i);
      if ( t.getChannel()==channel ) n++;
      if ( t.getType()==Token.EOF ) break;
    }
    return n;
  }
}
origin: antlr/intellij-plugin-v4

public static CommonTokenStream tokenizeANTLRGrammar(String text) {
  ANTLRInputStream input = new ANTLRInputStream(text);
  ANTLRv4Lexer lexer = new ANTLRv4Lexer(input);
  CommonTokenStream tokens = new TokenStreamSubset(lexer);
  tokens.fill();
  return tokens;
}
origin: wavefrontHQ/java

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}
origin: com.wavefront/java-lib

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}
origin: com.github.julianthome/inmemantlr-api

/**
 * get the tokens for a string -- only for lexer grammars
 * @param toParse string to be tokenized
 * @return a list of tokens
 * @throws IllegalWorkflowException in case no lexername is provided
 */
public List<Token> lex(String toParse) throws IllegalWorkflowException {
  if (lexerName.isEmpty())
    throw new IllegalWorkflowException("lexerName must not be empty " +
        "-- did you already run compile?");
  InmemantlrErrorListener el = new InmemantlrErrorListener();
  listener.reset();
  //CodePointCharStream input = CharStreams.fromString(toParse);
  CharStream input = provider.getCharStream(toParse);
  Objects.requireNonNull(input, "char stream must not be null");
  LOGGER.debug("load lexer {}", lexerName);
  Lexer lex = sc.instanciateLexer(input, lexerName, useCached);
  lex.addErrorListener(el);
  Objects.requireNonNull(lex, "lex must not be null");
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  return tokens.getTokens();
}
origin: apache/eagle

  public ORExpression parse() throws EagleQueryParseException {
    try {
      EagleFilterLexer lexer = new EagleFilterLexer(new ANTLRInputStream(query));
      CommonTokenStream tokens = new CommonTokenStream(lexer);
      tokens.fill();
      EagleFilterParser p = new EagleFilterParser(tokens);
      p.setErrorHandler(new EagleANTLRErrorStrategy());
      p.setBuildParseTree(true);
      EagleQueryFilterListenerImpl listener = new EagleQueryFilterListenerImpl();
      p.addParseListener(listener);
      EagleFilterParser.FilterContext fc = p.filter();
      if (fc.exception != null) {
        LOG.error("Can not successfully parse the query:" + query, fc.exception);
        throw fc.exception;
      }
      return listener.result();
    } catch (Exception ex) {
      LOG.error("Can not successfully parse the query:", ex);
      throw new EagleQueryParseException("can not successfully parse the query:" + query);
    }
  }
}
origin: org.apache.eagle/eagle-antlr

  public ORExpression parse() throws EagleQueryParseException{
    try{
      EagleFilterLexer lexer = new EagleFilterLexer(new ANTLRInputStream(_query));
      CommonTokenStream tokens = new CommonTokenStream(lexer);
      tokens.fill();
      EagleFilterParser p = new EagleFilterParser(tokens);
      p.setErrorHandler(new EagleANTLRErrorStrategy());
      p.setBuildParseTree(true);
      EagleQueryFilterListenerImpl listener = new EagleQueryFilterListenerImpl(); 
      p.addParseListener(listener);
      EagleFilterParser.FilterContext fc = p.filter();
      if(fc.exception != null){
        LOG.error("Can not successfully parse the query:" + _query, fc.exception);
        throw fc.exception;
      }
      return listener.result();
    }catch(Exception ex){
      LOG.error("Can not successfully parse the query:", ex);
      throw new EagleQueryParseException("can not successfully parse the query:" + _query);
    }
  }
}
origin: com.impetus.fabric/fabric-jdbc-driver-shaded

CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
origin: org.antlr/antlr4

CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
origin: io.virtdata/virtdata-lib-realer

CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
origin: com.tunnelvisionlabs/antlr4

CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
origin: org.sonarsource.slang/slang-antlr

@Override
public Tree parse(String slangCode) {
 SLangLexer lexer = new SLangLexer(CharStreams.fromString(slangCode));
 List<Comment> comments = new ArrayList<>();
 CommonTokenStream antlrTokens = new CommonTokenStream(lexer);
 antlrTokens.fill();
 List<org.sonarsource.slang.api.Token> tokens = new ArrayList<>();
 for (int index = 0; index < antlrTokens.size(); index++) {
  Token token = antlrTokens.get(index);
  TextRange textRange = getSlangTextRange(token);
  if (token.getChannel() == 1) {
   comments.add(comment(token, textRange));
  } else {
   Type type = Type.OTHER;
   if (KEYWORD_TOKEN_TYPES.contains(token.getType())) {
    type = Type.KEYWORD;
   } else if (token.getType() == SLangParser.StringLiteral) {
    type = Type.STRING_LITERAL;
   }
   tokens.add(new TokenImpl(textRange, token.getText(), type));
  }
 }
 SLangParser parser = new SLangParser(antlrTokens);
 parser.setErrorHandler(new ErrorStrategy());
 SLangParseTreeVisitor slangVisitor = new SLangParseTreeVisitor(comments, tokens);
 return slangVisitor.visit(parser.slangFile());
}
org.antlr.v4.runtimeCommonTokenStreamfill

Popular methods of CommonTokenStream

  • <init>
    Constructs a new CommonTokenStream using the specified token source and filtering tokens to the spec
  • getTokens
  • get
  • getHiddenTokensToLeft
  • size
  • LA
  • getHiddenTokensToRight
  • reset
  • seek
  • LB
  • getText
  • getTokenSource
  • getText,
  • getTokenSource,
  • lazyInit,
  • nextTokenOnChannel,
  • previousTokenOnChannel,
  • sync,
  • LT,
  • consume,
  • index

Popular in Java

  • Reading from database using SQL prepared statement
  • scheduleAtFixedRate (Timer)
  • getSystemService (Context)
  • getSharedPreferences (Context)
  • GridLayout (java.awt)
    The GridLayout class is a layout manager that lays out a container's components in a rectangular gri
  • BitSet (java.util)
    The BitSet class implements abit array [http://en.wikipedia.org/wiki/Bit_array]. Each element is eit
  • Iterator (java.util)
    An iterator over a sequence of objects, such as a collection.If a collection has been changed since
  • Stream (java.util.stream)
    A sequence of elements supporting sequential and parallel aggregate operations. The following exampl
  • ZipFile (java.util.zip)
    This class provides random read access to a zip file. You pay more to read the zip file's central di
  • Logger (org.slf4j)
    The org.slf4j.Logger interface is the main user entry point of SLF4J API. It is expected that loggin
  • Top 17 Plugins for Android Studio
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now