public LexerInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> channelNames, Collection<String> modeNames, ATN atn, CharStream input) { super(input); if (atn.grammarType != ATNType.LEXER) { throw new IllegalArgumentException("The ATN must be a lexer ATN."); } this.grammarFileName = grammarFileName; this.atn = atn; this.tokenNames = new String[atn.maxTokenType]; for (int i = 0; i < tokenNames.length; i++) { tokenNames[i] = vocabulary.getDisplayName(i); } this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]); this.channelNames = channelNames.toArray(new String[channelNames.size()]); this.modeNames = modeNames.toArray(new String[modeNames.size()]); this.vocabulary = vocabulary; this._decisionToDFA = new DFA[atn.getNumberOfDecisions()]; for (int i = 0; i < _decisionToDFA.length; i++) { _decisionToDFA[i] = new DFA(atn.getDecisionState(i), i); } this._interp = new LexerATNSimulator(this,atn,_decisionToDFA,_sharedContextCache); }
ATN atn = new ATN(grammarType, maxTokenType); atn.addState(null); continue; endStateNumbers.add(new Pair<BlockStartState, Integer>((BlockStartState)s, endStateNumber)); atn.addState(s); BasicBlockStartState bypassStart = new BasicBlockStartState(); bypassStart.ruleIndex = i; atn.addState(bypassStart); atn.addState(bypassStop); atn.defineDecisionState(bypassStart); atn.addState(matchState); matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i])); bypassStart.addTransition(new EpsilonTransition(matchState));
Token token = tokens.LT(1); ATNState state = recognizer.getInterpreter().atn.states.get(recognizer.getState()); IntervalSet nextTokens = recognizer.getATN().nextTokens(state); if (nextTokens.contains(token.getType())) { nextTokensContext = null;
@Override public void clearDFA() { for (int d = 0; d < decisionToDFA.length; d++) { decisionToDFA[d] = new DFA(atn.getDecisionState(d), d); } }
ATN atn = new ATN(grammarType, maxTokenType); atn.addState(null); continue; endStateNumbers.add(Tuple.create((BlockStartState)s, endStateNumber)); atn.addState(s); int previousSetCount = sets.size(); p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP)); atn.setHasUnicodeSMPTransitions(sets.size() > previousSetCount); BasicBlockStartState bypassStart = new BasicBlockStartState(); bypassStart.ruleIndex = i; atn.addState(bypassStart); atn.addState(bypassStop); atn.defineDecisionState(bypassStart); atn.addState(matchState); matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i])); bypassStart.addTransition(new EpsilonTransition(matchState));
public void defineMode(@NotNull String name, @NotNull TokensStartState s) { modeNameToStartState.put(name, s); modeToStartState.add(s); modeToDFA = Arrays.copyOf(modeToDFA, modeToStartState.size()); modeToDFA[modeToDFA.length - 1] = new DFA(s); defineDecisionState(s); }
public ATNState newState(GrammarAST node) { ATNState n = new BasicState(); n.setRuleIndex(currentRule.index); atn.addState(n); return n; }
@Override public void valueChanged(ListSelectionEvent e) { // previewState, project set later if ( e.getValueIsAdjusting() ) { return; // this seems to be "mouse down" but not mouse up } // get state for current grammar editor tab if ( project==null ) { return; } if ( previewState!=null && profilerDataTable.getModel().getClass()!=DefaultTableModel.class ) { int selectedRow = profilerDataTable.getSelectedRow(); if ( selectedRow==-1 ) { selectedRow = 0; } int decision = profilerDataTable.convertRowIndexToModel(selectedRow); int numberOfDecisions = previewState.g.atn.getNumberOfDecisions(); if ( decision<=numberOfDecisions ) { selectDecisionInGrammar(previewState, decision); highlightInputPhrases(previewState, decision); } } } }
public Handle elemList(List<Handle> els) { int n = els.size(); for (int i = 0; i < n - 1; i++) { // hook up elements (visit all but last) Handle el = els.get(i); // if el is of form o-x->o for x in {rule, action, pred, token, ...} // and not last in alt Transition tr = null; if ( el.left.getNumberOfTransitions()==1 ) tr = el.left.transition(0); boolean isRuleTrans = tr instanceof RuleTransition; if ( el.left.getStateType() == ATNState.BASIC && el.right.getStateType()== ATNState.BASIC && tr!=null && (isRuleTrans && ((RuleTransition)tr).followState == el.right || tr.target == el.right) ) { // we can avoid epsilon edge to next el if ( isRuleTrans ) ((RuleTransition)tr).followState = els.get(i+1).left; else tr.target = els.get(i+1).left; atn.removeState(el.right); // we skipped over this state } else { // need epsilon if previous block's right end node is complicated epsilon(el.right, els.get(i+1).left); } } Handle first = els.get(0); Handle last = els.get(n -1); if ( first==null || last==null ) { g.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, "element list has first|last == null"); } return new Handle(first.left, last.right); }
public ParserATNFactory(Grammar g) { if (g == null) { throw new NullPointerException("g"); } this.g = g; ATNType atnType = g instanceof LexerGrammar ? ATNType.LEXER : ATNType.PARSER; int maxTokenType = g.getMaxTokenType(); this.atn = new ATN(atnType, maxTokenType); }
/** * Gets the set of input symbols which could potentially follow the * previously matched symbol at the time this exception was thrown. * * <p>If the set of expected tokens is not known and could not be computed, * this method returns {@code null}.</p> * * @return The set of token types that could potentially follow the current * state in the ATN, or {@code null} if the information is not available. */ public IntervalSet getExpectedTokens() { if (recognizer != null) { return recognizer.getATN().getExpectedTokens(offendingState, ctx); } return null; }
@Override public void clearDFA() { for (int d = 0; d < decisionToDFA.length; d++) { decisionToDFA[d] = new DFA(atn.getDecisionState(d), d); } }
atn.modeNameToStartState.put(modeName, startState); atn.modeToStartState.add(startState); atn.defineDecisionState(startState);
public ATNState newState(GrammarAST node) { ATNState n = new BasicState(); n.setRuleIndex(currentRule.index); atn.addState(n); return n; }
protected void processParser() { g.decisionLOOK = new ArrayList<IntervalSet[]>(g.atn.getNumberOfDecisions()+1); for (DecisionState s : g.atn.decisionToState) { g.tool.log("LL1", "\nDECISION "+s.decision+" in rule "+g.getRule(s.ruleIndex).name); IntervalSet[] look; if ( s.nonGreedy ) { // nongreedy decisions can't be LL(1) look = new IntervalSet[s.getNumberOfTransitions()+1]; } else { LL1Analyzer anal = new LL1Analyzer(g.atn); look = anal.getDecisionLookahead(s); g.tool.log("LL1", "look=" + Arrays.toString(look)); } assert s.decision + 1 >= g.decisionLOOK.size(); Utils.setSize(g.decisionLOOK, s.decision+1); g.decisionLOOK.set(s.decision, look); g.tool.log("LL1", "LL(1)? " + disjoint(look)); } }
public Handle elemList(List<Handle> els) { int n = els.size(); for (int i = 0; i < n - 1; i++) { // hook up elements (visit all but last) Handle el = els.get(i); // if el is of form o-x->o for x in {rule, action, pred, token, ...} // and not last in alt Transition tr = null; if ( el.left.getNumberOfTransitions()==1 ) tr = el.left.transition(0); boolean isRuleTrans = tr instanceof RuleTransition; if ( el.left.getStateType() == ATNState.BASIC && el.right.getStateType()== ATNState.BASIC && tr!=null && (isRuleTrans && ((RuleTransition)tr).followState == el.right || tr.target == el.right) ) { // we can avoid epsilon edge to next el if ( isRuleTrans ) ((RuleTransition)tr).followState = els.get(i+1).left; else tr.target = els.get(i+1).left; atn.removeState(el.right); // we skipped over this state } else { // need epsilon if previous block's right end node is complicated epsilon(el.right, els.get(i+1).left); } } Handle first = els.get(0); Handle last = els.get(n -1); if ( first==null || last==null ) { g.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, "element list has first|last == null"); } return new Handle(first.left, last.right); }
public ParserATNFactory(Grammar g) { if (g == null) { throw new NullPointerException("g"); } this.g = g; ATNType atnType = g instanceof LexerGrammar ? ATNType.LEXER : ATNType.PARSER; int maxTokenType = g.getMaxTokenType(); this.atn = new ATN(atnType, maxTokenType); }
/** * Computes the set of input symbols which could follow the current parser * state and context, as given by {@link #getState} and {@link #getContext}, * respectively. * * @see ATN#getExpectedTokens(int, RuleContext) */ public IntervalSet getExpectedTokens() { return getATN().getExpectedTokens(getState(), getContext()); }
public ParserInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, ATN atn, TokenStream input) { super(input); this.grammarFileName = grammarFileName; this.atn = atn; this.tokenNames = new String[atn.maxTokenType]; for (int i = 0; i < tokenNames.length; i++) { tokenNames[i] = vocabulary.getDisplayName(i); } this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]); this.vocabulary = vocabulary; // init decision DFA int numberOfDecisions = atn.getNumberOfDecisions(); this.decisionToDFA = new DFA[numberOfDecisions]; for (int i = 0; i < numberOfDecisions; i++) { DecisionState decisionState = atn.getDecisionState(i); decisionToDFA[i] = new DFA(decisionState, i); } // get atn simulator that knows how to do predictions setInterpreter(new ParserATNSimulator(this, atn, decisionToDFA, sharedContextCache)); }
ATN atn = new ATN(grammarType, maxTokenType); atn.addState(null); continue; endStateNumbers.add(new Pair<BlockStartState, Integer>((BlockStartState)s, endStateNumber)); atn.addState(s); BasicBlockStartState bypassStart = new BasicBlockStartState(); bypassStart.ruleIndex = i; atn.addState(bypassStart); atn.addState(bypassStop); atn.defineDecisionState(bypassStart); atn.addState(matchState); matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i])); bypassStart.addTransition(new EpsilonTransition(matchState));