@Override public IElementType merge(final IElementType type, final Lexer originalLexer) { IElementType mergedTokenType; if (type != DATA) { while (true) { IElementType originalTokenType = originalLexer.getTokenType(); if (originalTokenType != null && originalTokenType != DATA) { originalLexer.advance(); } else { break; } } mergedTokenType = EEX; } else { mergedTokenType = type; } return mergedTokenType; } }
public void advance() { if (eexLexer.getTokenType() == ELIXIR) { elixirLexer.advance(); if (elixirLexer.getTokenType() == null) { eexLexer.advance(); } } else { eexLexer.advance(); if (eexLexer.getTokenType() == ELIXIR) { // start automatically does equivalent of `advance` since `elixirLexer` is also a look-ahead lexer elixirLexer.start(getBufferSequence(), eexLexer.getTokenStart(), eexLexer.getTokenEnd()); } } }
@Override public void advance() { if (afterPrefix) { myDelegate.advance(); } afterPrefix = true; afterEOF |= delegateEOF; delegateEOF |= myDelegate.getTokenType() == null; }
private void lexToken(Lexer lexer) { myTokenType = lexer.getTokenType(); if (myTokenType == LEFT_BRACE_CODE_START) { myTokenType = LEFT_BRACE; } myTokenStart = lexer.getTokenStart(); myState = lexer.getState(); myTokenEnd = lexer.getTokenEnd(); lexer.advance(); } }
@Override public IElementType merge(IElementType type, Lexer lexer) { for (MergeTuple currentTuple : mergeTuples) { TokenSet tokensToMerge = currentTuple.getTokensToMerge(); if (tokensToMerge.contains(type)) { IElementType current = lexer.getTokenType(); //merge all upcoming tokens into the target token type while (tokensToMerge.contains(current)) { lexer.advance(); current = lexer.getTokenType(); } return currentTuple.getTargetType(); } } return type; } }
@Override public void advance() { IElementType tokenType = myDelegate.getTokenType(); if (tokenType == BashTokenTypes.COMMENT) { scanWordsInToken(UsageSearchContext.IN_COMMENTS, false, false); advanceTodoItemCountsInToken(); } else if (tokenType == BashTokenTypes.STRING2) { addOccurrenceInToken(UsageSearchContext.IN_CODE | UsageSearchContext.IN_STRINGS | UsageSearchContext.IN_FOREIGN_LANGUAGES); scanWordsInToken(UsageSearchContext.IN_CODE | UsageSearchContext.IN_STRINGS, true, true); } else { addOccurrenceInToken(UsageSearchContext.IN_CODE | UsageSearchContext.IN_STRINGS); scanWordsInToken(UsageSearchContext.IN_CODE | UsageSearchContext.IN_STRINGS, true, false); } myDelegate.advance(); } }