@Override public Token<CPTokenId> nextToken() { //just read whole input while(input.read() != LexerInput.EOF) {}; //and create one big token return input.readLength() > 0 ? tokenFactory.createToken(CPTokenId.CSS) : null; }
@SuppressWarnings("unchecked") public CppLexer(Filter<CppTokenId> defaultFilter, LexerRestartInfo<CppTokenId> info) { super(info); Filter<CppTokenId> filter = (Filter<CppTokenId>) info.getAttributeValue(CndLexerUtilities.LEXER_FILTER); // NOI18N this.lexerFilter = filter != null ? filter : defaultFilter; }
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
public RubyCommentLexer(LexerRestartInfo<RubyCommentTokenId> info) { this.input = info.input(); this.tokenFactory = info.tokenFactory(); assert (info.state() == null); // passed argument always null }
private int nextChar() { previousLength = currentLength; int c = input.read(); currentLength = 1; return c; }
private Token<RhtmlTokenId> token(RhtmlTokenId id) { if(input.readLength() == 0) { new Exception("Error - token length is zero!; state = " + state).printStackTrace(); } Token<RhtmlTokenId> t = tokenFactory.createToken(id); return t; }
private void ungetCharIgnoreLineEnd(int c) { // <netbeans> if (lexerInput != null) { lexerInput.backup(1); return; } // </netbeans> ungetBuffer[ungetCursor++] = c; }
public PHPTopColoringLexer(LexerRestartInfo<PHPTopTokenId> info, State state) { this.input = info.input(); if (state == null) { this.state = State.OUTER; } else { this.state = state; } }
/** * A Lexer for ruby strings * @param substituting If true, handle substitution rules for double quoted strings, otherwise * single quoted strings. */ public RubyStringLexer(LexerRestartInfo<RubyStringTokenId> info, boolean substituting) { this.input = info.input(); this.tokenFactory = info.tokenFactory(); this.substituting = substituting; assert (info.state() == null); // passed argument always null }
@Override public int read() throws IOException { int c = input.read(); if (c == LexerInput.EOF) { // Private //return RubyYaccLexer.EOF; return -1; } return c; } }
/** * This function puts last n symbols back */ protected final void backup(int n, int lastChar) { if (lastChar == '\t') { if (lineColomn == 6) { lineColomn = n; } } lineColomn -= n; input.backup(n); }
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
@Override public int read(char[] buf, int off, int len) throws IOException { for (int i = 0; i < len; i++) { int c = input.read(); if (c == LexerInput.EOF) { return -1; } buf[i + off] = (char) c; } return len; }
protected final void backup(int n) { input.backup(n + escapedEatenChars); lastTokenEndedByEscapedLine = escapedEatenChars; tokenSplittedByEscapedLine -= escapedEatenChars; }
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
public void consume() { int c = input.read(); index++; charPositionInLine++; if (c == '\n') { line++; charPositionInLine = 0; } }
private int read() { int result = input.read(); if (result == LexerInput.EOF) { result = CharStream.EOF; } return result; }
public int read(char[] buf, int off, int len) throws IOException { for (int i = 0; i < len; i++) { int c = input.read(); if (c == LexerInput.EOF) { return -1; } buf[i + off] = (char)c; } return len; }
public char read () { index++; return (char) input.read (); }