private CharSequence getPossibleTagName() { int actChar; int prev_read = input.readLength(); //remember the size of the read sequence int read = 0; while(true) { actChar = input.read(); read++; if(!(Character.isLetter(actChar) || Character.isDigit(actChar) || (actChar == '_') || (actChar == '-') || (actChar == ':') || (actChar == '.') || (actChar == '/')) || (actChar == EOF)) { // EOL or not alpha //end of tagname CharSequence tagName = input.readText().subSequence(prev_read, prev_read + read - 1); input.backup(read); //put the lookahead text back to the buffer return tagName; } } }
@SuppressWarnings("fallthrough") protected final int read() { boolean skipEscapedLF = true; int c = input.read(); if (skipEscapedLF) { // skip escaped LF int next; while (c == '\\') { switch (input.read()) { case '\r': input.consumeNewline(); // nobreak case '\n': escapedLF = true; next = input.read(); break; default: input.backup(1); assert c == '\\' : "must be backslash " + (char)c; return c; // normal backslash, not escaped LF } c = next; } } return c; }
@Override public Token<CPTokenId> nextToken() { //just read whole input while(input.read() != LexerInput.EOF) {}; //and create one big token return input.readLength() > 0 ? tokenFactory.createToken(CPTokenId.CSS) : null; }
public Token<RubyStringTokenId> nextTokenSingleQuotes() { while (true) { int ch = input.read(); if (input.readLength() > 0) { return token(RubyStringTokenId.STRING_TEXT); } else { if (input.readLength() > 1) { // already read some text input.backup(1); input.readLength()); switch (ch = input.read()) { case '\\': case '\'':
private int nextChar() { previousLength = currentLength; int c = input.read(); currentLength = 1; return c; }
case IN_ERROR: { for (int i = 1, n = string.length(); i < n; i++) { int c = lexerInput.read(); assert c == string.charAt(i) : string + i + ":" + (char)c; int c = lexerInput.read(); assert c == '"' || c == '\'' : (char)c; stringMode = NO_LITERAL; } else { for (int i = 1, n = string.length()-1; i < n; i++) { int c = lexerInput.read(); assert c == string.charAt(i) : string + i + ":" + (char)c; int c = lexerInput.read(); assert c == '"' || c == '\'' : (char)c; stringMode = NO_LITERAL; int c = lexerInput.read(); assert c == '/' : (char)c; stringMode = NO_LITERAL; int last = string.lastIndexOf('/'); for (int i = 1; i < last; i++) { int c = lexerInput.read(); assert c == string.charAt(i) : string + i + ":" + (char)c; int last = string.lastIndexOf('/'); for (int i = last, n = string.length(); i < n; i++) {
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
private void ungetCharIgnoreLineEnd(int c) { // <netbeans> if (lexerInput != null) { lexerInput.backup(1); return; } // </netbeans> ungetBuffer[ungetCursor++] = c; }
private void checkToken(JspTokenId tokenId) { if(input.readLength() == 0) { LOGGER.log(Level.INFO, "Found zero length token: "); } LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] token ('" + input.readText().toString() + "'; id=" + tokenId + "; state=" + state() + ")\n"); }
/** * Returns the text matched by the current regular expression. */ public final String yytext() { return input.readText().toString(); }
int intId = antlrToken.getType(); int len = lexerInput.readLengthEOF() - myCharBuffer.getExtraCharCount(); if (antlrToken.getText() != null) { len = Math.max(len, antlrToken.getText().length()); LOG.log(Level.FINEST, "Counting length from {0} and {1}", new Object[]{lexerInput.readLengthEOF(), myCharBuffer.getExtraCharCount()}); break; case GroovyTokenTypes.EOF: if (lexerInput.readLength() > 0) { return recovery();
if (c == '\r') { lastTokenEndedByEscapedLine--; if (input.consumeNewline()) { lastTokenEndedByEscapedLine--; return token(CppTokenId.ELLIPSIS); } else { input.backup(2);
public Token<RubyStringTokenId> nextTokenDoubleQuotes() { while (true) { int ch = input.read(); if (input.readLength() > 0) { return token(RubyStringTokenId.STRING_TEXT); } else { int f = input.read(); if (input.readLength() > 2) { // already read some text input.backup(2); input.readLength()); c = input.read(); if (input.readLength() > 1) { // already read some text input.backup(1); input.readLength()); switch (ch = input.read()) { if (isHexDigit(input.read())) { if (isHexDigit(input.read())) { return token(RubyStringTokenId.STRING_ESCAPE); // valid unicode } else { input.backup(2);
private boolean followsScriptletExpressionDelimiter(int actChar) { if(actChar == '<') { int next = input.read(); if(next == '%') { next = input.read(); if(next == '=') { //yes, it follows return true; } else { input.backup(2); //backup %+next } } else { input.backup(1); //backup next } } return false; }
public void consume() { int c = input.read(); index++; charPositionInLine++; if (c == '\n') { line++; charPositionInLine = 0; } }
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
/** * This function puts last n symbols back */ protected final void backup(int n, int lastChar) { if (lastChar == '\t') { if (lineColomn == 6) { lineColomn = n; } } lineColomn -= n; input.backup(n); }
private Token<ELTokenId> token(ELTokenId tokenId) { if(LOG) { if(input.readLength() == 0) { LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] Found zero length token: "); //NOI18N } LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] token ('" + input.readText().toString() + "'; id=" + tokenId + ")\n"); //NOI18N } return tokenFactory.createToken(tokenId); }
/** * Returns the text matched by the current regular expression. */ public final String yytext() { return input.readText().toString(); }