/** * Returns the next token. * * @return the next token if it exists, <code>null</code> if no more tokens */ public Token next() { lastToken = token; token = new Token(); // Skip whitespace token.setWhitespace(getTokenOfCharClass(whitespaceSymbols)); // quoted strings currently ignored // get prepunctuation token.setPrepunctuation(getTokenOfCharClass(prepunctuationSymbols)); // get the symbol itself if (singleCharSymbols.indexOf(currentChar) != -1) { token.setWord(String.valueOf((char) currentChar)); getNextChar(); } else { token.setWord(getTokenNotOfCharClass(whitespaceSymbols)); } token.setPosition(currentPosition); token.setLineNumber(lineNumber); // This'll have token *plus* postpunctuation // Get postpunctuation removeTokenPostpunctuation(); return token; }
/** * Returns the next token. * * @return the next token if it exists, <code>null</code> if no more tokens */ public Token next() { lastToken = token; token = new Token(); // Skip whitespace token.setWhitespace(getTokenOfCharClass(whitespaceSymbols)); // quoted strings currently ignored // get prepunctuation token.setPrepunctuation(getTokenOfCharClass(prepunctuationSymbols)); // get the symbol itself if (singleCharSymbols.indexOf(currentChar) != -1) { token.setWord(String.valueOf((char) currentChar)); getNextChar(); } else { token.setWord(getTokenNotOfCharClass(whitespaceSymbols)); } token.setPosition(currentPosition); token.setLineNumber(lineNumber); // This'll have token *plus* postpunctuation // Get postpunctuation removeTokenPostpunctuation(); return token; }