/** * Constructor * @param source the string to be parsed */ Parser(String source,BuiltinRegistry registry) { stream = new Tokenizer(source, "()[], \t\n\r", "'\"", true); lookahead = null; this.registry=registry; }
/** * Test if there are more tokens which can be returned. */ public boolean hasMoreTokens() { if (lookahead == null) lookahead = getNextToken(); return lookahead != null; }
Tokenizer tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", true); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "("); assertEquals(tokenizer.nextToken(), "foo"); assertEquals(tokenizer.nextToken(), ","); assertEquals(tokenizer.nextToken(), "bar"); assertEquals(tokenizer.nextToken(), ")"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "'"); assertEquals(tokenizer.nextToken(), "i am a literal"); assertEquals(tokenizer.nextToken(), "'"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "so"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "there"); assertTrue( ! tokenizer.hasMoreTokens()); tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", false); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "foo"); assertEquals(tokenizer.nextToken(), "bar"); assertEquals(tokenizer.nextToken(), "i am a literal"); assertEquals(tokenizer.nextToken(), "so"); assertEquals(tokenizer.nextToken(), "there"); assertTrue( ! tokenizer.hasMoreTokens());
/** * Return the next token. * @throws java.util.NoSuchElementException if there are no more tokens available */ public String nextToken() { String result = null; if (lookahead != null) { result = lookahead; lookahead = null; } else { result = getNextToken(); } if (result == null) { throw new NoSuchElementException("No more elements in tokenized string"); } if (!returnDelims) { if (result.length() == 1) { char c = result.charAt(0); if (delim.indexOf(c) != -1 || literalDelim.indexOf(c) != -1) { return nextToken(); } } } return result; }
Tokenizer tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", true); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "("); assertEquals(tokenizer.nextToken(), "foo"); assertEquals(tokenizer.nextToken(), ","); assertEquals(tokenizer.nextToken(), "bar"); assertEquals(tokenizer.nextToken(), ")"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "'"); assertEquals(tokenizer.nextToken(), "i am a literal"); assertEquals(tokenizer.nextToken(), "'"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "so"); assertEquals(tokenizer.nextToken(), " "); assertEquals(tokenizer.nextToken(), "there"); assertTrue( ! tokenizer.hasMoreTokens()); tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", false); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "foo"); assertEquals(tokenizer.nextToken(), "bar"); assertEquals(tokenizer.nextToken(), "i am a literal"); assertEquals(tokenizer.nextToken(), "so"); assertEquals(tokenizer.nextToken(), "there"); assertTrue( ! tokenizer.hasMoreTokens());
/** * Return the next token. * @throws java.util.NoSuchElementException if there are no more tokens available */ public String nextToken() { String result = null; if (lookahead != null) { result = lookahead; lookahead = null; } else { result = getNextToken(); } if (result == null) { throw new NoSuchElementException("No more elements in tokenized string"); } if (!returnDelims) { if (result.length() == 1) { char c = result.charAt(0); if (delim.indexOf(c) != -1 || literalDelim.indexOf(c) != -1) { return nextToken(); } } } return result; }
/** * Constructor * @param source the string to be parsed */ Parser(String source,BuiltinRegistry registry) { stream = new Tokenizer(source, "()[], \t\n\r", "'\"", true); lookahead = null; this.registry=registry; }
/** * Test if there are more tokens which can be returned. */ public boolean hasMoreTokens() { if (lookahead == null) lookahead = getNextToken(); return lookahead != null; }