/** A {@link Parser} that recognizes the token identified by {@code tokenName}. */ public Parser<Token> token(String tokenName) { return Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenName))); }
/** * Gets a {@link Lexicon} instance with {@link Tokens#reserved(String)} as each operator's value * and a lexer that strives to try the shortest operator first. * * <p> Safely speaking, we can always start from the longest operator and falls back to shorter * ones. Yet shorter operators are more often used than longer ones and the scanning of them is * faster. However, scanning shorter operators first has the chance that a "==" is mistakenly * scanned as "=" followed by another "=". In order to avoid this, we analyze the prefix * relationship and make sure that prefixes are scanned after prefixes. */ static Lexicon lexicon(final Collection<String> operatorNames) { final Map<String, Object> operators = new HashMap<String, Object>(); final String[] ops = sort(operatorNames.toArray(new String[operatorNames.size()])); final Parser<?>[] lexers = new Parser<?>[ops.length]; for (int i = 0; i < ops.length; i++) { String s = ops[i]; Parser<?> scanner = s.length() == 1 ? Scanners.isChar(s.charAt(0)) : Scanners.string(s); Object value = Tokens.reserved(s); operators.put(s, value); lexers[i] = scanner.retn(value); } return new Lexicon(operators::get, Parsers.or(lexers)); }
/** * A {@link Parser} that recognizes a sequence of tokens identified by {@code tokenNames}, as an * atomic step. */ public Parser<?> phrase(String... tokenNames) { Parser<?>[] wordParsers = new Parser<?>[tokenNames.length]; for (int i = 0; i < tokenNames.length; i++) { wordParsers[i] = token(tokenNames[i]); } String phrase = Strings.join(" ", tokenNames); return Parsers.sequence(wordParsers).atomic().retn(phrase).label(phrase); }
static Lexicon lexicon( Parser<String> wordScanner, Collection<String> keywordNames, StringCase stringCase, final Function<String, ?> defaultMap) { HashMap<String, Object> map = new HashMap<String, Object>(); for (String n : unique(stringCase, keywordNames.toArray(new String[keywordNames.size()]))) { Object value = Tokens.reserved(n); map.put(stringCase.toKey(n), value); } Function<String, Object> keywordMap = stringCase.byKey(map::get); return new Lexicon(keywordMap, wordScanner.map(Lexicon.fallback(keywordMap, defaultMap))); } }
/** A {@link Parser} that recognizes a token identified by any of {@code tokenNames}. */ public Parser<Token> token(String... tokenNames) { if (tokenNames.length == 0) return Parsers.never(); @SuppressWarnings("unchecked") Parser<Token>[] ps = new Parser[tokenNames.length]; for(int i = 0; i < tokenNames.length; i++) { ps[i] = Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenNames[i]))); } return Parsers.or(ps); }