public static List<String> parseList(String line) { List<String> result = new ArrayList<String>(); QuotedStringTokenizer tokenizer = new QuotedStringTokenizer(line, null, true, ",", false); while (tokenizer.hasMoreTokens()) { result.add(tokenizer.unquoteToken(tokenizer.nextToken().trim())); } return result; }
public static String toLine(Map<String, String> parts) { QuotedStringTokenizer tokenizer = new QuotedStringTokenizer("", true); StringBuilder result = new StringBuilder(); for (Map.Entry<String, String> entry : parts.entrySet()) { if (result.length()>0) result.append(", "); result.append(tokenizer.quoteToken(entry.getKey())); if (entry.getValue() != null) result.append("="+tokenizer.quoteToken(entry.getValue())); } return result.toString(); }
public QuotedStringTokenizer(String stringToTokenize, String quoteChars, boolean includeQuotes, String delimiters, boolean includeDelimiters) { delegate = new StringTokenizer(stringToTokenize==null ? "" : stringToTokenize, (delimiters==null ? DEFAULT_DELIMITERS : delimiters), true); this.quoteChars = quoteChars==null ? DEFAULT_QUOTE_CHARS() : quoteChars; this.includeQuotes = includeQuotes; this.delimiters = delimiters==null ? DEFAULT_DELIMITERS : delimiters; this.includeDelimiters = includeDelimiters; updateNextToken(); }
public List<String> buildList(String stringToTokenize) { return new QuotedStringTokenizer(stringToTokenize, quoteChars, includeQuotes, delimiterChars, includeDelimiters).remainderAsList(); }
public synchronized String nextToken() { if (peekedNextToken==null) throw new NoSuchElementException(); String lastToken = peekedNextToken; updateNextToken(); return includeQuotes ? lastToken : unquoteToken(lastToken); }
/** as {@link #unwrapJsonishListIfPossible(String)} but throwing errors * if something which looks like a string or set of brackets is not well-formed * (this does the work for that method) * @throws IllegalArgumentException if looks to have quoted list or surrounding brackets but they are not syntactically valid */ public static List<String> unwrapOptionallyQuotedJavaStringList(String input) { if (input==null) return null; MutableList<String> result = MutableList.of(); String i1 = input.trim(); boolean inBrackets = (i1.startsWith("[") && i1.endsWith("]")); if (inBrackets) i1 = i1.substring(1, i1.length()-1).trim(); QuotedStringTokenizer qst = new QuotedStringTokenizer(i1, "\"", true, ",", false); while (qst.hasMoreTokens()) { String t = qst.nextToken().trim(); if (isWrappedInDoubleQuotes(t)) result.add(unwrapJavaString(t)); else { if (inBrackets && (t.indexOf('[')>=0 || t.indexOf(']')>=0)) throw new IllegalArgumentException("Literal square brackets must be quoted, in element '"+t+"'"); result.add(t.trim()); } } return result; } }
public List<String> remainderAsList() { List<String> l = new ArrayList<String>(); while (hasMoreTokens()) l.add(nextToken()); return l; }
private void assertQuoteUnquoteFor(String unquoted) { String quoted = defaultTokenizer.quoteToken(unquoted); String reunquoted = defaultTokenizer.unquoteToken(quoted); //System.out.println("orig="+unquoted+" quoted="+quoted+" reunquoted="+reunquoted); assertEquals(reunquoted, unquoted); }
public QuotedStringTokenizer build(String stringToTokenize) { return new QuotedStringTokenizer(stringToTokenize, quoteChars, includeQuotes, delimiterChars, includeDelimiters); } public List<String> buildList(String stringToTokenize) {
private void assertUnquoteFor(String expected, String quoted) { String unquoted = defaultTokenizer.unquoteToken(quoted); //System.out.println("expected="+expected+" quoted="+quoted+" unquoted="+unquoted); assertEquals(unquoted, expected); } }
@Test public void testTokenizingBuilder() throws Exception { Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().buildList("foo bar")); Assert.assertEquals(Arrays.asList("foo,bar"), QuotedStringTokenizer.builder().buildList("foo,bar")); Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().delimiterChars(",").buildList("foo,bar")); Assert.assertEquals(Arrays.asList("foo", " bar"), QuotedStringTokenizer.builder().delimiterChars(",").buildList("foo, bar")); Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().addDelimiterChars(",").buildList("foo, bar")); }
public static boolean hasOpenQuote(String stringToCheck) { return hasOpenQuote(stringToCheck, DEFAULT_QUOTE_CHARS); }
protected synchronized void updateNextToken() { peekedNextToken = null; String token; do { if (!delegate.hasMoreTokens()) return; token = delegate.nextToken(); //skip delimeters } while (!includeDelimiters && token.matches("["+delimiters+"]+")); StringBuffer nextToken = new StringBuffer(token); pullUntilValid(nextToken); peekedNextToken = nextToken.toString(); }
private void testResultingTokens(String input, QuotedStringTokenizer tok, String... expectedTokens) { List<String> actual = new LinkedList<String>(); while (tok.hasMoreTokens()) actual.add(tok.nextToken()); assertEquals(actual, Arrays.asList(expectedTokens), "Wrong tokens returned."); }
public static int getWordCount(String phrase, boolean respectQuotes) { if (phrase==null) return 0; phrase = phrase.trim(); if (respectQuotes) return new QuotedStringTokenizer(phrase).remainderAsList().size(); else return Collections.list(new StringTokenizer(phrase)).size(); }
private void testResultingTokens(String input, String quoteChars, boolean includeQuotes, String delimiterChars, boolean includeDelimiters, String... expectedTokens) { QuotedStringTokenizer tok = new QuotedStringTokenizer(input, quoteChars, includeQuotes, delimiterChars, includeDelimiters); testResultingTokens(input, tok, expectedTokens); }
@Test public void testCommaInQuotes() throws Exception { List<String> l = QuotedStringTokenizer.builder().addDelimiterChars(",").buildList("location1,byon:(hosts=\"loc2,loc3\"),location4"); Assert.assertEquals(Arrays.asList("location1", "byon:(hosts=\"loc2,loc3\")", "location4"), l); }
private void pullUntilValid(StringBuffer nextToken) { while (hasOpenQuote(nextToken.toString(), quoteChars) && delegate.hasMoreTokens()) { //keep appending until the quote is ended or there are no more quotes nextToken.append(delegate.nextToken()); } }
Map<String,String> result = new LinkedHashMap<String,String>(); QuotedStringTokenizer tokenizer = new QuotedStringTokenizer(line, null, true, ",", false); while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken().trim(); String unquotedKey = tokenizer.unquoteToken(token); result.put(unquotedKey, null); String unquotedKey = tokenizer.unquoteToken(token.substring(0, index).trim()); String unquotedVal = tokenizer.unquoteToken(token.substring(index+1).trim()); result.put(unquotedKey, unquotedVal); String unquotedKey = tokenizer.unquoteToken(token.substring(0, index).trim()); result.put(unquotedKey, "");
public static String toLine(Collection<String> parts) { QuotedStringTokenizer tokenizer = new QuotedStringTokenizer("", false); StringBuilder result = new StringBuilder(); for (String part : parts) { result.append(tokenizer.quoteToken(part)+", "); } if (result.length() > 0) result.deleteCharAt(result.length()-1); return result.toString(); }