@Test public void testTokenizing() throws Exception { testResultingTokens("foo,bar,baz", "\"", false, ",", false, "foo", "bar", "baz"); testResultingTokens("\"foo,bar\",baz", "\"", false, ",", false, "foo,bar", "baz"); testResultingTokens("\"foo,,bar\",baz", "\"", false, ",", false, "foo,,bar", "baz"); // Have seen "the operator ""foo"" is not recognised" entries in BAML CSV files. testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ",", false, "foo bar baz"); testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ",", false, "foo bar baz"); // FIXME: would like to return empty tokens when we encounter adjacent delimiters, but need // to work around brain-dead java.util.StringTokenizer to do this. // testResultingTokens("foo,,baz", "\"", false, ",", false, "foo", "", "baz"); }
private void testResultingTokens(String input, String quoteChars, boolean includeQuotes, String delimiterChars, boolean includeDelimiters, String... expectedTokens) { QuotedStringTokenizer tok = new QuotedStringTokenizer(input, quoteChars, includeQuotes, delimiterChars, includeDelimiters); testResultingTokens(input, tok, expectedTokens); }