/** * Reset this tokenizer, giving it a new input string to parse. * In this manner you can re-use a tokenizer with the same settings * on multiple input lines. * * @param input the new string to tokenize, null sets no text to parse * @return this, to enable chaining */ public StrTokenizer reset(final String input) { reset(); if (input != null) { this.chars = input.toCharArray(); } else { this.chars = null; } return this; }
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final char[] input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
/** * Reset this tokenizer, giving it a new input string to parse. * In this manner you can re-use a tokenizer with the same settings * on multiple input lines. * * @param input the new character array to tokenize, not cloned, null sets no text to parse * @return this, to enable chaining */ public StrTokenizer reset(final char[] input) { reset(); this.chars = ArrayUtils.clone(input); return this; }
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final String input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
/** * Gets a new tokenizer instance which parses Comma Separated Value strings * initializing it with the given input. The default for CSV processing * will be trim whitespace from both ends (which can be overridden with * the setTrimmer method). * * @param input the text to parse * @return a new tokenizer instance which parses Comma Separated Value strings */ public static StrTokenizer getCSVInstance(final String input) { final StrTokenizer tok = getCSVClone(); tok.reset(input); return tok; }
/** * Gets a new tokenizer instance which parses Comma Separated Value strings * initializing it with the given input. The default for CSV processing * will be trim whitespace from both ends (which can be overridden with * the setTrimmer method). * * @param input the text to parse * @return a new tokenizer instance which parses Comma Separated Value strings */ public static StrTokenizer getCSVInstance(final char[] input) { final StrTokenizer tok = getCSVClone(); tok.reset(input); return tok; }
/** * Creates a new instance of this Tokenizer. The new instance is reset so that * it will be at the start of the token list. * * @return a new instance of this Tokenizer which has been reset. * @throws CloneNotSupportedException if there is a problem cloning */ Object cloneReset() throws CloneNotSupportedException { // this method exists to enable 100% test coverage final StrTokenizer cloned = (StrTokenizer) super.clone(); if (cloned.chars != null) { cloned.chars = cloned.chars.clone(); } cloned.reset(); return cloned; }
@Test public void testCloneNull() { final StrTokenizer tokenizer = new StrTokenizer((char[]) null); // Start sanity check assertNull(tokenizer.nextToken()); tokenizer.reset(); assertNull(tokenizer.nextToken()); // End sanity check final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone(); tokenizer.reset(); assertNull(tokenizer.nextToken()); assertNull(clonedTokenizer.nextToken()); }
@Test public void testCloneReset() { final char[] input = new char[]{'a'}; final StrTokenizer tokenizer = new StrTokenizer(input); // Start sanity check assertEquals("a", tokenizer.nextToken()); tokenizer.reset(input); assertEquals("a", tokenizer.nextToken()); // End sanity check final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone(); input[0] = 'b'; tokenizer.reset(input); assertEquals("b", tokenizer.nextToken()); assertEquals("a", clonedTokenizer.nextToken()); }
@Test public void testChaining() { final StrTokenizer tok = new StrTokenizer(); assertEquals(tok, tok.reset()); assertEquals(tok, tok.reset("")); assertEquals(tok, tok.reset(new char[0])); assertEquals(tok, tok.setDelimiterChar(' ')); assertEquals(tok, tok.setDelimiterString(" ")); assertEquals(tok, tok.setDelimiterMatcher(null)); assertEquals(tok, tok.setQuoteChar(' ')); assertEquals(tok, tok.setQuoteMatcher(null)); assertEquals(tok, tok.setIgnoredChar(' ')); assertEquals(tok, tok.setIgnoredMatcher(null)); assertEquals(tok, tok.setTrimmerMatcher(null)); assertEquals(tok, tok.setEmptyTokenAsNull(false)); assertEquals(tok, tok.setIgnoreEmptyTokens(false)); }
@Test public void testReset() { final StrTokenizer tok = new StrTokenizer("a b c"); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertEquals("c", tok.next()); assertFalse(tok.hasNext()); tok.reset(); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertEquals("c", tok.next()); assertFalse(tok.hasNext()); }
assertEquals("b", t.next()); t.reset(); final String[] tokens3 = t.getTokenArray(); assertEquals(4, tokens3.length);
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final char[] input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
StrTokenizer tokenizer = StrTokenizer.getTSVInstance(); while ( (line = ...) != null) { tokenizer.reset(line); String tokArray[] = tokenizer.getTokenArray(); }
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final String input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final String input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
// get a csv instance (which is cloned, so we can customize it) StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); // Set delimiter char tokenizer.setDelimiterChar('|'); Scanner scanner = new Scanner(new File("file.psv")); while (scanner.hasNextLine()) { // set the input on the tokenizer tokenizer.reset(scanner.nextLine()); // get the tokens String toks[] = tokenizer.getTokenArray(); }
private List<Column> initColumns(final StrTokenizer st, final String headerLine) { st.reset(headerLine); String[] headers = st.getTokenArray(); List<Column> columns = newArrayListWithCapacity(headers.length); for (String header : headers) { columns.add(new Column(header)); } return columns; }