/** * Yields the tokens resulting from the splitting of the supplied * <code>line</code>. * * @param line the line to be tokenized (can be <code>null</code>) * * @return the resulting tokens */ @Override public FieldSet tokenize(String line) { if (line == null) { line = ""; } List<String> tokens = new ArrayList<>(doTokenize(line)); // if names are set and strict flag is false if ( ( names.length != 0 ) && ( ! strict ) ) { adjustTokenCountIfNecessary( tokens ); } String[] values = tokens.toArray(new String[tokens.size()]); if (names.length == 0) { return fieldSetFactory.create(values); } else if (values.length != names.length) { throw new IncorrectTokenCountException(names.length, values.length, line); } return fieldSetFactory.create(values, names); }
/** * Columns names are considered to be specified if they are not <code>null</code> or empty. */ public void testHasNames() { AbstractLineTokenizer tokenizer = new AbstractLineTokenizer() { @Override protected List<String> doTokenize(String line) { return null; } }; assertFalse(tokenizer.hasNames()); tokenizer.setNames((String) null); assertFalse(tokenizer.hasNames()); tokenizer.setNames(new ArrayList<String>().toArray(new String[0])); assertFalse(tokenizer.hasNames()); tokenizer.setNames("name1", "name2"); assertTrue(tokenizer.hasNames()); }
@Test(expected=IllegalArgumentException.class) public void testDelimitedLineTokenizerNullDelimiter() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(null); tokenizer.tokenize("a b c"); }
/** * Yields the tokens resulting from the splitting of the supplied * <code>line</code>. * * @param line the line to be tokenized (can be <code>null</code>) * * @return the resulting tokens */ @Override public FieldSet tokenize(String line) { if (line == null) { line = ""; } List<String> tokens = new ArrayList<String>(doTokenize(line)); // if names are set and strict flag is false if ( ( names.length != 0 ) && ( ! strict ) ) { adjustTokenCountIfNecessary( tokens ); } String[] values = tokens.toArray(new String[tokens.size()]); if (names.length == 0) { return fieldSetFactory.create(values); } else if (values.length != names.length) { throw new IncorrectTokenCountException(names.length, values.length, line); } return fieldSetFactory.create(values, names); }
@Test public void testDelimitedLineTokenizerChar() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" "); FieldSet line = tokenizer.tokenize("a b c"); assertEquals(3, line.getFieldCount()); }
@Test public void testDelimitedLineTokenizerNewlineToken() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n"); FieldSet line = tokenizer.tokenize("a b\n c"); assertEquals(2, line.getFieldCount()); assertEquals("a b", line.readString(0)); assertEquals("c", line.readString(1)); }
@Test public void testDelimitedLineTokenizerWrappedToken() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\nrap"); FieldSet line = tokenizer.tokenize("a b\nrap c"); assertEquals(2, line.getFieldCount()); assertEquals("a b", line.readString(0)); assertEquals("c", line.readString(1)); }
@Test public void testDelimitedLineTokenizerString() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" b "); FieldSet line = tokenizer.tokenize("a b c"); assertEquals(2, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("c", line.readString(1)); }
@Test public void testDelimitedLineTokenizerStringsOverlapWithoutSeparation() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize("a | | b"); assertEquals(2, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("| b", line.readString(1)); }
@Test public void testDelimitedLineTokenizerStringsOverlap() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize("a | | | b"); assertEquals(3, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("|", line.readString(1)); assertEquals("b", line.readString(2)); }
@Test public void testDelimitedLineTokenizerStringBeginningOfLine() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize(" | a | b"); assertEquals(3, line.getFieldCount()); assertEquals("", line.readString(0)); assertEquals("a", line.readString(1)); assertEquals("b", line.readString(2)); }
@Test public void testDelimitedLineTokenizerStringEndOfLine() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize("a | b | "); assertEquals(3, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("b", line.readString(1)); assertEquals("", line.readString(2)); }