@Override public StepDataInterface getStepData() { return new CsvInputData(); }
/** * Check to see if the buffer size is large enough given the data.endBuffer pointer.<br> * Resize the buffer if there is not enough room. * * @return false if everything is OK, true if there is a problem and we should stop. * @throws IOException * in case there is a I/O problem (read error) */ boolean resizeBufferIfNeeded() throws IOException { if ( endOfBuffer() ) { // Oops, we need to read more data... // Better resize this before we read other things in it... // resizeByteBufferArray(); // Also read another chunk of data, now that we have the space for it... // int n = readBufferFromFile(); // If we didn't manage to read something, we return true to indicate we're done // return n < 0; } return false; }
data.resizeBufferIfNeeded(); if ( data.newLineFound() ) { data.moveEndBufferPointer( false ); return data.newLineFound(); } else { data.moveEndBufferPointer( false );
if ( data.resizeBufferIfNeeded() ) { if ( data.delimiterFound() ) { delimiterFound = true; } else if ( ( !meta.isNewlinePossibleInFields() || outputIndex == data.fieldsMapping.size() - 1 ) && data.newLineFound() ) { data.moveEndBufferPointer(); if ( data.newLineFound() ) { } else if ( data.enclosureFound() && !ignoreEnclosuresInField ) { int enclosurePosition = data.getEndBuffer(); int fieldFirstBytePosition = data.getStartBuffer(); if ( fieldFirstBytePosition == enclosurePosition ) { boolean keepGoing; do { if ( data.moveEndBufferPointer() ) { enclosureFound = false; break; keepGoing = !data.enclosureFound(); if ( !keepGoing ) { if ( !data.endOfBuffer() && data.moveEndBufferPointer() ) { break; data.moveEndBufferPointer();
@Test public void testRemoveEscapedEnclosuresWithOneEscapedInMiddle() { CsvInputData csvInputData = new CsvInputData(); csvInputData.enclosure = "\"".getBytes(); String result = new String( csvInputData.removeEscapedEnclosures( "abcd \"\" defg".getBytes(), 1 ) ); assertEquals( "abcd \" defg", result ); }
data.closeFile(); while ( !data.newLineFound() ) { data.moveEndBufferPointer(); data.moveEndBufferPointer(); if ( data.newLineFound() ) { data.moveEndBufferPointer(); data.setStartBuffer( data.getEndBuffer() );
/** * Moves the endBuffer pointer by one.<br> * If there is not enough room in the buffer to go there, resize the byte buffer and read more data.<br> * if there is no more data to read and if the endBuffer pointer has reached the end of the byte buffer, we return * true.<br> * * @return true if we reached the end of the byte buffer. * @throws IOException * In case we get an error reading from the input file. */ boolean moveEndBufferPointer() throws IOException { return moveEndBufferPointer( true ); }
@Test public void testRemoveEscapedEnclosuresWithCharacterInTheMiddleOfThem() { CsvInputData csvInputData = new CsvInputData(); csvInputData.enclosure = "\"".getBytes(); String result = new String( csvInputData.removeEscapedEnclosures( "345\"\"1\"\"abc".getBytes(), 2 ) ); assertEquals( "345\"1\"abc", result ); } }
@Test public void testRemoveEscapedEnclosuresWithOneByItself() { CsvInputData csvInputData = new CsvInputData(); csvInputData.enclosure = "\"".getBytes(); String result = new String( csvInputData.removeEscapedEnclosures( "\"\"".getBytes(), 1 ) ); assertEquals( "\"", result ); }
/** * Initialize step info. */ @Before public void before() { meta = new CsvInputMeta(); meta.setDefault(); data = new CsvInputData(); data.outputRowMeta = new RowMeta(); }
@Test public void testRemoveEscapedEnclosuresWithTwoEscapedInMiddle() { CsvInputData csvInputData = new CsvInputData(); csvInputData.enclosure = "\"".getBytes(); String result = new String( csvInputData.removeEscapedEnclosures( "abcd \"\"\"\" defg".getBytes(), 2 ) ); assertEquals( "abcd \"\" defg", result ); }
private StepMetaDataCombi createBaseCombi( File sharedFile, boolean headerPresent, String delimiter ) { StepMetaDataCombi combi = new StepMetaDataCombi(); CsvInputData data = new CsvInputData(); CsvInputMeta meta = createMeta( sharedFile, createInputFileFields( "Field_000", "Field_001" ), headerPresent, delimiter ); CsvInput csvInput = createCsvInput(); csvInput.init( meta, data ); combi.step = csvInput; combi.data = data; combi.meta = meta; return combi; }
@Test public void testRemoveEscapedEnclosuresWithTwoByThemselves() { CsvInputData csvInputData = new CsvInputData(); csvInputData.enclosure = "\"".getBytes(); String result = new String( csvInputData.removeEscapedEnclosures( "\"\"\"\"".getBytes(), 2 ) ); assertEquals( "\"\"", result ); }
public void doTest( File file ) throws Exception { CsvInputData data = new CsvInputData(); CsvInputMeta meta = createMeta( file, createInputFileFields( "a", "b" ) ); List<Object[]> actual; try { csvInput.init( meta, data ); actual = TransTestingUtil.execute( csvInput, meta, data, 2, false ); } finally { csvInput.dispose( meta, data ); } List<Object[]> expected = Arrays.asList( new Object[] { "a", "b", 1L }, new Object[] { "a", null, 2L } ); TransTestingUtil.assertResult( expected, actual ); }
private void doTest( final String fileEncoding, final String stepEncoding, final String testData ) throws Exception { String testFilePath = createTestFile( fileEncoding, testData ).getAbsolutePath(); CsvInputMeta meta = createStepMeta( testFilePath, stepEncoding ); CsvInputData data = new CsvInputData(); CsvInput csvInput = new CsvInput( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); csvInput.init( meta, data ); csvInput.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { for ( int i = 0; i < rowMeta.size(); i++ ) { assertEquals( "Value", row[ i ] ); } } } ); boolean haveRowsToRead; do { haveRowsToRead = !csvInput.processRow( meta, data ); } while ( !haveRowsToRead ); csvInput.dispose( meta, data ); assertEquals( 2, csvInput.getLinesWritten() ); }
private void doTest( final String fileEncoding, final String stepEncoding, final String testData, final String delimiter, final boolean useHeader ) throws Exception { String testFilePath = createTestFile( fileEncoding, testData ).getAbsolutePath(); CsvInputMeta meta = createStepMeta( testFilePath, stepEncoding, delimiter, useHeader ); CsvInputData data = new CsvInputData(); CsvInput csvInput = new CsvInput( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); csvInput.init( meta, data ); csvInput.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { for ( int i = 0; i < rowMeta.size(); i++ ) { Assert.assertEquals( "Value", row[ i ] ); } } } ); boolean haveRowsToRead; do { haveRowsToRead = !csvInput.processRow( meta, data ); } while ( !haveRowsToRead ); csvInput.dispose( meta, data ); Assert.assertEquals( 2, csvInput.getLinesWritten() ); }
public void doTest( String content, String enclosure ) throws Exception { RowSet output = new QueueRowSet(); File tmp = createTestFile( "utf-8", content ); try { CsvInputMeta meta = createMeta( tmp, createInputFileFields( "f1", "f2" ), enclosure ); CsvInputData data = new CsvInputData(); csvInput.init( meta, data ); csvInput.addRowSetToOutputRowSets( output ); try { csvInput.processRow( meta, data ); } finally { csvInput.dispose( meta, data ); } } finally { tmp.delete(); } Object[] row = output.getRowImmediate(); assertNotNull( row ); assertEquals( "value1", row[ 0 ] ); assertEquals( "value2", row[ 1 ] ); assertNull( output.getRowImmediate() ); }
public void doTest( String content, String[] expected ) throws Exception { RowSet output = new QueueRowSet(); File tmp = createTestFile( encoding, content ); try { CsvInputMeta meta = createMeta( tmp, createInputFileFields( "f1", "f2", "f3" ) ); CsvInputData data = new CsvInputData(); csvInput.init( meta, data ); csvInput.addRowSetToOutputRowSets( output ); try { csvInput.processRow( meta, data ); } finally { csvInput.dispose( meta, data ); } } finally { tmp.delete(); } Object[] row = output.getRowImmediate(); assertNotNull( row ); assertEquals( expected[0], row[0] ); assertEquals( expected[1], row[1] ); assertEquals( expected[2], row[2] ); assertNull( output.getRowImmediate() ); }
private void doTest( String content ) throws Exception { RowSet output = new QueueRowSet(); File tmp = createTestFile( "utf-8", content ); try { CsvInputMeta meta = createMeta( tmp, createInputFileFields( "f1", "f2", "f3" ) ); CsvInputData data = new CsvInputData(); csvInput.init( meta, data ); csvInput.addRowSetToOutputRowSets( output ); try { csvInput.processRow( meta, data ); } finally { csvInput.dispose( meta, data ); } } finally { tmp.delete(); } Object[] row = output.getRowImmediate(); assertNotNull( row ); assertEquals( "value1", row[ 0 ] ); assertEquals( "value2", row[ 1 ] ); assertEquals( "value3", row[ 2 ] ); assertNull( output.getRowImmediate() ); }