private CsvInputMeta createMeta( File file, TextFileInputField[] fields ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setDelimiter( delimiter ); meta.setEncoding( encoding ); meta.setEnclosure( enclosure ); meta.setBufferSize( "1024" ); meta.setInputFields( fields ); meta.setHeaderPresent( false ); return meta; } }
private CsvInputMeta createMeta( File file, TextFileInputField[] fields ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setDelimiter( "," ); meta.setEncoding( "utf-8" ); meta.setBufferSize( "1024" ); meta.setInputFields( fields ); meta.setHeaderPresent( false ); meta.setRowNumField( "rownum" ); return meta; } }
private CsvInputMeta createMeta( File file, TextFileInputField[] fields ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setDelimiter( "delimiter" ); meta.setEncoding( "utf-8" ); meta.setEnclosure( "\"" ); meta.setBufferSize( "1024" ); meta.setInputFields( fields ); meta.setHeaderPresent( false ); return meta; } }
private CsvInputMeta createMeta( File file, TextFileInputField[] fields, String enclosure ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setDelimiter( ";" ); meta.setEncoding( "utf-8" ); meta.setEnclosure( enclosure ); meta.setBufferSize( "1024" ); meta.setInputFields( fields ); meta.setHeaderPresent( false ); return meta; } }
private CsvInputMeta createMeta( File file, TextFileInputField[] fields, boolean headerPresent, String delimiter ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setDelimiter( delimiter ); meta.setEncoding( "utf-8" ); meta.setEnclosure( "\"" ); meta.setBufferSize( "1024" ); if ( !headerPresent ) { meta.setInputFields( fields ); } meta.setHeaderPresent( headerPresent ); meta.setRunningInParallel( true ); return meta; }
private CsvInputMeta createStepMeta( final String testFilePath, final String encoding, final String delimiter, final boolean useHeader ) { final CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( testFilePath ); meta.setDelimiter( delimiter ); meta.setEncoding( encoding ); meta.setEnclosure( "\"" ); meta.setBufferSize( "50000" ); meta.setInputFields( getInputFileFields() ); meta.setHeaderPresent( useHeader ); return meta; }
@Test public void testMultiCharDelimOptions() throws Exception { meta.setDelimiter( "|||" ); init( "multi_delim.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, { "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } ); }
private CsvInputMeta createStepMeta( final String testFilePath, final String encoding ) { final CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( testFilePath ); meta.setDelimiter( "\t" ); meta.setEncoding( encoding ); meta.setEnclosure( "\"" ); meta.setBufferSize( "50000" ); meta.setInputFields( getInputFileFields() ); meta.setHeaderPresent( true ); return meta; }
@Test public void testSemicolonOptions() throws Exception { meta.setDelimiter( ";" ); init( "semicolon.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, { "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } ); }
inputMeta.setDelimiter( wDelimiter.getText() ); inputMeta.setEnclosure( wEnclosure.getText() ); inputMeta.setBufferSize( wBufferSize.getText() );
@Test public void testClone() { final CsvInputMeta original = new CsvInputMeta(); original.setDelimiter( ";" ); original.setEnclosure( "'" ); final TextFileInputField[] originalFields = new TextFileInputField[ 1 ]; final TextFileInputField originalField = new TextFileInputField(); originalField.setName( "field" ); originalFields[ 0 ] = originalField; original.setInputFields( originalFields ); final CsvInputMeta clone = (CsvInputMeta) original.clone(); // verify that the clone and its input fields are "equal" to the originals, but not the same objects Assert.assertNotSame( original, clone ); Assert.assertEquals( original.getDelimiter(), clone.getDelimiter() ); Assert.assertEquals( original.getEnclosure(), clone.getEnclosure() ); Assert.assertNotSame( original.getInputFields(), clone.getInputFields() ); Assert.assertNotSame( original.getInputFields()[ 0 ], clone.getInputFields()[ 0 ] ); Assert.assertEquals( original.getInputFields()[ 0 ].getName(), clone.getInputFields()[ 0 ].getName() ); } }
protected void doInputWiring( Query query, TransMeta transMeta ) { // // CSV FILE LOCATION AND FIELDS // InlineEtlPhysicalModel physicalModel = (InlineEtlPhysicalModel) query.getLogicalModel().getPhysicalModel(); CsvInputMeta csvinput = (CsvInputMeta) getStepMeta( transMeta, "CSV file input" ).getStepMetaInterface(); //$NON-NLS-1$ // the file name might need to be translated to the correct location here if ( csvFileLoc != null ) { csvinput.setFilename( csvFileLoc + physicalModel.getFileLocation() ); } else { csvinput.setFilename( physicalModel.getFileLocation() ); } csvinput.setDelimiter( physicalModel.getDelimiter() ); csvinput.setEnclosure( physicalModel.getEnclosure() ); csvinput.setHeaderPresent( physicalModel.getHeaderPresent() ); // update fields LogicalTable table = query.getLogicalModel().getLogicalTables().get( 0 ); csvinput.allocate( table.getLogicalColumns().size() ); for ( int i = 0; i < csvinput.getInputFields().length; i++ ) { // Update csv input LogicalColumn col = table.getLogicalColumns().get( i ); csvinput.getInputFields()[i] = new TextFileInputField(); String fieldName = (String) col.getProperty( InlineEtlPhysicalColumn.FIELD_NAME ); if ( logger.isDebugEnabled() ) { logger.debug( "FROM CSV: " + fieldName ); //$NON-NLS-1$ } csvinput.getInputFields()[i].setName( fieldName ); csvinput.getInputFields()[i].setType( convertType( col.getDataType() ) ); } }
csvInputMeta.setDelimiter( fileInfo.getDelimiter() ); csvInputMeta.setEnclosure( fileInfo.getEnclosure() ); csvInputMeta.setEncoding( fileInfo.getEncoding() );