private int createAndRunOneStep( File sharedFile, int stepNr, int totalNumberOfSteps, boolean headersPresent, String delimiter ) throws Exception { StepMetaDataCombi combiStep1 = createBaseCombi( sharedFile, headersPresent, delimiter ); configureData( (CsvInputData) combiStep1.data, stepNr, totalNumberOfSteps ); return processRows( combiStep1 ); }
@Test public void twoByteNewLineIndicator_NoNewLineAtTheEnd_2Threads() throws Exception { final String fileContent = "a;1\r\n" + "b;2"; final int totalNumberOfSteps = 2; File sharedFile = createTestFile( "UTF-8", fileContent ); int t1 = createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ); int t2 = createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ); assertEquals( 2, t1 + t2 ); }
private StepMetaDataCombi createBaseCombi( File sharedFile, boolean headerPresent, String delimiter ) { StepMetaDataCombi combi = new StepMetaDataCombi(); CsvInputData data = new CsvInputData(); CsvInputMeta meta = createMeta( sharedFile, createInputFileFields( "Field_000", "Field_001" ), headerPresent, delimiter ); CsvInput csvInput = createCsvInput(); csvInput.init( meta, data ); combi.step = csvInput; combi.data = data; combi.meta = meta; return combi; }
private int createAndRunOneStep( File sharedFile, int stepNr, int totalNumberOfSteps ) throws Exception { return createAndRunOneStep( sharedFile, stepNr, totalNumberOfSteps, false, ";" ); }
@Test public void PDI_16589_twoByteNewLineIndicator_withHeaders_NewLineAtTheEnd_4Threads() throws Exception { final int totalNumberOfSteps = 4; final String fileContent = "Col1,Col2\r\n" + "a,1\r\n" + "b,2\r\n" + "c,3\r\n" + "d,4\r\n" + "e,5\r\n" + "f,6\r\n" + "g,7\r\n" + "h,8\r\n" + "i,9\r\n" + "jk,10\r\n" + "lm,11\r\n"; File sharedFile = createTestFile( "UTF-8", fileContent ); int t1 = createAndRunOneStep( sharedFile, 0, totalNumberOfSteps, true, "," ); int t2 = createAndRunOneStep( sharedFile, 1, totalNumberOfSteps, true, "," ); int t3 = createAndRunOneStep( sharedFile, 2, totalNumberOfSteps, true, "," ); int t4 = createAndRunOneStep( sharedFile, 3, totalNumberOfSteps, true, "," ); assertEquals( 11, t1 + t2 + t3 + t4 ); }
@Test public void twoByteNewLineIndicator_NewLineAtTheEnd_2Threads() throws Exception { final String fileContent = "a;1\r\n" + "b;2\r\n"; final int totalNumberOfSteps = 2; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 1 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 1 ); }
@Test public void oneByteNewLineIndicator_NewLineAtTheEnd_2Threads() throws Exception { final int totalNumberOfSteps = 2; final String fileContent = "a;1\r" + "b;2\r"; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 1 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 1 ); }
@Test public void oneByteNewLineIndicator_NoNewLineAtTheEnd_2Threads() throws Exception { final int totalNumberOfSteps = 2; final String fileContent = "a;1\r" + "b;2\r" + "c;3"; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 2 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 1 ); }
@Test public void mixedBytesNewLineIndicator_NewLineAtTheEnd_2Threads() throws Exception { final String fileContent = "abcd;1\r\n" + "b;2\r" + "d;3\r"; final int totalNumberOfSteps = 2; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 1 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 2 ); }
@Test public void twoByteNewLineIndicator_NewLineAtTheEnd_3Threads() throws Exception { final String fileContent = "a;1\r\n" + "b;2\r\n" // thread 1 should read until this line + "c;3\r\n" + "d;4\r\n" // thread 2 should read until this line + "e;5\r\n" + "f;6\r\n"; // thread 3 should read until this line final int totalNumberOfSteps = 3; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 2 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 2 ); assertEquals( createAndRunOneStep( sharedFile, 2, totalNumberOfSteps ), 2 ); }
@Test public void PDI_15162_mixedByteNewLineIndicator_NewLineAtTheEnd_2Threads() throws Exception { final int totalNumberOfSteps = 2; final String fileContent = "ab;111\r\n" + "bc;222\r\n" + "cd;333\r\n" + "de;444\r\n" + "ef;555\r" + "fg;666\r\n" + "gh;777\r\n" + "hi;888\r\n" + "ij;999\r" + "jk;000\r"; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 5 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 5 ); }
/** * Here files content is 16 bytes summary, where 8 of this bytes is the first line, 5 is the second one, 3 is the * last. * <p> * As we are running this with 2 threads, we expect: 1st thread to read 1st line 2nd thread to read 2nd and 3d line. */ @Test public void mixedBytesNewLineIndicator_NoNewLineAtTheEnd_2Threads() throws Exception { final String fileContent = "abcd;1\r\n" + "b;2\r\n" + "d;3"; final int totalNumberOfSteps = 2; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 1 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 2 ); }
@Test public void PDI_15162_mixedByteNewLineIndicator_NoNewLineAtTheEnd_2Threads() throws Exception { final int totalNumberOfSteps = 2; final String fileContent = "ab;111\r\n" + "bc;222\r\n" + "cd;333\r\n" + "de;444\r\n" + "ef;555\r" + "fg;666\r\n" + "gh;777\r\n" + "hi;888\r\n" + "ij;999\r" + "jk;000"; File sharedFile = createTestFile( "UTF-8", fileContent ); assertEquals( createAndRunOneStep( sharedFile, 0, totalNumberOfSteps ), 5 ); assertEquals( createAndRunOneStep( sharedFile, 1, totalNumberOfSteps ), 5 ); }