@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Test public void testQuotedText() throws IOException { runQuotedText( "normchar", testDelimited, ",", false ); }
@Test public void testHeaderAll() throws IOException { Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, testDelimited, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerall" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 11, 5 ); }
@Test public void testFieldCoercion() throws IOException { // 75.185.76.245 - - [01/Sep/2007:00:01:03 +0000] "POST /mt-tb.cgi/235 HTTP/1.1" 403 174 "-" "Opera/9.10 (Windows NT 5.1; U; ru)" "-" DateType dateType = new DateType( TestConstants.APACHE_DATE_FORMAT, TimeZone.getDefault(), Locale.US ); Type[] types = new Type[]{ String.class, // ip String.class, // - String.class, // - dateType, // date String.class, // request int.class, // code long.class, // bytes String.class, // - String.class, // agent String.class // - }; Fields fields = new Fields( "ip", "client", "user", "date", "request", "code", "bytes", "referrer", "agent", "na" ); fields = fields.applyTypes( types ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, inputFileApacheClean, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, getOutputPath( getTestName() ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); pipe = new Each( pipe, new Fields( "date" ), AssertionLevel.STRICT, new AssertExpression( "date instanceof Long", Object.class ) ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 9, 10 ); } }
Tap input = getPlatform().getDelimitedFile( fields, false, delimiter, "\"", types, inputData, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, false, delimiter, "\"", types, getOutputPath( "quoted/" + path + "" + useAll ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); validateLength( flow, results.length, 5 ); assertEquals( tuples[ count++ ], tuple ); assertEquals( tuples[ count++ ], tuple );
@Test public void testStrict() throws IOException { Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, false, false, ",", "\"", null, testDelimitedExtraField, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, false, false, ",", "\"", null, getOutputPath( "strict" + ComparePlatformsTest.NONDETERMINISTIC ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); try { flow.complete(); fail( "should fail on too many fields" ); } catch( Exception exception ) { // ignore } }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }
@Test public void testHeader() throws IOException { Class[] types = new Class[]{String.class, String.class, String.class, String.class, long.class}; Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", types, testDelimited, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", types, getOutputPath( "header" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 11, 5 ); }
@Test public void testFieldCoercion() throws IOException { // 75.185.76.245 - - [01/Sep/2007:00:01:03 +0000] "POST /mt-tb.cgi/235 HTTP/1.1" 403 174 "-" "Opera/9.10 (Windows NT 5.1; U; ru)" "-" DateType dateType = new DateType( TestConstants.APACHE_DATE_FORMAT, TimeZone.getDefault(), Locale.US ); Type[] types = new Type[]{ String.class, // ip String.class, // - String.class, // - dateType, // date String.class, // request int.class, // code long.class, // bytes String.class, // - String.class, // agent String.class // - }; Fields fields = new Fields( "ip", "client", "user", "date", "request", "code", "bytes", "referrer", "agent", "na" ); fields = fields.applyTypes( types ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, inputFileApacheClean, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, getOutputPath( getTestName() ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); pipe = new Each( pipe, new Fields( "date" ), AssertionLevel.STRICT, new AssertExpression( "date instanceof Long", Object.class ) ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 9, 10 ); } }
Tap input = getPlatform().getDelimitedFile( fields, false, delimiter, "\"", types, inputData, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, false, delimiter, "\"", types, getOutputPath( "quoted/" + path + "" + useAll ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); validateLength( flow, results.length, 5 ); assertEquals( tuples[ count++ ], tuple ); assertEquals( tuples[ count++ ], tuple );
@Test public void testStrict() throws IOException { Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, false, false, ",", "\"", null, testDelimitedExtraField, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, false, false, ",", "\"", null, getOutputPath( "strict" + ComparePlatformsTest.NONDETERMINISTIC ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); try { flow.complete(); fail( "should fail on too many fields" ); } catch( Exception exception ) { // ignore } }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }
@Test public void testHeader() throws IOException { Class[] types = new Class[]{String.class, String.class, String.class, String.class, long.class}; Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", types, testDelimited, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", types, getOutputPath( "header" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 11, 5 ); }
@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Test public void testQuotedTextSpecCharAll() throws IOException { runQuotedText( "specchar", testDelimitedSpecialCharData, "|", true ); }
@Test public void testHeaderAll() throws IOException { Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); Tap input = getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, testDelimited, SinkMode.KEEP ); Tap output = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerall" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); validateLength( flow, 11, 5 ); }
@Test public void testQuotedTextAll() throws IOException { runQuotedText( "normchar", testDelimited, ",", true ); }
@Test public void testQuotedTextSpecChar() throws IOException { runQuotedText( "specchar", testDelimitedSpecialCharData, "|", false ); }
@Test public void testQuotedTextSpecChar() throws IOException { runQuotedText( "specchar", testDelimitedSpecialCharData, "|", false ); }
@Test public void testQuotedTextSpecCharAll() throws IOException { runQuotedText( "specchar", testDelimitedSpecialCharData, "|", true ); }