protected Tap getSinkTapFor( String sinkName, Stereotype<Protocol, Format> stereotype ) { List<Resource<Protocol, Format, SinkMode>> resources = getSinkResources( sinkName ); Tap[] taps = createTapFor( stereotype, resources, Role.sink ); if( taps == null ) return null; if( taps.length == 1 ) return taps[ 0 ]; return new MultiSinkTap( taps ); }
@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Test public void testMultiSinkTap() throws IOException { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileJoined ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "number", "lower", "upper" ), "\t" ) ); Tap lhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "lower" ), getOutputPath( "multisink/lhs" ), SinkMode.REPLACE ); Tap rhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "upper" ), getOutputPath( "multisink/rhs" ), SinkMode.REPLACE ); Tap sink = new MultiSinkTap( lhsSink, rhsSink ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openTapForRead( lhsSink ), 5 ); validateLength( flow.openTapForRead( rhsSink ), 5 ); }
@Test public void testMultiSinkTap() throws IOException { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileJoined ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "number", "lower", "upper" ), "\t" ) ); Tap lhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "lower" ), getOutputPath( "multisink/lhs" ), SinkMode.REPLACE ); Tap rhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "upper" ), getOutputPath( "multisink/rhs" ), SinkMode.REPLACE ); Tap sink = new MultiSinkTap( lhsSink, rhsSink ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openTapForRead( lhsSink ), 5 ); validateLength( flow.openTapForRead( rhsSink ), 5 ); }