@Override public void sinkConfInit( FlowProcess<? extends Config> flowProcess, Config conf ) { bridge( flowProcess, conf ); }
@Override public long getNumChildTaps() { return getTaps().length; }
protected Tap getSinkTapFor( String sinkName, Stereotype<Protocol, Format> stereotype ) { List<Resource<Protocol, Format, SinkMode>> resources = getSinkResources( sinkName ); Tap[] taps = createTapFor( stereotype, resources, Role.sink ); if( taps == null ) return null; if( taps.length == 1 ) return taps[ 0 ]; return new MultiSinkTap( taps ); }
@Override public Scheme getScheme() { if( super.getScheme() != null ) return super.getScheme(); Set<Fields> fields = new HashSet<Fields>(); for( Tap child : getTaps() ) fields.add( child.getSinkFields() ); // if all schemes have the same sink fields, the just use the scheme if( fields.size() == 1 ) { setScheme( getTaps()[ 0 ].getScheme() ); return super.getScheme(); } Fields allFields = Fields.merge( fields.toArray( new Fields[ fields.size() ] ) ); setScheme( new NullScheme( allFields, allFields ) ); return super.getScheme(); }
@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Test public void testHeaderFieldsAll() throws IOException { Tap input = getPlatform().getDelimitedFile( Fields.UNKNOWN, true, true, ",", "\"", null, testDelimitedHeader, SinkMode.KEEP ); Tap output1 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall1" ), SinkMode.REPLACE ); Tap output2 = getPlatform().getDelimitedFile( Fields.ALL, true, true, ",", "\"", null, getOutputPath( "headerfieldsall2" ), SinkMode.REPLACE ); Tap output = new MultiSinkTap( output1, output2 ); Pipe pipe = new Pipe( "pipe" ); Flow flow = getPlatform().getFlowConnector().connect( input, output, pipe ); flow.complete(); Fields fields = new Fields( "first", "second", "third", "fourth", "fifth" ); TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getDelimitedFile( fields, true, true, ",", "\"", null, output1.getIdentifier(), SinkMode.REPLACE ) ); validateLength( iterator, 13, 5 ); assertHeaders( output1, flow ); assertHeaders( output2, flow ); }
@Override public Iterator<Child> getChildTaps() { return Arrays.asList( getTaps() ).iterator(); }
@Test public void testMultiSinkTap() throws IOException { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileJoined ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "number", "lower", "upper" ), "\t" ) ); Tap lhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "lower" ), getOutputPath( "multisink/lhs" ), SinkMode.REPLACE ); Tap rhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "upper" ), getOutputPath( "multisink/rhs" ), SinkMode.REPLACE ); Tap sink = new MultiSinkTap( lhsSink, rhsSink ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openTapForRead( lhsSink ), 5 ); validateLength( flow.openTapForRead( rhsSink ), 5 ); }
@Test public void testMultiSinkTap() throws IOException { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileJoined ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "number", "lower", "upper" ), "\t" ) ); Tap lhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "lower" ), getOutputPath( "multisink/lhs" ), SinkMode.REPLACE ); Tap rhsSink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "number", "upper" ), getOutputPath( "multisink/rhs" ), SinkMode.REPLACE ); Tap sink = new MultiSinkTap( lhsSink, rhsSink ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openTapForRead( lhsSink ), 5 ); validateLength( flow.openTapForRead( rhsSink ), 5 ); }
@Override public boolean createResource( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.createResource( conf ) ) return false; } return true; }
@Override public void presentSinkFields( FlowProcess<? extends Config> flowProcess, Fields fields ) { for( Tap child : getTaps() ) child.presentSinkFields( flowProcess, fields ); }
@Override public boolean rollbackResource( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.rollbackResource( conf ) ) return false; } return true; }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Config> flowProcess, Output output ) throws IOException { return new MultiSinkCollector( flowProcess, getTaps() ); }
@Override public boolean deleteResource( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.deleteResource( conf ) ) return false; } return true; }
@Override public boolean resourceExists( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.resourceExists( conf ) ) return false; } return true; }
@Override public boolean commitResource( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.commitResource( conf ) ) return false; } return true; }
private void bridge( FlowProcess flowProcess, Object conf ) { childConfigs = new ArrayList<>(); for( int i = 0; i < getTaps().length; i++ ) { Tap tap = getTaps()[ i ]; Object newConfig = flowProcess.copyConfig( conf ); tap.sinkConfInit( flowProcess, newConfig ); childConfigs.add( flowProcess.diffConfigIntoMap( conf, newConfig ) ); } }