protected SinkStage createSinkStage( Tap element ) { return new SinkStage( flowProcess, element ); }
@Override public void receive( Duct previous, int ordinal, TupleEntry tupleEntry ) { try { timedAdd( StepCounters.Write_Duration, tupleEntry ); flowProcess.increment( StepCounters.Tuples_Written, 1 ); flowProcess.increment( SliceCounters.Tuples_Written, 1 ); } catch( OutOfMemoryError error ) { handleReThrowableException( "out of memory, try increasing task memory allocation", error ); } catch( CascadingException exception ) { handleException( exception, tupleEntry ); } catch( Throwable throwable ) { handleException( new DuctException( "internal error: " + tupleEntry.getTuple().print(), throwable ), tupleEntry ); } }
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalOutput.getClass().getSimpleName(), getSink() ); logicalOutput.start(); collector = new OldOutputCollector( logicalOutput ); super.prepare(); }
@Override public void cleanup() { try { super.cleanup(); } finally { try { if( logicalOutput.isCommitRequired() ) commit( logicalOutput ); } catch( Exception exception ) { LOG.warn( "exception on output close", exception ); } } }
@Override public void prepare() { try { // todo: pass the resolved fields down collector = sink.openForWrite( flowProcess, getOutput() ); if( sink.getSinkFields().isAll() ) { Fields fields = getIncomingScopes().get( 0 ).getIncomingTapFields(); collector.setFields( fields ); } } catch( IOException exception ) { throw new DuctException( "failed opening sink", exception ); } }
/** * Returns a Set as a given tap may be bound to multiple branches * * @param duct * @return */ private Set<String> getTapBranchNamesFor( Duct duct ) { if( ( (Tap) ( (ElementDuct) duct ).getFlowElement() ).isTemporary() ) return Collections.emptySet(); if( duct instanceof SourceStage ) return node.getSourceTapNames( (Tap) ( (SourceStage) duct ).getFlowElement() ); else if( duct instanceof SinkStage ) return node.getSinkTapNames( (Tap) ( (SinkStage) duct ).getFlowElement() ); else throw new IllegalStateException( "duct does not wrap a Tap: " + duct.getClass().getCanonicalName() ); }
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalOutput.getClass().getSimpleName(), getSink() ); logicalOutput.start(); collector = new OldOutputCollector( logicalOutput ); super.prepare(); }
@Override public void cleanup() { try { super.cleanup(); } finally { try { if( logicalOutput.isCommitRequired() ) commit( logicalOutput ); } catch( Exception exception ) { LOG.warn( "exception on output close", exception ); } } }
@Override protected SinkStage createSinkStage( Tap element ) { return new SinkStage( tapFlowProcess( element ), element ); }