public TezMergeGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); Set<LogicalInput> inputs = new HashSet<>( logicalInputs.getValues() ); if( inputs.size() != 1 ) throw new IllegalArgumentException( "only supports a single input" ); this.logicalInputs = logicalInputs; }
public TezMergeGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); Set<LogicalInput> inputs = new HashSet<>( logicalInputs.getValues() ); if( inputs.size() != 1 ) throw new IllegalArgumentException( "only supports a single input" ); this.logicalInputs = logicalInputs; }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutput != null ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } super.prepare(); }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutput != null ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } super.prepare(); }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutputs != null ) { for( LogicalOutput logicalOutput : logicalOutputs ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } if( role != IORole.source ) collector = new MeasuredOutputCollector( flowProcess, SliceCounters.Write_Duration, createOutputCollector() ); super.prepare(); }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutputs != null ) { for( LogicalOutput logicalOutput : logicalOutputs ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } if( role != IORole.source ) collector = new MeasuredOutputCollector( flowProcess, SliceCounters.Write_Duration, createOutputCollector() ); super.prepare(); }
protected Throwable map() throws Exception { Throwable localThrowable = null; try { start( this ); // if multiple ordinals, an input could be duplicated if sourcing multiple paths LogicalInput logicalInput = Util.getFirst( logicalInputs.getValues() ); KeyValueReader reader = (KeyValueReader) logicalInput.getReader(); while( reader.next() ) { Tuple currentKey = (Tuple) reader.getCurrentKey(); valueEntry.setTuple( currentKey ); next.receive( this, 0, valueEntry ); } complete( this ); } catch( Throwable throwable ) { if( !( throwable instanceof OutOfMemoryError ) ) LOG.error( "caught throwable", throwable ); return throwable; } return localThrowable; }
protected Throwable map() throws Exception { Throwable localThrowable = null; try { start( this ); // if multiple ordinals, an input could be duplicated if sourcing multiple paths LogicalInput logicalInput = Util.getFirst( logicalInputs.getValues() ); KeyValueReader reader = (KeyValueReader) logicalInput.getReader(); while( reader.next() ) { Tuple currentKey = (Tuple) reader.getCurrentKey(); valueEntry.setTuple( currentKey ); next.receive( this, 0, valueEntry ); } complete( this ); } catch( Throwable throwable ) { if( !( throwable instanceof OutOfMemoryError ) ) LOG.error( "caught throwable", throwable ); return throwable; } return localThrowable; }
continue; for( KeyValuesReader reader : readers.getValues( ordinal ) )
continue; for( KeyValuesReader reader : readers.getValues( ordinal ) )
LogicalInput logicalInput = Util.getFirst( logicalInputs.getValues() );
LogicalInput logicalInput = Util.getFirst( logicalInputs.getValues() );