private SortedListMultiMap<Integer, KeyValuesReader> getKeyValuesReaders() throws Exception { SortedListMultiMap<Integer, KeyValuesReader> readers = new SortedListMultiMap<>(); for( Map.Entry<Integer, List<LogicalInput>> entry : logicalInputs.getEntries() ) { for( LogicalInput logicalInput : entry.getValue() ) readers.put( entry.getKey(), (KeyValuesReader) logicalInput.getReader() ); } return readers; }
@Override protected List createCollection() { List<Iterable<Tuple>> collection = super.createCollection(); Collections.addAll( collection, array ); // init with nulls return collection; } };
public TezMergeGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); Set<LogicalInput> inputs = new HashSet<>( logicalInputs.getValues() ); if( inputs.size() != 1 ) throw new IllegalArgumentException( "only supports a single input" ); this.logicalInputs = logicalInputs; }
int size = current == null ? readers.getKeys().size() : current.size(); continue; for( KeyValuesReader reader : readers.getValues( ordinal ) ) iterables.set( currentKey, ordinal, currentValues ); return iterables.pollFirstEntry();
/** * Maps each input to an ordinal on the flowelement. an input may be bound to multiple ordinals. * * @param element */ private SortedListMultiMap<Integer, LogicalInput> createInputMap( FlowElement element ) { String id = FlowElements.id( element ); SortedListMultiMap<Integer, LogicalInput> ordinalMap = new SortedListMultiMap<>(); for( LogicalInput logicalInput : inputMap.values() ) { Configuration configuration = inputConfigMap.get( logicalInput ); String foundID = configuration.get( "cascading.node.source" ); if( Util.isEmpty( foundID ) ) throw new IllegalStateException( "cascading.node.source property not set on source LogicalInput" ); if( !foundID.equals( id ) ) continue; String values = configuration.get( "cascading.node.ordinals", "" ); List<Integer> ordinals = Util.split( Integer.class, ",", values ); for( Integer ordinal : ordinals ) ordinalMap.put( ordinal, logicalInput ); } return ordinalMap; }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutput != null ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } super.prepare(); }
public TezGroupGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); this.logicalInputs = logicalInputs; }
private SortedListMultiMap<Tuple, Iterable<Tuple>> getSortedMultiMap( final int length ) { return new SortedListMultiMap<Tuple, Iterable<Tuple>>( getKeyComparator(), length ) { Iterable<Tuple>[] array = new Iterable[ length ]; @Override protected List createCollection() { List<Iterable<Tuple>> collection = super.createCollection(); Collections.addAll( collection, array ); // init with nulls return collection; } }; }
public V get( K key, int pos ) { List<V> multiValues = getMultiValues( key ); return multiValues.get( pos ); }
public Map.Entry<K, List<V>> firstEntry() { return ( (TreeMap) getMap() ).firstEntry(); }
int size = current == null ? readers.getKeys().size() : current.size(); continue; for( KeyValuesReader reader : readers.getValues( ordinal ) ) iterables.set( currentKey, ordinal, currentValues ); return iterables.pollFirstEntry();
/** * Maps each input to an ordinal on the flowelement. an input may be bound to multiple ordinals. * * @param element */ private SortedListMultiMap<Integer, LogicalInput> createInputMap( FlowElement element ) { String id = FlowElements.id( element ); SortedListMultiMap<Integer, LogicalInput> ordinalMap = new SortedListMultiMap<>(); for( LogicalInput logicalInput : inputMap.values() ) { Configuration configuration = inputConfigMap.get( logicalInput ); String foundID = configuration.get( "cascading.node.source" ); if( Util.isEmpty( foundID ) ) throw new IllegalStateException( "cascading.node.source property not set on source LogicalInput" ); if( !foundID.equals( id ) ) continue; String values = configuration.get( "cascading.node.ordinals", "" ); List<Integer> ordinals = Util.split( Integer.class, ",", values ); for( Integer ordinal : ordinals ) ordinalMap.put( ordinal, logicalInput ); } return ordinalMap; }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutput != null ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } super.prepare(); }
public TezGroupGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); this.logicalInputs = logicalInputs; }
private SortedListMultiMap<Tuple, Iterable<Tuple>> getSortedMultiMap( final int length ) { return new SortedListMultiMap<Tuple, Iterable<Tuple>>( getKeyComparator(), length ) { Iterable<Tuple>[] array = new Iterable[ length ]; @Override protected List createCollection() { List<Iterable<Tuple>> collection = super.createCollection(); Collections.addAll( collection, array ); // init with nulls return collection; } }; }
public V set( K key, int pos, V value ) { List<V> multiValues = getMultiValues( key ); return multiValues.set( pos, value ); }
public Map.Entry<K, List<V>> pollFirstEntry() { return ( (TreeMap) getMap() ).pollFirstEntry(); } }
private SortedListMultiMap<Integer, KeyValuesReader> getKeyValuesReaders() throws Exception { SortedListMultiMap<Integer, KeyValuesReader> readers = new SortedListMultiMap<>(); for( Map.Entry<Integer, List<LogicalInput>> entry : logicalInputs.getEntries() ) { for( LogicalInput logicalInput : entry.getValue() ) readers.put( entry.getKey(), (KeyValuesReader) logicalInput.getReader() ); } return readers; }
public TezMergeGate( FlowProcess flowProcess, Splice splice, IORole role, SortedListMultiMap<Integer, LogicalInput> logicalInputs ) { super( flowProcess, splice, role ); if( logicalInputs == null || logicalInputs.getKeys().size() == 0 ) throw new IllegalArgumentException( "inputs must not be null or empty" ); Set<LogicalInput> inputs = new HashSet<>( logicalInputs.getValues() ); if( inputs.size() != 1 ) throw new IllegalArgumentException( "only supports a single input" ); this.logicalInputs = logicalInputs; }
@Override public void prepare() { try { if( logicalInputs != null ) { for( LogicalInput logicalInput : logicalInputs.getValues() ) { LOG.info( "calling {}#start() on: {} {}, for {} inputs", logicalInput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ), logicalInputs.getValues().size() ); logicalInput.start(); } } if( logicalOutputs != null ) { for( LogicalOutput logicalOutput : logicalOutputs ) { LOG.info( "calling {}#start() on: {} {}", logicalOutput.getClass().getSimpleName(), getSplice(), Pipe.id( getSplice() ) ); logicalOutput.start(); } } } catch( Exception exception ) { throw new CascadingException( "unable to start input/output", exception ); } if( role != IORole.source ) collector = new MeasuredOutputCollector( flowProcess, SliceCounters.Write_Duration, createOutputCollector() ); super.prepare(); }