@Override public boolean doSpill( Spillable spillable, int size ) { return current() <= size; }
@Override public Collection<Tuple> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleList( spillThreshold, tupleSerialization, codec ); } }
@Override public Map<Tuple, Collection<Tuple>> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleMap( capacity, loadFactor, mapThreshold, listThreshold, flowProcess ); } }
int current() { return Math.max( minThreshold, Math.min( getInitListThreshold(), getMapThreshold() / size() ) ); }
public HadoopSpillableTupleMap( int initialCapacity, float loadFactor, int mapThreshold, int listThreshold, FlowProcess<? extends Configuration> flowProcess ) { super( initialCapacity, loadFactor, mapThreshold, listThreshold ); this.flowProcess = flowProcess; this.spillStrategy = getSpillStrategy(); FactoryLoader loader = FactoryLoader.getInstance(); this.tupleCollectionFactory = loader.loadFactoryFrom( flowProcess, TupleMapFactory.TUPLE_MAP_FACTORY, HadoopTupleCollectionFactory.class ); }
@Override protected Collection<Tuple> createTupleCollection( Tuple tuple ) { Collection<Tuple> collection = tupleCollectionFactory.create( flowProcess ); if( collection instanceof Spillable ) { ( (Spillable) collection ).setGrouping( tuple ); ( (Spillable) collection ).setSpillListener( getSpillListener() ); ( (Spillable) collection ).setSpillStrategy( spillStrategy ); } return collection; }
@Override public void initialize( FlowProcess<? extends Configuration> flowProcess ) { this.spillThreshold = getThreshold( flowProcess, defaultListThreshold ); this.codec = HadoopSpillableTupleList.getCodec( flowProcess, HadoopSpillableTupleList.defaultCodecs ); this.tupleSerialization = new TupleSerialization( flowProcess ); }
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
compressor = getCompressor(); outputStream = codec.createOutputStream( outputStream, compressor );
@Override public Collection<Tuple> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleList( spillThreshold, tupleSerialization, codec ); } }
int current() { return Math.max( minThreshold, Math.min( getInitListThreshold(), getMapThreshold() / size() ) ); }
@Override public Map<Tuple, Collection<Tuple>> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleMap( capacity, loadFactor, mapThreshold, listThreshold, flowProcess ); } }
@Override public String getSpillReason( Spillable spillable ) { return "met current threshold: " + current(); } };
public HadoopSpillableTupleMap( int initialCapacity, float loadFactor, int mapThreshold, int listThreshold, FlowProcess<? extends Configuration> flowProcess ) { super( initialCapacity, loadFactor, mapThreshold, listThreshold ); this.flowProcess = flowProcess; this.spillStrategy = getSpillStrategy(); FactoryLoader loader = FactoryLoader.getInstance(); this.tupleCollectionFactory = loader.loadFactoryFrom( flowProcess, TupleMapFactory.TUPLE_MAP_FACTORY, HadoopTupleCollectionFactory.class ); }
@Override protected Collection<Tuple> createTupleCollection( Tuple tuple ) { Collection<Tuple> collection = tupleCollectionFactory.create( flowProcess ); if( collection instanceof Spillable ) { ( (Spillable) collection ).setGrouping( tuple ); ( (Spillable) collection ).setSpillListener( getSpillListener() ); ( (Spillable) collection ).setSpillStrategy( spillStrategy ); } return collection; }
@Override public void initialize( FlowProcess<? extends Configuration> flowProcess ) { this.spillThreshold = getThreshold( flowProcess, defaultListThreshold ); this.codec = HadoopSpillableTupleList.getCodec( flowProcess, HadoopSpillableTupleList.defaultCodecs ); this.tupleSerialization = new TupleSerialization( flowProcess ); }
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
public Collection<Tuple> create(FlowProcess<? extends Configuration> flowProcess) { return new HadoopSpillableTupleList(this.spillThreshold, this.tupleSerialization, this.codec); } }
@Override public boolean doSpill( Spillable spillable, int size ) { return current() <= size; }
@Override public String getSpillReason( Spillable spillable ) { return "met current threshold: " + current(); } };