@Override public Collection<Tuple> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleList( spillThreshold, tupleSerialization, codec ); } }
jobConf.set( "cascading.serialization.tokens", "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName() ); // not using Text, just testing parsing HadoopSpillableTupleList list = new HadoopSpillableTupleList( threshold, codec, jobConf ); double random = Math.random(); list.add( new Tuple( i, aString, random, new Text( aString ), new TestText( aString ), new Tuple( "inner tuple", new BytesWritable( aString.getBytes() ) ) ) ); assertEquals( "not equal: list.size();", size, list.size() ); assertEquals( "not equal: list.getNumFiles()", spills, list.spillCount() ); Iterator<Tuple> iterator = list.iterator();
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
@Override public void initialize( FlowProcess<? extends Configuration> flowProcess ) { this.spillThreshold = getThreshold( flowProcess, defaultListThreshold ); this.codec = HadoopSpillableTupleList.getCodec( flowProcess, HadoopSpillableTupleList.defaultCodecs ); this.tupleSerialization = new TupleSerialization( flowProcess ); }
compressor = getCompressor(); outputStream = codec.createOutputStream( outputStream, compressor );
decompressor = getDecompressor(); inputStream = codec.createInputStream( inputStream, decompressor );
jobConf.set( "cascading.serialization.tokens", "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName() ); // not using Text, just testing parsing HadoopSpillableTupleList list = new HadoopSpillableTupleList( threshold, codec, jobConf ); double random = Math.random(); list.add( new Tuple( i, aString, random, new Text( aString ), new TestText( aString ), new Tuple( "inner tuple", new BytesWritable( aString.getBytes() ) ) ) ); assertEquals( "not equal: list.size();", size, list.size() ); assertEquals( "not equal: list.getNumFiles()", spills, list.spillCount() ); Iterator<Tuple> iterator = list.iterator();
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
@Override public void initialize( FlowProcess<? extends Configuration> flowProcess ) { this.spillThreshold = getThreshold( flowProcess, defaultListThreshold ); this.codec = HadoopSpillableTupleList.getCodec( flowProcess, HadoopSpillableTupleList.defaultCodecs ); this.tupleSerialization = new TupleSerialization( flowProcess ); }
compressor = getCompressor(); outputStream = codec.createOutputStream( outputStream, compressor );
decompressor = getDecompressor(); inputStream = codec.createInputStream( inputStream, decompressor );
@Override public Collection<Tuple> create( FlowProcess<? extends Configuration> flowProcess ) { return new HadoopSpillableTupleList( spillThreshold, tupleSerialization, codec ); } }
public Collection<Tuple> create(FlowProcess<? extends Configuration> flowProcess) { return new HadoopSpillableTupleList(this.spillThreshold, this.tupleSerialization, this.codec); } }