@Override MapReduceFlow createFlow() { Configuration c = configuration; if (job != null) { c = ConfigurationUtils.merge(configuration, job.getConfiguration()); } Configuration conf = ConfigurationUtils.createFrom(c, properties); JobConf jobConf = new JobConf(conf); return new MapReduceFlow(beanName, jobConf, deleteSinkOnInit, false); }
FileOutputFormat.setOutputPath( firstConf, new Path( remove( sinkPath1, true ) ) ); Flow firstMR = new MapReduceFlow( firstConf, true ); FileOutputFormat.setOutputPath( secondConf, new Path( remove( sinkPath2, true ) ) ); Flow secondMR = new MapReduceFlow( secondConf, true ); org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath( job, new Path( remove( sinkPath3, true ) ) ); Flow thirdMR = new MapReduceFlow( new JobConf( job.getConfiguration() ), true );
FileOutputFormat.setOutputPath( firstConf, new Path( remove( sinkPath1, true ) ) ); Flow firstMR = new MapReduceFlow( firstConf, true ); FileOutputFormat.setOutputPath( secondConf, new Path( remove( sinkPath2, true ) ) ); Flow secondMR = new MapReduceFlow( secondConf, true ); org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath( job, new Path( remove( sinkPath3, true ) ) ); Flow thirdMR = new MapReduceFlow( new JobConf( job.getConfiguration() ), true );
@Test public void testFlow() throws IOException { getPlatform().copyFromLocal( inputFileApache ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf = new JobConf( defaultConf ); conf.setJobName( "mrflow" ); conf.setOutputKeyClass( LongWritable.class ); conf.setOutputValueClass( Text.class ); conf.setMapperClass( IdentityMapper.class ); conf.setReducerClass( IdentityReducer.class ); conf.setInputFormat( TextInputFormat.class ); conf.setOutputFormat( TextOutputFormat.class ); FileInputFormat.setInputPaths( conf, new Path( inputFileApache ) ); String outputPath = getOutputPath( "flowTest" ); FileOutputFormat.setOutputPath( conf, new Path( outputPath ) ); Flow flow = new MapReduceFlow( "mrflow", conf, true ); validateLength( new Hfs( new TextLine(), inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); }
@Test public void testFlow() throws IOException { getPlatform().copyFromLocal( inputFileApache ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf = new JobConf( defaultConf ); conf.setJobName( "mrflow" ); conf.setOutputKeyClass( LongWritable.class ); conf.setOutputValueClass( Text.class ); conf.setMapperClass( IdentityMapper.class ); conf.setReducerClass( IdentityReducer.class ); conf.setInputFormat( TextInputFormat.class ); conf.setOutputFormat( TextOutputFormat.class ); FileInputFormat.setInputPaths( conf, new Path( inputFileApache ) ); String outputPath = getOutputPath( "flowTest" ); FileOutputFormat.setOutputPath( conf, new Path( outputPath ) ); Flow flow = new MapReduceFlow( "mrflow", conf, true ); validateLength( new Hfs( new TextLine(), inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); }