conf_cube.set("mapreduce.map.java.opts", java_opts); AvroJob job = AvroJob.getAvroJob(conf_cube); job.setJobName(this.getUniqueCode() + "_cube"); if (conf_cube.getInt(ConstantConf.CUBE_REDUCE_NUM, 0) != 0) {
private void runJobComb_mid(List<String> inputs, String output, Integer reduceNum) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf_comb = null == conf ? new Configuration(conf) : conf; AvroJob job = AvroJob.getAvroJob(conf_comb); job.setJobName(this.getUniqueCode() + "_comb_mid"); job.setNumReduceTasks(reduceNum); FileSystem fs = FileSystem.get(conf_comb); for (String input : inputs) { Path inputPath = new Path(input); if (fs.exists(inputPath)) { AvroPairInputFormat.addInputPath(job, inputPath); LOG.info("add input file: " + inputPath); } else { LOG.warn("skip not exist file: " + inputPath); } } job.setInputFormatClass(AvroPairInputFormat.class); job.setMapperClass(ReportMRComb.M_mid.class); // read from interm, do not setCombinerClass!! job.setReducerClass(ReportMRComb.R.class); FileOutputFormat.setOutputPath(job, new Path(output)); job.setOutputFormatClass(HdfsOutputFormat.class); job.setOutputKeyClass(ReportKey.class); job.setOutputValueClass(ReportValue.class); if (!job.waitForCompletion(true)) { LOG.error("job.waitForCompletion return false"); throw new IOException(("JobComb_mid.waitForCompletion return false")); } }
AvroJob job = AvroJob.getAvroJob(conf_comb); job.setJobName(this.getUniqueCode() + "_comb_org"); if (conf_comb.getInt(ConstantConf.COMB_REDUCE_NUM, 0) != 0) {