public boolean runJob(AvroJob job) throws IOException, InterruptedException, ClassNotFoundException { job.submit(); String idf = System.getProperty(OOZIE_JAVA_MAIN_NEW_JOBID_FILE); if (idf != null) { File idFile = new File(idf); String jobId = job.getJobID().toString(); LOG.info("Save the Job ID:" + jobId + " to the file:" + idf); Properties props = new Properties(); props.setProperty("id", jobId); OutputStream os = new FileOutputStream(idFile); props.store(os, "Store Job ID"); os.close(); } return job.waitForCompletion(true); }
job.setOutputValueClass(ReportValue.class); if (!job.waitForCompletion(true)) { LOG.error("job.waitForCompletion return false"); throw new IOException(("JobCube.waitForCompletion return false"));
private void runJobComb_mid(List<String> inputs, String output, Integer reduceNum) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf_comb = null == conf ? new Configuration(conf) : conf; AvroJob job = AvroJob.getAvroJob(conf_comb); job.setJobName(this.getUniqueCode() + "_comb_mid"); job.setNumReduceTasks(reduceNum); FileSystem fs = FileSystem.get(conf_comb); for (String input : inputs) { Path inputPath = new Path(input); if (fs.exists(inputPath)) { AvroPairInputFormat.addInputPath(job, inputPath); LOG.info("add input file: " + inputPath); } else { LOG.warn("skip not exist file: " + inputPath); } } job.setInputFormatClass(AvroPairInputFormat.class); job.setMapperClass(ReportMRComb.M_mid.class); // read from interm, do not setCombinerClass!! job.setReducerClass(ReportMRComb.R.class); FileOutputFormat.setOutputPath(job, new Path(output)); job.setOutputFormatClass(HdfsOutputFormat.class); job.setOutputKeyClass(ReportKey.class); job.setOutputValueClass(ReportValue.class); if (!job.waitForCompletion(true)) { LOG.error("job.waitForCompletion return false"); throw new IOException(("JobComb_mid.waitForCompletion return false")); } }
job.setOutputValueClass(ReportValue.class); if (!job.waitForCompletion(true)) { LOG.error("job.waitForCompletion return false"); throw new IOException(("JobComb_org.waitForCompletion return false"));