/** * Records information about a completed job into the history table of a Fiji instance. * * If the attempt fails due to an IOException (a Fiji cannot be made, there is no job history * table, its content is corrupted, etc.), we catch the exception and warn the user. * * @param fiji Fiji instance to write the job record to. * @throws IOException on I/O error. */ private void recordJobHistory(Fiji fiji) throws IOException { final Job job = getHadoopJob(); JobHistoryFijiTable jobHistory = null; try { jobHistory = JobHistoryFijiTable.open(fiji); jobHistory.recordJob(job, mJobStartTime, mJobEndTime); } catch (IOException ioe) { // We swallow errors for recording jobs, because it's a non-fatal error for the task. LOG.warn( "Error recording job {} in history table of Fiji instance {}:\n" + "{}\n" + "This does not affect the success of job {}.\n", getHadoopJob().getJobID(), fiji.getURI(), StringUtils.stringifyException(ioe), getHadoopJob().getJobID()); } finally { IOUtils.closeQuietly(jobHistory); } }
final Configuration conf = getHadoopJob().getConfiguration(); final Set<FijiURI> instanceURIs = Sets.newHashSet(); if (conf.get(FijiConfKeys.FIJI_INPUT_TABLE_URI) != null) { "Error recording job {} in history table of Fiji instance {}: {}\n" + "This does not affect the success of job {}.", getHadoopJob().getJobID(), instanceURI, ioe.getMessage(), getHadoopJob().getJobID());
/** * Runs the job to invert the email domain popularity map and sort by popularity. * * @param inputPath The map from email domains to their popularity. * @param outputPath The output path for the sorted map of popularity to email domains. * @param numSplits The number of output file shards to write. * @param conf Base Hadoop configuration. * @return Whether the job was successful. * @throws Exception If there is an exception. */ private boolean invertAndSortByPopularity( Path inputPath, Path outputPath, int numSplits, Configuration conf) throws Exception { LOG.info("Configuring a fiji mapreduce job..."); FijiMapReduceJobBuilder jobBuilder = FijiMapReduceJobBuilder.create() .withConf(conf) .withInput(MapReduceJobInputs.newSequenceFileMapReduceJobInput(inputPath)) .withMapper(InvertCountMapper.class) .withReducer(TextListReducer.class) .withOutput(MapReduceJobOutputs.newAvroKeyValueMapReduceJobOutput(outputPath, numSplits)); LOG.info("Building the transform job..."); FijiMapReduceJob job = jobBuilder.build(); // Configure the job to sort by decreasing key, so the most popular email domain is first. job.getHadoopJob().setSortComparatorClass(DescendingIntWritableComparator.class); LOG.info("Running the transform job..."); return job.run(); }