Refine search
HCatMapRedUtil.createTaskAttemptContext(context); configureDynamicStorageHandler(currTaskContext, dynamicPartValues); localJobInfo = HCatBaseOutputFormat.getJobInfo(currTaskContext.getConfiguration()); ReflectionUtils.newInstance(storageHandler.getSerDeClass(), currTaskContext.getJobConf()); try { InternalUtil.initializeOutputSerDe(currSerDe, currTaskContext.getConfiguration(), localJobInfo); } catch (SerDeException e) { currTaskContext.getJobConf()); currTaskContext.getJobConf().getOutputCommitter(); currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible()); currTaskContext.getConfiguration().set( "mapred.work.output.dir", new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext) .getWorkPath().toString()); Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), "")); baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()), currTaskContext.getJobConf(), childPath.toString(), InternalUtil.createReporter(currTaskContext));
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); if (!fs.rename(taskOutput, finalOutputPath)) { if (!fs.delete(finalOutputPath, true)) { throw new IOException("Failed to delete earlier output of task: " + attemptId);
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
Path getTempTaskOutputPath(TaskAttemptContext taskContext) throws IOException { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } return null; }
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); // Get the file-system for the task output directory FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit if (fs.exists(taskOutputPath)) { return true; } } return false; }
public void abortTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); context.getProgressible().progress(); fs.delete(taskOutputPath, true); } }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); if (fs.exists(taskOutputPath)) { throw new IOException(); } } } }
@Override public void abortTask(TaskAttemptContext taskContext) throws IOException { writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME); }
@Override public String generateFileName(JobContext context, String topic, String brokerId, int partitionId, int count, long offset, String encodedPartition) { //adding taskId to filename to avoid name collision int taskId = ((TaskAttemptContext) context).getTaskAttemptID().getTaskID().getId(); return super.generateFileName(context, topic, String.valueOf(System.currentTimeMillis() / 1000), taskId, count, 0, encodedPartition); }
public JobConf getJobConf() { return (JobConf) getConfiguration(); }
private void logTask(TaskAttemptContext taskContext) { if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format( "Old-style output committer is used in the task: {0} ({1})", taskContext.getJobName(), taskContext.getTaskAttemptID())); } } }
private Counter getTopicSkipOldCounter() { try { //In Hadoop 2, TaskAttemptContext.getCounter() is available Method getCounterMethod = context.getClass().getMethod("getCounter", String.class, String.class); return ((Counter) getCounterMethod.invoke(context, "total", "skip-old")); } catch (NoSuchMethodException e) { //In Hadoop 1, TaskAttemptContext.getCounter() is not available //Have to cast context to TaskAttemptContext in the mapred package, then get a StatusReporter instance org.apache.hadoop.mapred.TaskAttemptContext mapredContext = (org.apache.hadoop.mapred.TaskAttemptContext) context; return ((StatusReporter) mapredContext.getProgressible()).getCounter("total", "skip-old"); } catch (IllegalArgumentException e) { log.error("IllegalArgumentException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (IllegalAccessException e) { log.error("IllegalAccessException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (InvocationTargetException e) { log.error("InvocationTargetException obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } }
ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job); committer = outputFormat.getOutputCommitter(taskContext); } else {
InterruptedException { jobContext = new JobContext(job, id, reporter); taskContext = new TaskAttemptContext(job, taskId, reporter); if (getState() == TaskStatus.State.UNASSIGNED) { setState(TaskStatus.State.RUNNING); LOG.debug("using new api for output committer"); outputFormat = ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job); committer = outputFormat.getOutputCommitter(taskContext); } else {
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); if (!fs.rename(taskOutput, finalOutputPath)) { if (!fs.delete(finalOutputPath, true)) { throw new IOException("Failed to delete earlier output of task: " + attemptId);
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
Path getTempTaskOutputPath(TaskAttemptContext taskContext) throws IOException { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); try { FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } catch (IOException ie) { LOG.warn("Error getting temp task output in " + outputPath, ie); throw ie; } } return null; }
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { try { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); // Get the file-system for the task output directory FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit if (fs.exists(taskOutputPath)) { return true; } } } catch (IOException ioe) { throw ioe; } return false; }