private TaskAttemptContext getContext(String nameOutput) throws IOException { TaskAttemptContext taskContext = taskContexts.get(nameOutput); if (taskContext != null) { return taskContext; } // The following trick leverages the instantiation of a record writer via // the job thus supporting arbitrary output formats. Job job = Job.getInstance(context.getConfiguration()); job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput)); job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput)); job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput)); taskContext = new TaskAttemptContextImpl(job.getConfiguration(), context .getTaskAttemptID(), new WrappedStatusReporter(context)); taskContexts.put(nameOutput, taskContext); return taskContext; }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath * <b>Warning</b>: when the baseOutputPath is a path that resolves * outside of the final job output directory, the directory is created * immediately and then persists through subsequent task retries, breaking * the concept of output committing. */ @SuppressWarnings("unchecked") public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) throws IOException, InterruptedException { checkBaseOutputPath(baseOutputPath); if (jobOutputFormatContext == null) { jobOutputFormatContext = new TaskAttemptContextImpl(context.getConfiguration(), context.getTaskAttemptID(), new WrappedStatusReporter(context)); } getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ @SuppressWarnings("unchecked") public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) throws IOException, InterruptedException { checkBaseOutputPath(baseOutputPath); TaskAttemptContext taskContext = new TaskAttemptContextImpl(context.getConfiguration(), context.getTaskAttemptID(), new WrappedStatusReporter(context)); getRecordWriter(taskContext, baseOutputPath).write(key, value); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ @SuppressWarnings("unchecked") public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) throws IOException, InterruptedException { checkBaseOutputPath(baseOutputPath); if (jobOutputFormatContext == null) { jobOutputFormatContext = new TaskAttemptContextImpl(context.getConfiguration(), context.getTaskAttemptID(), new WrappedStatusReporter(context)); } getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ @SuppressWarnings("unchecked") public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) throws IOException, InterruptedException { checkBaseOutputPath(baseOutputPath); if (jobOutputFormatContext == null) { jobOutputFormatContext = new TaskAttemptContextImpl(context.getConfiguration(), context.getTaskAttemptID(), new WrappedStatusReporter(context)); } getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ @SuppressWarnings("unchecked") public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) throws IOException, InterruptedException { checkBaseOutputPath(baseOutputPath); if (jobOutputFormatContext == null) { jobOutputFormatContext = new TaskAttemptContextImpl(context.getConfiguration(), context.getTaskAttemptID(), new WrappedStatusReporter(context)); } getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value); }
private TaskAttemptContext getContext(String nameOutput) throws IOException { TaskAttemptContext taskContext = taskContexts.get(nameOutput); if (taskContext != null) { return taskContext; } // The following trick leverages the instantiation of a record writer via // the job thus supporting arbitrary output formats. Job job = Job.getInstance(context.getConfiguration()); job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput)); job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput)); job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput)); taskContext = new TaskAttemptContextImpl(job.getConfiguration(), context .getTaskAttemptID(), new WrappedStatusReporter(context)); taskContexts.put(nameOutput, taskContext); return taskContext; }
private TaskAttemptContext getContext(String nameOutput) throws IOException { TaskAttemptContext taskContext = taskContexts.get(nameOutput); if (taskContext != null) { return taskContext; } // The following trick leverages the instantiation of a record writer via // the job thus supporting arbitrary output formats. Job job = Job.getInstance(context.getConfiguration()); job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput)); job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput)); job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput)); taskContext = new TaskAttemptContextImpl(job.getConfiguration(), context .getTaskAttemptID(), new WrappedStatusReporter(context)); taskContexts.put(nameOutput, taskContext); return taskContext; }
private TaskAttemptContext getContext(String nameOutput) throws IOException { TaskAttemptContext taskContext = taskContexts.get(nameOutput); if (taskContext != null) { return taskContext; } // The following trick leverages the instantiation of a record writer via // the job thus supporting arbitrary output formats. Job job = Job.getInstance(context.getConfiguration()); job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput)); job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput)); job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput)); taskContext = new TaskAttemptContextImpl(job.getConfiguration(), context .getTaskAttemptID(), new WrappedStatusReporter(context)); taskContexts.put(nameOutput, taskContext); return taskContext; }
private TaskAttemptContext getContext(String nameOutput) throws IOException { TaskAttemptContext taskContext = taskContexts.get(nameOutput); if (taskContext != null) { return taskContext; } // The following trick leverages the instantiation of a record writer via // the job thus supporting arbitrary output formats. Job job = new Job(context.getConfiguration()); job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput)); job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput)); job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput)); taskContext = new TaskAttemptContextImpl(job.getConfiguration(), context .getTaskAttemptID(), new WrappedStatusReporter(context)); taskContexts.put(nameOutput, taskContext); return taskContext; }