/** {@inheritDoc} */ @Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx) throws IOException { return new TestOutputCommitter(ctx, (FileOutputCommitter)super.getOutputCommitter(ctx)); } }
@Override public Void call() throws IOException, InterruptedException { final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]); outputCommitter.setupTask(taCtx[taskIdx]); final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]); writeOutput(rw, taCtx[taskIdx]); outputCommitter.commitTask(taCtx[taskIdx]); return null; } });
/** * Follow the entire committer chain down and create a new committer from * the output format. * @throws Throwable on a failure. */ @Test public void testFileOutputFormatBinding() throws Throwable { Configuration conf = newBondedConfiguration(); conf.set(FileOutputFormat.OUTDIR, HTTP_PATH.toUri().toString()); TextOutputFormat<String, String> off = new TextOutputFormat<>(); SimpleCommitter committer = (SimpleCommitter) off.getOutputCommitter(taskAttempt(conf)); assertEquals("Wrong output path from "+ committer, HTTP_PATH, committer.getOutputPath()); }
/** * Follow the entire committer chain down and create a new committer from * the output format. * @throws Throwable on a failure. */ @Test public void testFileOutputFormatBindingNoPath() throws Throwable { Configuration conf = new Configuration(); conf.unset(FileOutputFormat.OUTDIR); // set up for the schema factory conf.set(COMMITTER_FACTORY_CLASS, NAMED_COMMITTER_FACTORY); conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName()); httpToSimpleFactory(conf); TextOutputFormat<String, String> off = new TextOutputFormat<>(); SimpleCommitter committer = (SimpleCommitter) off.getOutputCommitter(taskAttempt(conf)); assertNull("Output path from "+ committer, committer.getOutputPath()); }
final int taskIdx = i; executor.submit((Callable<Void>) () -> { final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]); outputCommitter.setupTask(taCtx[taskIdx]); final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);