@Before public void setUp() throws IOException { cleanup(); }
private void validateContent(File dir) throws IOException { File expectedFile = new File(dir, partFile); assertTrue("Could not find "+expectedFile, expectedFile.exists()); StringBuffer expectedOutput = new StringBuffer(); expectedOutput.append(key1).append('\t').append(val1).append("\n"); expectedOutput.append(val1).append("\n"); expectedOutput.append(val2).append("\n"); expectedOutput.append(key2).append("\n"); expectedOutput.append(key1).append("\n"); expectedOutput.append(key2).append('\t').append(val2).append("\n"); String output = slurp(expectedFile); assertEquals(output, expectedOutput.toString()); }
@Test public void testAbortV1() throws IOException, InterruptedException { testAbortInternal(1); }
writeOutput(theRecordWriter, tContext); validateContent(outDir); FileUtil.fullyDelete(new File(outDir.toString()));
RecordWriter<?, ?> theRecordWriter = theOutputFormat .getRecordWriter(tContext); writeOutput(theRecordWriter, tContext); th = ie; assertNotNull(th); assertTrue(th instanceof IOException); assertTrue(th.getMessage().contains("fake delete failed")); File jobTmpDir = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME).toString()); File taskTmpDir = new File(jobTmpDir, "_" + taskID); File expectedFile = new File(taskTmpDir, partFile); assertTrue(expectedFile + " does not exists", expectedFile.exists()); th = ie; assertNotNull(th); assertTrue(th instanceof IOException); assertTrue(th.getMessage().contains("fake delete failed")); assertTrue("job temp dir does not exists", jobTmpDir.exists()); FileUtil.fullyDelete(new File(outDir.toString()));
@SuppressWarnings("unchecked") public void testAbort() throws IOException, InterruptedException { Job job = Job.getInstance(); FileOutputFormat.setOutputPath(job, outDir); Configuration conf = job.getConfiguration(); conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt); JobContext jContext = new JobContextImpl(conf, taskID.getJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext); // do setup committer.setupJob(jContext); committer.setupTask(tContext); // write output TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext); writeOutput(theRecordWriter, tContext); // do abort committer.abortTask(tContext); File expectedFile = new File(new Path(committer.getWorkPath(), partFile) .toString()); assertFalse("task temp dir still exists", expectedFile.exists()); committer.abortJob(jContext, JobStatus.State.FAILED); expectedFile = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME) .toString()); assertFalse("job temp dir still exists", expectedFile.exists()); assertEquals("Output directory not empty", 0, new File(outDir.toString()) .listFiles().length); FileUtil.fullyDelete(new File(outDir.toString())); }
writeOutput(theRecordWriter, tContext);
writeOutput(theRecordWriter, tContext); expectedOutput.append(key2).append('\t').append(val2).append("\n"); String output = UtilsForTests.slurp(expectedFile); assertEquals(output, expectedOutput.toString()); FileUtil.fullyDelete(new File(outDir.toString()));
@Test public void testCommitterV1() throws Exception { testCommitterInternal(1, false); }
@Test public void testCommitterRepeatableV1() throws Exception { testCommitterRetryInternal(1); }
writeOutput(theRecordWriter, tContext); if (commitVersion == 1) { assertTrue("Version 1 commits to temporary dir " + jtd, jtd.exists()); validateContent(jtd); } else { assertFalse("Version 2 commits to output dir " + jtd, jtd.exists()); if (recoveryVersion == 1) { assertTrue("Version 1 recovers to " + jtd2, jtd2.exists()); validateContent(jtd2); } else { assertFalse("Version 2 commits to output dir " + jtd2, jtd2.exists()); validateContent(outDir); FileUtil.fullyDelete(new File(outDir.toString()));
RecordWriter<?, ?> theRecordWriter = theOutputFormat .getRecordWriter(tContext); writeOutput(theRecordWriter, tContext);
@Test public void testCommitterV2() throws Exception { testCommitterInternal(2, false); }
@Test public void testCommitterRepeatableV2() throws Exception { testCommitterRetryInternal(2); }
writeOutput(theRecordWriter, tContext); validateContent(outDir);
writeOutput(theRecordWriter, tContext);
@Test public void testCommitterV2TaskCleanupEnabled() throws Exception { testCommitterInternal(2, true); }
@Test public void testAbortV2() throws IOException, InterruptedException { testAbortInternal(2); }
@After public void tearDown() throws IOException { cleanup(); }
writeOutput(theRecordWriter, tContext);