void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
@Test public void testCloseWithNoData() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableFsDataOutputStream stream = writer.open(path); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertTrue(fileContents.getKey().getName().startsWith(".part-0.inprogress.")); Assert.assertTrue(fileContents.getValue().isEmpty()); } stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertTrue(fileContents.getValue().isEmpty()); } }
@Test public void testCommitAfterNormalClose() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1, fileContents.getValue()); } } }
void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
writer.recoverForCommit(recoverable).commit();
writer.recoverForCommit(recoverable).commit();
recoveredStream.closeForCommit().commit();
@Test public void testCommitAfterPersist() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } } }
@Test(expected = IOException.class) public void testResumeAfterCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); RecoverableWriter.ResumeRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); } // this should throw an exception as the file is already committed writer.recover(recoverable); fail(); }