private void commitRecoveredPendingFiles(final BucketState<BucketID> state) throws IOException { // we commit pending files for checkpoints that precess the last successful one, from which we are recovering for (List<CommitRecoverable> committables: state.getCommittableFilesPerCheckpoint().values()) { for (CommitRecoverable committable: committables) { fsWriter.recoverForCommit(committable).commitAfterRecovery(); } } }
void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }
writer.recoverForCommit(recoverable).commit();
writer.recoverForCommit(recoverable).commit();
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
private void commitRecoveredPendingFiles(final BucketState<BucketID> state) throws IOException { // we commit pending files for checkpoints that precess the last successful one, from which we are recovering for (List<CommitRecoverable> committables: state.getCommittableFilesPerCheckpoint().values()) { for (CommitRecoverable committable: committables) { fsWriter.recoverForCommit(committable).commitAfterRecovery(); } } }
private void commitRecoveredPendingFiles(final BucketState<BucketID> state) throws IOException { // we commit pending files for checkpoints that precess the last successful one, from which we are recovering for (List<CommitRecoverable> committables: state.getCommittableFilesPerCheckpoint().values()) { for (CommitRecoverable committable: committables) { fsWriter.recoverForCommit(committable).commitAfterRecovery(); } } }
void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { checkNotNull(fsWriter); Iterator<Map.Entry<Long, List<CommitRecoverable>>> it = pendingPartsPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { Map.Entry<Long, List<CommitRecoverable>> entry = it.next(); for (CommitRecoverable committable : entry.getValue()) { fsWriter.recoverForCommit(committable).commit(); } it.remove(); } cleanupOutdatedResumables(checkpointId); }
private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }
private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }