private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }
@Test public void testResumeWithWrongOffset() throws Exception { // this is a rather unrealistic scenario, but it is to trigger // truncation of the file and try to resume with missing data. final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter.ResumeRecoverable recoverable1; final RecoverableWriter.ResumeRecoverable recoverable2; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable1 = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable2 = stream.persist(); stream.write(testData3.getBytes(StandardCharsets.UTF_8)); } try (RecoverableFsDataOutputStream ignored = writer.recover(recoverable1)) { // this should work fine } catch (Exception e) { fail(); } // this should throw an exception try (RecoverableFsDataOutputStream ignored = writer.recover(recoverable2)) { fail(); } catch (IOException e) { // we expect this return; } fail(); }
private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }
deserializer.deserialize(serializer.getVersion(), serializedRecoverable); try (final RecoverableFsDataOutputStream recoveredStream = newWriter.recover(recoveredRecoverable)) {
private void restoreInProgressFile(final BucketState<BucketID> state) throws IOException { if (!state.hasInProgressResumableFile()) { return; } // we try to resume the previous in-progress file final ResumeRecoverable resumable = state.getInProgressResumableFile(); if (fsWriter.supportsResume()) { final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); inProgressPart = partFileFactory.resumeFrom( bucketId, stream, resumable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. fsWriter.recoverForCommit(resumable).commitAfterRecovery(); } if (fsWriter.requiresCleanupOfRecoverableState()) { fsWriter.cleanupRecoverableState(resumable); } }
@Test(expected = IOException.class) public void testResumeAfterCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); RecoverableWriter.ResumeRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); } // this should throw an exception as the file is already committed writer.recover(recoverable); fail(); }