RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }
RecoverableWriter.ResumeRecoverable persist() throws IOException { return currentPartStream.persist(); }
@Override public long getSize() throws IOException { return currentPartStream.getPos(); }
@Test(expected = IOException.class) public void testResumeAfterCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); RecoverableWriter.ResumeRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); } // this should throw an exception as the file is already committed writer.recover(recoverable); fail(); }
stream.write((PENDING_CONTENT + '-' + j).getBytes(Charset.forName("UTF-8"))); recoverables.add(stream.closeForCommit().getRecoverable()); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); SimpleVersionedStringSerializer.INSTANCE ); stream.close();
@Test(expected = IOException.class) public void testExceptionWritingAfterCloseForCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().getRecoverable(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); fail(); } }
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }
@Test public void testResumeWithWrongOffset() throws Exception { // this is a rather unrealistic scenario, but it is to trigger // truncation of the file and try to resume with missing data. final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter.ResumeRecoverable recoverable1; final RecoverableWriter.ResumeRecoverable recoverable2; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable1 = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable2 = stream.persist(); stream.write(testData3.getBytes(StandardCharsets.UTF_8)); } try (RecoverableFsDataOutputStream ignored = writer.recover(recoverable1)) { // this should work fine } catch (Exception e) { fail(); } // this should throw an exception try (RecoverableFsDataOutputStream ignored = writer.recover(recoverable2)) { fail(); } catch (IOException e) { // we expect this return; } fail(); }
@Test public void testCommitAfterPersist() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } } }
@Test public void testCommitAfterNormalClose() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1, fileContents.getValue()); } } }
recoverables.put(INIT_EMPTY_PERSIST, stream.persist()); stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverables.put(INTERM_WITH_STATE_PERSIST, stream.persist()); recoverables.put(INTERM_WITH_NO_ADDITIONAL_STATE_PERSIST, stream.persist()); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverables.put(FINAL_WITH_EXTRA_STATE, stream.persist()); recoveredStream.write(testData3.getBytes(StandardCharsets.UTF_8)); recoveredStream.closeForCommit().commit();
stream.write((PENDING_CONTENT + '-' + j).getBytes(Charset.forName("UTF-8"))); recoverables.add(stream.closeForCommit().getRecoverable());
@Test public void testCloseWithNoData() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableFsDataOutputStream stream = writer.open(path); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertTrue(fileContents.getKey().getName().startsWith(".part-0.inprogress.")); Assert.assertTrue(fileContents.getValue().isEmpty()); } stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertTrue(fileContents.getValue().isEmpty()); } }
RecoverableWriter.ResumeRecoverable persist() throws IOException { return currentPartStream.persist(); }
@Override public long getSize() throws IOException { return currentPartStream.getPos(); }
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }
RecoverableWriter.ResumeRecoverable persist() throws IOException { return currentPartStream.persist(); }
@Override public long getSize() throws IOException { return currentPartStream.getPos(); }
RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }