@Override public RecoverableWriter createRecoverableWriter() throws IOException { return unsafeFileSystem.createRecoverableWriter(); }
private RecoverableWriter getNewFileSystemWriter() throws Exception { return getFileSystem().createRecoverableWriter(); }
this.fsWriter = FileSystem.get(basePath.toUri()).createRecoverableWriter(); } catch (IOException e) { LOG.error("Unable to create filesystem for path: {}", basePath);
@Test public void testSerializationEmpty() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final RecoverableWriter writer = fs.createRecoverableWriter(); final Path testBucket = new Path(testFolder.getPath(), "test"); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, null, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); Assert.assertNull(recoveredState.getInProgressResumableFile()); Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty()); }
final RecoverableWriter writer = fs.createRecoverableWriter();
final RecoverableWriter writer = fs.createRecoverableWriter();
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }
@Override public RecoverableWriter createRecoverableWriter() throws IOException { return unsafeFileSystem.createRecoverableWriter(); }
this.fsWriter = FileSystem.get(basePath.toUri()).createRecoverableWriter(); } catch (IOException e) { LOG.error("Unable to create filesystem for path: {}", basePath);
this.fsWriter = FileSystem.get(basePath.toUri()).createRecoverableWriter(); } catch (IOException e) { LOG.error("Unable to create filesystem for path: {}", basePath);