private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception { for (byte[] serializedRecoveredState : bucketStates.get()) { final BucketState<BucketID> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize( bucketStateSerializer, serializedRecoveredState); handleRestoredBucketState(recoveredState); } }
@VisibleForTesting BucketState<BucketID> deserializeV1(DataInputView in) throws IOException { final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in); final String bucketPathStr = in.readUTF(); final long creationTime = in.readLong(); current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
@Test public void testSerializationRoundTrip() throws IOException { final SimpleVersionedSerializer<String> utfEncoder = new SimpleVersionedSerializer<String>() { private static final int VERSION = Integer.MAX_VALUE / 2; // version should occupy many bytes @Override public int getVersion() { return VERSION; } @Override public byte[] serialize(String str) throws IOException { return str.getBytes(StandardCharsets.UTF_8); } @Override public String deserialize(int version, byte[] serialized) throws IOException { assertEquals(VERSION, version); return new String(serialized, StandardCharsets.UTF_8); } }; final String testString = "dugfakgs"; final DataOutputSerializer out = new DataOutputSerializer(32); SimpleVersionedSerialization.writeVersionAndSerialize(utfEncoder, testString, out); final byte[] outBytes = out.getCopyOfBuffer(); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(utfEncoder, testString); assertArrayEquals(bytes, outBytes); final DataInputDeserializer in = new DataInputDeserializer(bytes); final String deserialized = SimpleVersionedSerialization.readVersionAndDeSerialize(utfEncoder, in); final String deserializedFromBytes = SimpleVersionedSerialization.readVersionAndDeSerialize(utfEncoder, outBytes); assertEquals(testString, deserialized); assertEquals(testString, deserializedFromBytes); }
@Test public void testSerializationEmpty() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final RecoverableWriter writer = fs.createRecoverableWriter(); final Path testBucket = new Path(testFolder.getPath(), "test"); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, null, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); Assert.assertNull(recoveredState.getInProgressResumableFile()); Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty()); }
final String deserialized = SimpleVersionedSerialization.readVersionAndDeSerialize(emptySerializer, in); final String deserializedFromBytes = SimpleVersionedSerialization.readVersionAndDeSerialize(emptySerializer, outBytes); assertEquals(testString, deserialized); assertEquals(testString, deserializedFromBytes);
final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);
final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }
private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception { for (byte[] serializedRecoveredState : bucketStates.get()) { final BucketState<BucketID> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize( bucketStateSerializer, serializedRecoveredState); handleRestoredBucketState(recoveredState); } }
private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception { for (byte[] serializedRecoveredState : bucketStates.get()) { final BucketState<BucketID> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize( bucketStateSerializer, serializedRecoveredState); handleRestoredBucketState(recoveredState); } }
@VisibleForTesting BucketState<BucketID> deserializeV1(DataInputView in) throws IOException { final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in); final String bucketPathStr = in.readUTF(); final long creationTime = in.readLong(); current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
@VisibleForTesting BucketState<BucketID> deserializeV1(DataInputView in) throws IOException { final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in); final String bucketPathStr = in.readUTF(); final long creationTime = in.readLong(); current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);