return serializer.deserialize(version, dataOnly);
/** * Deserializes the version and datum from a stream. * * <p>This method deserializes data serialized via * {@link #writeVersionAndSerialize(SimpleVersionedSerializer, Object, DataOutputView)}. * * <p>The first four bytes will be interpreted as the version. The next four bytes will be * interpreted as the length of the datum bytes, then length-many bytes will be read. * Finally, the datum is deserialized via the {@link SimpleVersionedSerializer#deserialize(int, byte[])} * method. * * @param serializer The serializer to serialize the datum with. * @param in The stream to deserialize from. */ public static <T> T readVersionAndDeSerialize(SimpleVersionedSerializer<T> serializer, DataInputView in) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(in, "in"); final int version = in.readInt(); final int length = in.readInt(); final byte[] data = new byte[length]; in.readFully(data); return serializer.deserialize(version, data); }
final byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer(); final RecoverableWriter.ResumeRecoverable recoveredRecoverable = deserializer.deserialize(serializer.getVersion(), serializedRecoverable);
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
return serializer.deserialize(version, dataOnly);
/** * Deserializes the version and datum from a stream. * * <p>This method deserializes data serialized via * {@link #writeVersionAndSerialize(SimpleVersionedSerializer, Object, DataOutputView)}. * * <p>The first four bytes will be interpreted as the version. The next four bytes will be * interpreted as the length of the datum bytes, then length-many bytes will be read. * Finally, the datum is deserialized via the {@link SimpleVersionedSerializer#deserialize(int, byte[])} * method. * * @param serializer The serializer to serialize the datum with. * @param in The stream to deserialize from. */ public static <T> T readVersionAndDeSerialize(SimpleVersionedSerializer<T> serializer, DataInputView in) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(in, "in"); final int version = in.readInt(); final int length = in.readInt(); final byte[] data = new byte[length]; in.readFully(data); return serializer.deserialize(version, data); }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
final byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
final byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); resumables.add(commitableSerializer.deserialize(committableVersion, bytes));