/** * Utility method to conjure up a new scope for the generic parameters. */ @SuppressWarnings("unchecked") private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility( TypeSerializer<?> serializer, TypeSerializerSnapshot<?> snapshot) { TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer; TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot; return typedSnapshot.resolveSchemaCompatibility(typedSerializer); }
/** * Creates the restore serializer from the pos-th config snapshot. */ public <T> TypeSerializer<T> getRestoredNestedSerializer(int pos) { checkArgument(pos < nestedSnapshots.length); @SuppressWarnings("unchecked") TypeSerializerSnapshot<T> snapshot = (TypeSerializerSnapshot<T>) nestedSnapshots[pos]; return snapshot.restoreSerializer(); }
private TypeSerializerSchemaCompatibility<String> snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( TypeSerializer<?>[] testNestedSerializers, String initialOuterConfiguration, String newOuterConfiguration) throws IOException { TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer(initialOuterConfiguration, testNestedSerializers); TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration(); DataOutputSerializer out = new DataOutputSerializer(128); TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot); DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot( in, Thread.currentThread().getContextClassLoader()); TestCompositeTypeSerializer newTestSerializer = new TestCompositeTypeSerializer(newOuterConfiguration, testNestedSerializers); return testSerializerSnapshot.resolveSchemaCompatibility(newTestSerializer); }
/** * Writes the given snapshot to the out stream. One should always use this method to write * snapshots out, rather than directly calling {@link #writeSnapshot(DataOutputView)}. * * <p>The snapshot written with this method can be read via {@link #readVersionedSnapshot(DataInputView, ClassLoader)}. */ static void writeVersionedSnapshot(DataOutputView out, TypeSerializerSnapshot<?> snapshot) throws IOException { out.writeUTF(snapshot.getClass().getName()); out.writeInt(snapshot.getCurrentVersion()); snapshot.writeSnapshot(out); }
@Test public void testSnapshotConfigurationAndReconfigure() throws Exception { final TypeSerializer<T> serializer = getSerializer(); final TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot, serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<T> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), getSerializer()); } TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer()); assertTrue(strategy.isCompatibleAsIs()); TypeSerializer<T> restoreSerializer = restoredConfig.restoreSerializer(); assertEquals(serializer.getClass(), restoreSerializer.getClass()); }
@Test public void testSerializerDeserializationFailure() throws Exception { TestSerializer ser = new TestSerializer(); TypeSerializerConfigSnapshot<Object> snap = (TypeSerializerConfigSnapshot<Object>) ser.snapshotConfiguration(); snap.setPriorSerializer(ser); DataOutputSerializer out = new DataOutputSerializer(64); TypeSerializerSnapshot.writeVersionedSnapshot(out, snap); TypeSerializerSnapshot<Object> readBack = TypeSerializerSnapshot.readVersionedSnapshot( new DataInputDeserializer(out.getCopyOfBuffer()), getClass().getClassLoader()); assertNotNull(readBack); try { readBack.restoreSerializer(); fail("expected exception"); } catch (IllegalStateException e) { // expected } ((TypeSerializerConfigSnapshot<Object>) readBack).setPriorSerializer( new UnloadableDummyTypeSerializer<>(new byte[0])); try { readBack.restoreSerializer(); fail("expected exception"); } catch (IllegalStateException e) { // expected } }
/** * Binary format layout of a written serializer snapshot is as follows: * * <ul> * <li>1. Format version of this util.</li> * <li>2. Name of the TypeSerializerSnapshot class.</li> * <li>3. The version of the TypeSerializerSnapshot's binary format.</li> * <li>4. The actual serializer snapshot data.</li> * </ul> */ @SuppressWarnings("deprecation") @Override public void write(DataOutputView out) throws IOException { setSerializerForWriteIfOldPath(serializerSnapshot, serializer); // write the format version of this utils format super.write(out); TypeSerializerSnapshot.writeVersionedSnapshot(out, serializerSnapshot); }
/** * Deserialization path for Flink versions 1.7+. */ @VisibleForTesting static <T> TypeSerializerSnapshot<T> deserializeV2(DataInputView in, ClassLoader cl) throws IOException { return TypeSerializerSnapshot.readVersionedSnapshot(in, cl); }
/** * Serialize an (avro)TypeSerializerSnapshot and deserialize it. */ private static <T> AvroSerializerSnapshot<T> roundTrip(TypeSerializerSnapshot<T> original) throws IOException { // writeSnapshot(); DataOutputSerializer out = new DataOutputSerializer(1024); original.writeSnapshot(out); // init AvroSerializerSnapshot<T> restored = new AvroSerializerSnapshot<>(); // readSnapshot(); DataInputView in = new DataInputDeserializer(out.wrapAsByteBuffer()); restored.readSnapshot(restored.getCurrentVersion(), in, original.getClass().getClassLoader()); return restored; }
@Test public void testRestoreCompositeTypeSerializer() throws IOException { // the target compatibilities of the nested serializers doesn't matter, // because we're only testing the restore serializer TypeSerializer<?>[] testNestedSerializers = { new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), new NestedSerializer(TargetCompatibility.INCOMPATIBLE), new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION) }; TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer("outer-config", testNestedSerializers); TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration(); DataOutputSerializer out = new DataOutputSerializer(128); TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot); DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot( in, Thread.currentThread().getContextClassLoader()); // now, restore the composite type serializer; // the restored nested serializer should be a RestoredNestedSerializer testSerializer = (TestCompositeTypeSerializer) testSerializerSnapshot.restoreSerializer(); Assert.assertTrue(testSerializer.getNestedSerializers()[0].getClass() == RestoredNestedSerializer.class); Assert.assertTrue(testSerializer.getNestedSerializers()[1].getClass() == RestoredNestedSerializer.class); Assert.assertTrue(testSerializer.getNestedSerializers()[2].getClass() == RestoredNestedSerializer.class); }
/** * Writes the composite snapshot of all the contained serializers. */ public final void writeNestedSerializerSnapshots(DataOutputView out) throws IOException { out.writeInt(MAGIC_NUMBER); out.writeInt(VERSION); out.writeInt(nestedSnapshots.length); for (TypeSerializerSnapshot<?> snap : nestedSnapshots) { TypeSerializerSnapshot.writeVersionedSnapshot(out, snap); } }
/** * Writes the given snapshot to the out stream. One should always use this method to write * snapshots out, rather than directly calling {@link #writeSnapshot(DataOutputView)}. * * <p>The snapshot written with this method can be read via {@link #readVersionedSnapshot(DataInputView, ClassLoader)}. */ static void writeVersionedSnapshot(DataOutputView out, TypeSerializerSnapshot<?> snapshot) throws IOException { out.writeUTF(snapshot.getClass().getName()); out.writeInt(snapshot.getCurrentVersion()); snapshot.writeSnapshot(out); }
/** * Reads the composite snapshot of all the contained serializers. */ public static NestedSerializersSnapshotDelegate readNestedSerializerSnapshots(DataInputView in, ClassLoader cl) throws IOException { final int magicNumber = in.readInt(); if (magicNumber != MAGIC_NUMBER) { throw new IOException(String.format("Corrupt data, magic number mismatch. Expected %8x, found %8x", MAGIC_NUMBER, magicNumber)); } final int version = in.readInt(); if (version != VERSION) { throw new IOException("Unrecognized version: " + version); } final int numSnapshots = in.readInt(); final TypeSerializerSnapshot<?>[] nestedSnapshots = new TypeSerializerSnapshot<?>[numSnapshots]; for (int i = 0; i < numSnapshots; i++) { nestedSnapshots[i] = TypeSerializerSnapshot.readVersionedSnapshot(in, cl); } return new NestedSerializersSnapshotDelegate(nestedSnapshots); }
@SuppressWarnings("unchecked") private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility( TypeSerializer<?> serializer, TypeSerializerSnapshot<?> snapshot) { TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer; TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot; return typedSnapshot.resolveSchemaCompatibility(typedSerializer); } }
@SuppressWarnings("deprecation") @Test public void movingForward() throws IOException { TypeSerializerSnapshot<ElementT> previousSnapshot = snapshotUnderTest(); TypeSerializer<ElementT> restoredSerializer = previousSnapshot.restoreSerializer(); TypeSerializerSnapshot<ElementT> nextSnapshot = restoredSerializer.snapshotConfiguration(); assertThat(nextSnapshot, instanceOf(testSpecification.snapshotClass)); TypeSerializerSnapshot<ElementT> nextSnapshotDeserialized = writeAndThenReadTheSnapshot(restoredSerializer, nextSnapshot); assertThat(nextSnapshotDeserialized, allOf( notNullValue(), not(instanceOf(TypeSerializerConfigSnapshot.class)) )); }
@Test public void testSerializeConfigWhenSerializerMissing() throws Exception { TestSerializer ser = new TestSerializer(); TypeSerializerConfigSnapshot<Object> snap = (TypeSerializerConfigSnapshot<Object>) ser.snapshotConfiguration(); try { TypeSerializerSnapshot.writeVersionedSnapshot(new DataOutputSerializer(64), snap); fail("exception expected"); } catch (IllegalStateException e) { // expected } }
/** * Deserialization path for Flink versions 1.7+. */ @VisibleForTesting static <T> TypeSerializerSnapshot<T> deserializeV2(DataInputView in, ClassLoader cl) throws IOException { return TypeSerializerSnapshot.readVersionedSnapshot(in, cl); }
public static <T> CompatibilityResult<T> resolveCompatibilityResult( TypeSerializerSnapshot<T> precedingSerializerConfigSnapshot, TypeSerializer<T> newSerializer) { TypeSerializerSchemaCompatibility<T> compatibility = precedingSerializerConfigSnapshot.resolveSchemaCompatibility(newSerializer); // everything except "compatible" maps to "requires migration". // at the entry point of the new-to-old-bridge (in the TypeSerializerConfigSnapshot), we // interpret "requiresMigration" as 'incompatible'. That is a precaution because // serializers could previously not specify the 'incompatible' case. return compatibility.isCompatibleAsIs() ? CompatibilityResult.compatible() : CompatibilityResult.requiresMigration(); } }
@Test public void restoredSerializerIsAbleToDeserializePreviousData() throws IOException { TypeSerializerSnapshot<ElementT> snapshot = snapshotUnderTest(); TypeSerializer<ElementT> serializer = snapshot.restoreSerializer(); DataInputView input = dataUnderTest(); final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher; for (int i = 0; i < testSpecification.testDataCount; i++) { final ElementT result = serializer.deserialize(input); assertThat(result, matcher); } }
/** * Binary format layout of a written serializer snapshot is as follows: * * <ul> * <li>1. Format version of this util.</li> * <li>2. Name of the TypeSerializerSnapshot class.</li> * <li>3. The version of the TypeSerializerSnapshot's binary format.</li> * <li>4. The actual serializer snapshot data.</li> * </ul> */ @SuppressWarnings("deprecation") @Override public void write(DataOutputView out) throws IOException { setSerializerForWriteIfOldPath(serializerSnapshot, serializer); // write the format version of this utils format super.write(out); TypeSerializerSnapshot.writeVersionedSnapshot(out, serializerSnapshot); }