/** * Utility method to conjure up a new scope for the generic parameters. */ @SuppressWarnings("unchecked") private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility( TypeSerializer<?> serializer, TypeSerializerSnapshot<?> snapshot) { TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer; TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot; return typedSnapshot.resolveSchemaCompatibility(typedSerializer); }
@SuppressWarnings("unchecked") private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility( TypeSerializer<?> serializer, TypeSerializerSnapshot<?> snapshot) { TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer; TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot; return typedSnapshot.resolveSchemaCompatibility(typedSerializer); } }
public static <T> CompatibilityResult<T> resolveCompatibilityResult( TypeSerializerSnapshot<T> precedingSerializerConfigSnapshot, TypeSerializer<T> newSerializer) { TypeSerializerSchemaCompatibility<T> compatibility = precedingSerializerConfigSnapshot.resolveSchemaCompatibility(newSerializer); // everything except "compatible" maps to "requires migration". // at the entry point of the new-to-old-bridge (in the TypeSerializerConfigSnapshot), we // interpret "requiresMigration" as 'incompatible'. That is a precaution because // serializers could previously not specify the 'incompatible' case. return compatibility.isCompatibleAsIs() ? CompatibilityResult.compatible() : CompatibilityResult.requiresMigration(); } }
@Test public void specifiedNewSerializerHasExpectedCompatibilityResultsWithSnapshot() { TypeSerializerSnapshot<ElementT> snapshot = snapshotUnderTest(); TypeSerializerSchemaCompatibility<ElementT> result = snapshot.resolveSchemaCompatibility(testSpecification.createSerializer()); assertThat(result, hasSameCompatibilityType(testSpecification.expectedCompatibilityResult)); }
@Test public void anAvroSnapshotIsCompatibleWithItsOriginatingSerializer() { AvroSerializer<GenericRecord> serializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL); TypeSerializerSnapshot<GenericRecord> snapshot = serializer.snapshotConfiguration(); assertThat(snapshot.resolveSchemaCompatibility(serializer), isCompatibleAsIs()); }
@Test public void validSchemaEvaluationShouldResultInCRequiresMigration() { final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_NAME); final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL); TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration(); assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isCompatibleAfterMigration()); }
@Test public void testMigrationStrategyForRemovedAvroDependency() throws Exception { KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig()); // read configuration again from bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot; try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClass> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA); assertTrue(compatResult.isCompatibleAsIs()); }
@Test public void nonValidSchemaEvaluationShouldResultInCompatibleSerializers() { final AvroSerializer<GenericRecord> originalSerializer = new AvroSerializer<>(GenericRecord.class, FIRST_REQUIRED_LAST_OPTIONAL); final AvroSerializer<GenericRecord> newSerializer = new AvroSerializer<>(GenericRecord.class, BOTH_REQUIRED); TypeSerializerSnapshot<GenericRecord> originalSnapshot = originalSerializer.snapshotConfiguration(); assertThat(originalSnapshot.resolveSchemaCompatibility(newSerializer), isIncompatible()); }
@Test @SuppressWarnings({"unchecked", "rawtypes"}) public void changingFromGenericToSpecificWithCompatibleSchemaShouldResultInCompatibleSerializers() { // starting with a generic serializer AvroSerializer<Object> generic = new AvroSerializer(GenericRecord.class, User.SCHEMA$); TypeSerializerSnapshot<Object> genericSnapshot = generic.snapshotConfiguration(); // then upgrading to a specific serializer AvroSerializer<Object> specificSerializer = new AvroSerializer(User.class); specificSerializer.snapshotConfiguration(); assertThat(genericSnapshot.resolveSchemaCompatibility(specificSerializer), isCompatibleAsIs()); }
@Test public void testSnapshotConfigurationAndReconfigure() throws Exception { final TypeSerializer<T> serializer = getSerializer(); final TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot, serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<T> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), getSerializer()); } TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer()); assertTrue(strategy.isCompatibleAsIs()); TypeSerializer<T> restoreSerializer = restoredConfig.restoreSerializer(); assertEquals(serializer.getClass(), restoreSerializer.getClass()); }
@Test public void testConfigurationSnapshotSerialization() throws Exception { EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration(), serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<PublicEnum> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializer); } TypeSerializerSchemaCompatibility<PublicEnum> compatResult = restoredConfig.resolveSchemaCompatibility(serializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues())); }
private TypeSerializerSchemaCompatibility<String> snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( TypeSerializer<?>[] testNestedSerializers, String initialOuterConfiguration, String newOuterConfiguration) throws IOException { TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer(initialOuterConfiguration, testNestedSerializers); TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration(); DataOutputSerializer out = new DataOutputSerializer(128); TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot); DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot( in, Thread.currentThread().getContextClassLoader()); TestCompositeTypeSerializer newTestSerializer = new TestCompositeTypeSerializer(newOuterConfiguration, testNestedSerializers); return testSerializerSnapshot.resolveSchemaCompatibility(newTestSerializer); }
return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
/** * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed. */ @Test public void testMigrationStrategyWithDifferentKryoType() throws Exception { KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA); serializedConfig = out.toByteArray(); } KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig()); // read configuration again from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClassB> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB); assertTrue(compatResult.isIncompatible()); }
/** * Verifies that reconfiguring with a config snapshot of a preceding POJO serializer * with different POJO type will result in INCOMPATIBLE. */ @Test public void testReconfigureWithDifferentPojoType() throws Exception { PojoSerializer<SubTestUserClassB> pojoSerializer1 = (PojoSerializer<SubTestUserClassB>) TypeExtractor.getForClass(SubTestUserClassB.class).createSerializer(new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer1); serializedConfig = out.toByteArray(); } PojoSerializer<SubTestUserClassA> pojoSerializer2 = (PojoSerializer<SubTestUserClassA>) TypeExtractor.getForClass(SubTestUserClassA.class).createSerializer(new ExecutionConfig()); // read configuration again from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer2); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<SubTestUserClassA> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer2); assertTrue(compatResult.isIncompatible()); }
pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleAsIs());
pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
/** * Utility method to conjure up a new scope for the generic parameters. */ @SuppressWarnings("unchecked") private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility( TypeSerializer<?> serializer, TypeSerializerSnapshot<?> snapshot) { TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer; TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot; return typedSnapshot.resolveSchemaCompatibility(typedSerializer); }