@Override protected boolean matchesSafely(TypeSerializerSchemaCompatibility<T> testResultCompatibility) { if (expectedCompatibilty.isCompatibleAsIs()) { return testResultCompatibility.isCompatibleAsIs(); } else if (expectedCompatibilty.isIncompatible()) { return testResultCompatibility.isCompatibleAfterMigration(); } else if (expectedCompatibilty.isIncompatible()) { return testResultCompatibility.isIncompatible(); } else if (expectedCompatibilty.isCompatibleWithReconfiguredSerializer()) { return testResultCompatibility.isCompatibleWithReconfiguredSerializer(); } return false; }
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
if (outerCompatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); resolveCompatibility(newNestedSerializers[i], nestedSnapshots[i]); if (compatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible();
if (compatibilityResult.isIncompatible()) { throw new FlinkRuntimeException( new StateMigrationException("The new priority queue serializer must not be incompatible."));
/** * Check that removing enum fields requires migration */ @Test public void checkRemovedField() throws Exception { Assert.assertTrue(checkCompatibility(ENUM_A, ENUM_C).isIncompatible()); }
if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible.");
@Test public void testOuterSnapshotCompatibilityPrecedence() throws IOException { final String INIT_OUTER_CONFIG = "outer-config"; final String INCOMPAT_OUTER_CONFIG = "incompat-outer-config"; TypeSerializer<?>[] testNestedSerializers = { new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), }; TypeSerializerSchemaCompatibility<String> compatibility = snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( testNestedSerializers, INIT_OUTER_CONFIG, INCOMPAT_OUTER_CONFIG); // even though nested serializers are compatible, incompatibility of the outer // snapshot should have higher precedence in the final result Assert.assertTrue(compatibility.isIncompatible()); }
if (compatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible();
/** * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed. */ @Test public void testMigrationStrategyWithDifferentKryoType() throws Exception { KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA); serializedConfig = out.toByteArray(); } KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig()); // read configuration again from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClassB> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB); assertTrue(compatResult.isIncompatible()); }
/** * Verifies that reconfiguring with a config snapshot of a preceding POJO serializer * with different POJO type will result in INCOMPATIBLE. */ @Test public void testReconfigureWithDifferentPojoType() throws Exception { PojoSerializer<SubTestUserClassB> pojoSerializer1 = (PojoSerializer<SubTestUserClassB>) TypeExtractor.getForClass(SubTestUserClassB.class).createSerializer(new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer1); serializedConfig = out.toByteArray(); } PojoSerializer<SubTestUserClassA> pojoSerializer2 = (PojoSerializer<SubTestUserClassA>) TypeExtractor.getForClass(SubTestUserClassA.class).createSerializer(new ExecutionConfig()); // read configuration again from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer2); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<SubTestUserClassA> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer2); assertTrue(compatResult.isIncompatible()); }
@Test public void testIncompatiblePrecedence() throws IOException { final String OUTER_CONFIG = "outer-config"; final TypeSerializer<?>[] testNestedSerializers = { new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION), new NestedSerializer(TargetCompatibility.INCOMPATIBLE), new NestedSerializer(TargetCompatibility.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER) }; TypeSerializerSchemaCompatibility<String> compatibility = snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( testNestedSerializers, OUTER_CONFIG, OUTER_CONFIG); Assert.assertTrue(compatibility.isIncompatible()); }
@Test public void testBridgeCompatibilityCheck() throws Exception { TestSerializerConfigSnapshot snap = new TestSerializerConfigSnapshot(); TestSerializer serCompat = new TestSerializer(true); TypeSerializerSchemaCompatibility<Object> resultCompat = snap.resolveSchemaCompatibility(serCompat); assertTrue(resultCompat.isCompatibleAsIs()); TestSerializer serIncompat = new TestSerializer(false); TypeSerializerSchemaCompatibility<Object> resultIncompat = snap.resolveSchemaCompatibility(serIncompat); assertTrue(resultIncompat.isIncompatible()); }
if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible.");
@Nonnull public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { this.isRegisteredWithIncompatibleSerializer = true; } this.registeredSerializer = newSerializer; return result; }
@Nonnull public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { this.isRegisteredWithIncompatibleSerializer = true; } this.registeredSerializer = newSerializer; return result; }
@SuppressWarnings("unchecked") @Nonnull @Override public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> create( @Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) { final HeapPriorityQueueSnapshotRestoreWrapper existingState = registeredPQStates.get(stateName); if (existingState != null) { // TODO we implement the simple way of supporting the current functionality, mimicking keyed state // because this should be reworked in FLINK-9376 and then we should have a common algorithm over // StateMetaInfoSnapshot that avoids this code duplication. TypeSerializerSchemaCompatibility<T> compatibilityResult = existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer); if (compatibilityResult.isIncompatible()) { throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible.")); } else { registeredPQStates.put( stateName, existingState.forUpdatedSerializer(byteOrderedElementSerializer)); } return existingState.getPriorityQueue(); } else { final RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo = new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer); return createInternal(metaInfo); } }
@SuppressWarnings("unchecked") @Nonnull @Override public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> create( @Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) { final HeapPriorityQueueSnapshotRestoreWrapper existingState = registeredPQStates.get(stateName); if (existingState != null) { // TODO we implement the simple way of supporting the current functionality, mimicking keyed state // because this should be reworked in FLINK-9376 and then we should have a common algorithm over // StateMetaInfoSnapshot that avoids this code duplication. TypeSerializerSchemaCompatibility<T> compatibilityResult = existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer); if (compatibilityResult.isIncompatible()) { throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible.")); } else { registeredPQStates.put( stateName, existingState.forUpdatedSerializer(byteOrderedElementSerializer)); } return existingState.getPriorityQueue(); } else { final RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo = new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer); return createInternal(metaInfo); } }
/** * Resolves the compatibility of the nested serializer snapshots with the nested * serializers of the new outer serializer. */ public <T> TypeSerializerSchemaCompatibility<T> resolveCompatibilityWithNested( TypeSerializerSchemaCompatibility<?> outerCompatibility, TypeSerializer<?>... newNestedSerializers) { checkArgument(newNestedSerializers.length == nestedSnapshots.length, "Different number of new serializers and existing serializer configuration snapshots"); // compatibility of the outer serializer's format if (outerCompatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); } // check nested serializers for compatibility boolean nestedSerializerRequiresMigration = false; for (int i = 0; i < nestedSnapshots.length; i++) { TypeSerializerSchemaCompatibility<?> compatibility = resolveCompatibility(newNestedSerializers[i], nestedSnapshots[i]); if (compatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); } if (compatibility.isCompatibleAfterMigration()) { nestedSerializerRequiresMigration = true; } } return (nestedSerializerRequiresMigration || !outerCompatibility.isCompatibleAsIs()) ? TypeSerializerSchemaCompatibility.compatibleAfterMigration() : TypeSerializerSchemaCompatibility.compatibleAsIs(); }
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer, @Nullable StateSnapshotTransformer<SV> snapshotTransformer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (!s.isCompatibleAsIs()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer, @Nullable StateSnapshotTransformer<SV> snapshotTransformer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (!s.isCompatibleAsIs()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }