@Override public KryoSerializer<T> duplicate() { return new KryoSerializer<>(this); }
@Override protected <T> TypeSerializer<T> createSerializer(Class<T> type) { return new KryoSerializer<T>(type, ec); }
@Override @PublicEvolving public TypeSerializer<T> createSerializer(ExecutionConfig config) { if (config.hasGenericTypesDisabled()) { throw new UnsupportedOperationException( "Generic types have been disabled in the ExecutionConfig and type " + this.typeClass.getName() + " is treated as a generic type."); } return new KryoSerializer<T>(this.typeClass, config); }
@Override protected <T> TypeSerializer<T> createComponentSerializer(Class<T> type) { return new KryoSerializer<T>(type, new ExecutionConfig()); } }
@Override protected <T> TypeSerializer<T> createSerializer(Class<T> type) { return new KryoSerializer<T>(type, new ExecutionConfig()); } }
public ContentDumpSinkFunction() { super( new KryoSerializer<>(ContentTransaction.class, new ExecutionConfig()), VoidSerializer.INSTANCE, clock); }
@Override protected TypeSerializer<Object> createSerializer() { return new KryoSerializer<>(Object.class, new ExecutionConfig()); }
@Override @PublicEvolving @SuppressWarnings("unchecked") public TypeSerializer<T> createSerializer(ExecutionConfig config) { if (config.isForceKryoEnabled()) { return new KryoSerializer<>(getTypeClass(), config); } if (config.isForceAvroEnabled()) { return AvroUtils.getAvroUtils().createAvroSerializer(getTypeClass()); } return createPojoSerializer(config); }
@Test public void testMigrationStrategyForRemovedAvroDependency() throws Exception { KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig()); // read configuration again from bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot; try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClass> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA); assertTrue(compatResult.isCompatibleAsIs()); }
private void runDuplicateSerializerTest(ExecutionConfig executionConfig) { final KryoSerializer<WrappedString> original = new KryoSerializer<>(WrappedString.class, executionConfig); final KryoSerializer<WrappedString> duplicate = original.duplicate(); WrappedString testString = new WrappedString("test"); String copyWithOriginal = original.copy(testString).content; String copyWithDuplicate = duplicate.copy(testString).content; Assert.assertTrue(copyWithOriginal.startsWith(testString.content)); Assert.assertTrue(copyWithDuplicate.startsWith(testString.content)); // check that both serializer instances have appended a different identity hash Assert.assertNotEquals(copyWithOriginal, copyWithDuplicate); }
/** * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed. */ @Test public void testMigrationStrategyWithDifferentKryoType() throws Exception { KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA); serializedConfig = out.toByteArray(); } KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig()); // read configuration again from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClassB> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB); assertTrue(compatResult.isIncompatible()); }
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); setupEnvironment(env, pt); KeyedStream<Event, Integer> source = env.addSource(createEventSource(pt)) .name("EventSource") .uid("EventSource") .assignTimestampsAndWatermarks(createTimestampExtractor(pt)) .keyBy(Event::getKey); List<TypeSerializer<ComplexPayload>> stateSer = Collections.singletonList(new KryoSerializer<>(ComplexPayload.class, env.getConfig())); KeyedStream<Event, Integer> afterStatefulOperations = isOriginalJobVariant(pt) ? applyOriginalStatefulOperations(source, stateSer, Collections.emptyList()) : applyUpgradedStatefulOperations(source, stateSer, Collections.emptyList()); afterStatefulOperations .flatMap(createSemanticsCheckMapper(pt)) .name("SemanticsCheckMapper") .addSink(new PrintSinkFunction<>()); env.execute("General purpose test job"); }
@Test public void validateReferenceMappingEnabled() { KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); Kryo kryo = serializer.getKryo(); assertTrue(kryo.getReferences()); } }
/** * FLINK-6775, tests that the returned serializer is duplicated. * This allows to share the state descriptor across threads. */ @Test public void testSerializerDuplication() throws Exception { // we need a serializer that actually duplicates for testing (a stateful one) // we use Kryo here, because it meets these conditions TypeSerializer<String> statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); TestStateDescriptor<String> descr = new TestStateDescriptor<>("foobar", statefulSerializer); TypeSerializer<String> serializerA = descr.getSerializer(); TypeSerializer<String> serializerB = descr.getSerializer(); // check that the retrieved serializers are not the same assertNotSame(serializerA, serializerB); }
/** * FLINK-6775. * * <p>Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @Test public void testSerializerDuplication() { // we need a serializer that actually duplicates for testing (a stateful one) // we use Kryo here, because it meets these conditions TypeSerializer<String> statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); ListStateDescriptor<String> descr = new ListStateDescriptor<>("foobar", statefulSerializer); TypeSerializer<String> serializerA = descr.getElementSerializer(); TypeSerializer<String> serializerB = descr.getElementSerializer(); // check that the retrieved serializers are not the same assertNotSame(serializerA, serializerB); TypeSerializer<List<String>> listSerializerA = descr.getSerializer(); TypeSerializer<List<String>> listSerializerB = descr.getSerializer(); assertNotSame(listSerializerA, listSerializerB); } }
@Test public void testTypeRegistration() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. Assert.assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); // register again and make sure classes are still registered ExecutionConfig conf2 = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf2, new HashSet<Class<?>>()); KryoSerializer<String> kryo2 = new KryoSerializer<>(String.class, conf); assertTrue(kryo2.getKryo().getRegistration(FromNested.class).getId() > 0); }
@Test public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception { // guard our test assumptions. assertEquals("broken test assumption", -1, new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo() .getRegistration(File.class).getId()); final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(File.class); final TestStateDescriptor<Path> original = new TestStateDescriptor<>("test", Path.class); TestStateDescriptor<Path> clone = CommonTestUtils.createCopySerializable(original); clone.initializeSerializerUnlessSet(config); // serialized one (later initialized) carries the registration assertTrue(((KryoSerializer<?>) clone.getSerializer()).getKryo() .getRegistration(File.class).getId() > 0); }
@Test public void testReducingStateDescriptor() throws Exception { ReduceFunction<String> reducer = (a, b) -> a; TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); ReducingStateDescriptor<String> descr = new ReducingStateDescriptor<>("testName", reducer, serializer); assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertEquals(serializer, descr.getSerializer()); assertEquals(reducer, descr.getReduceFunction()); ReducingStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr); assertEquals("testName", copy.getName()); assertNotNull(copy.getSerializer()); assertEquals(serializer, copy.getSerializer()); }
@Test public void testTypeRegistrationFromTypeInfo() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(new GenericTypeInfo<>(ClassWithNested.class), conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); } }
@Test public void testListStateDescriptor() throws Exception { TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); ListStateDescriptor<String> descr = new ListStateDescriptor<>("testName", serializer); assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertTrue(descr.getSerializer() instanceof ListSerializer); assertNotNull(descr.getElementSerializer()); assertEquals(serializer, descr.getElementSerializer()); ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr); assertEquals("testName", copy.getName()); assertNotNull(copy.getSerializer()); assertTrue(copy.getSerializer() instanceof ListSerializer); assertNotNull(copy.getElementSerializer()); assertEquals(serializer, copy.getElementSerializer()); }