/** * Returns generic type information for any Java object. The serialization logic will * use the general purpose serializer Kryo. * * <p>Generic types are black-boxes for Flink, but allow any object and null values in fields. * * <p>By default, serialization of this type is not very efficient. Please read the documentation * about how to improve efficiency (namely by pre-registering classes). * * @param genericClass any Java class */ public static <T> TypeInformation<T> GENERIC(Class<T> genericClass) { return new GenericTypeInfo<>(genericClass); }
@SuppressWarnings("unchecked") @Override @PublicEvolving public TypeComparator<T> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) { if (isKeyType()) { @SuppressWarnings("rawtypes") GenericTypeComparator comparator = new GenericTypeComparator(sortOrderAscending, createSerializer(executionConfig), this.typeClass); return (TypeComparator<T>) comparator; } throw new UnsupportedOperationException("Types that do not implement java.lang.Comparable cannot be used as keys."); }
public static void recursivelyRegisterType(TypeInformation<?> typeInfo, ExecutionConfig config, Set<Class<?>> alreadySeen) { if (typeInfo instanceof GenericTypeInfo) { GenericTypeInfo<?> genericTypeInfo = (GenericTypeInfo<?>) typeInfo; Serializers.recursivelyRegisterType(genericTypeInfo.getTypeClass(), config, alreadySeen); } else if (typeInfo instanceof CompositeType) { List<GenericTypeInfo<?>> genericTypesInComposite = new ArrayList<>(); getContainedGenericTypes((CompositeType<?>)typeInfo, genericTypesInComposite); for (GenericTypeInfo<?> gt : genericTypesInComposite) { Serializers.recursivelyRegisterType(gt.getTypeClass(), config, alreadySeen); } } else if (typeInfo instanceof ObjectArrayTypeInfo) { ObjectArrayTypeInfo<?, ?> objectArrayTypeInfo = (ObjectArrayTypeInfo<?, ?>) typeInfo; recursivelyRegisterType(objectArrayTypeInfo.getComponentInfo(), config, alreadySeen); } }
private static KeyedStateBackend<ByteBuffer> getKeyedStateBackend(int numberOfKeyGroups, KeyGroupRange keyGroupRange) { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), numberOfKeyGroups, keyGroupRange, new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey(ByteBuffer.wrap( CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "1"))); return keyedStateBackend; } catch (Exception e) { throw new RuntimeException(e); } }
private static KeyedStateBackend<ByteBuffer> getKeyedStateBackend( int numberOfKeyGroups, KeyGroupRange keyGroupRange) { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), numberOfKeyGroups, keyGroupRange, new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey( ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "1"))); return keyedStateBackend; } catch (Exception e) { throw new RuntimeException(e); } } }
@Override protected GenericTypeInfo<?>[] getTestData() { return new GenericTypeInfo<?>[] { new GenericTypeInfo<>(TestClass.class), new GenericTypeInfo<>(AlternativeClass.class) }; }
private static KeyedStateBackend<ByteBuffer> getKeyedStateBackend( int numberOfKeyGroups, KeyGroupRange keyGroupRange) { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), numberOfKeyGroups, keyGroupRange, new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey( ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "1"))); return keyedStateBackend; } catch (Exception e) { throw new RuntimeException(e); } } }
@SuppressWarnings("unchecked") @Override @PublicEvolving public TypeComparator<T> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) { if (isKeyType()) { @SuppressWarnings("rawtypes") GenericTypeComparator comparator = new GenericTypeComparator(sortOrderAscending, createSerializer(executionConfig), this.typeClass); return (TypeComparator<T>) comparator; } throw new UnsupportedOperationException("Types that do not implement java.lang.Comparable cannot be used as keys."); }
if (!(isClassType(type) && (clazz = typeToClass(type)).isAssignableFrom(((GenericTypeInfo<?>) typeInfo).getTypeClass()))) { throw new InvalidTypesException("Generic type '" + ((GenericTypeInfo<?>) typeInfo).getTypeClass().getCanonicalName() + "' or a subclass of it expected but was '" + clazz.getCanonicalName() + "'.");
private TypeInformation getFieldType(HCatFieldSchema fieldSchema) { switch(fieldSchema.getType()) { case INT: return BasicTypeInfo.INT_TYPE_INFO; case TINYINT: return BasicTypeInfo.BYTE_TYPE_INFO; case SMALLINT: return BasicTypeInfo.SHORT_TYPE_INFO; case BIGINT: return BasicTypeInfo.LONG_TYPE_INFO; case BOOLEAN: return BasicTypeInfo.BOOLEAN_TYPE_INFO; case FLOAT: return BasicTypeInfo.FLOAT_TYPE_INFO; case DOUBLE: return BasicTypeInfo.DOUBLE_TYPE_INFO; case STRING: return BasicTypeInfo.STRING_TYPE_INFO; case BINARY: return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO; case ARRAY: return new GenericTypeInfo(List.class); case MAP: return new GenericTypeInfo(Map.class); case STRUCT: return new GenericTypeInfo(List.class); default: throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered."); } }
@Override protected StateInternals createStateInternals() { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey( ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "Hello"))); return new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of()); } catch (Exception e) { throw new RuntimeException(e); } }
@SuppressWarnings("unchecked") @Override @PublicEvolving public TypeComparator<T> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) { if (isKeyType()) { @SuppressWarnings("rawtypes") GenericTypeComparator comparator = new GenericTypeComparator(sortOrderAscending, createSerializer(executionConfig), this.typeClass); return (TypeComparator<T>) comparator; } throw new UnsupportedOperationException("Types that do not implement java.lang.Comparable cannot be used as keys."); }
/** * Recursively converts extracted AvroTypeInfo into a RowTypeInfo nested structure with deterministic field order. * Replaces generic Utf8 with basic String type information. */ private static TypeInformation<?> convertToTypeInformation(TypeInformation<?> extracted, Schema schema) { if (schema.getType() == Schema.Type.RECORD) { final List<Schema.Field> fields = schema.getFields(); final AvroTypeInfo<?> avroTypeInfo = (AvroTypeInfo<?>) extracted; final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()]; final String[] names = new String[fields.size()]; for (int i = 0; i < fields.size(); i++) { final Schema.Field field = fields.get(i); types[i] = convertToTypeInformation(avroTypeInfo.getTypeAt(field.name()), field.schema()); names[i] = field.name(); } return new RowTypeInfo(types, names); } else if (extracted instanceof GenericTypeInfo<?>) { final GenericTypeInfo<?> genericTypeInfo = (GenericTypeInfo<?>) extracted; if (genericTypeInfo.getTypeClass() == Utf8.class) { return BasicTypeInfo.STRING_TYPE_INFO; } } return extracted; } }
"and must be processed as GenericType. Please read the Flink documentation " + "on \"Data Types & Serialization\" for details of the effect on performance."); return new GenericTypeInfo<OUT>(clazz); "and must be processed as GenericType. Please read the Flink documentation " + "on \"Data Types & Serialization\" for details of the effect on performance."); return new GenericTypeInfo<OUT>(clazz); genericClass = typeToClass(fieldType); pojoFields.add(new PojoField(field, new GenericTypeInfo<OUT>((Class<OUT>) genericClass)));
@Override protected StateInternals createStateInternals() { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey( ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "Hello"))); return new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of()); } catch (Exception e) { throw new RuntimeException(e); } } }
public static void recursivelyRegisterType(TypeInformation<?> typeInfo, ExecutionConfig config, Set<Class<?>> alreadySeen) { if (typeInfo instanceof GenericTypeInfo) { GenericTypeInfo<?> genericTypeInfo = (GenericTypeInfo<?>) typeInfo; Serializers.recursivelyRegisterType(genericTypeInfo.getTypeClass(), config, alreadySeen); } else if (typeInfo instanceof CompositeType) { List<GenericTypeInfo<?>> genericTypesInComposite = new ArrayList<>(); getContainedGenericTypes((CompositeType<?>)typeInfo, genericTypesInComposite); for (GenericTypeInfo<?> gt : genericTypesInComposite) { Serializers.recursivelyRegisterType(gt.getTypeClass(), config, alreadySeen); } } else if (typeInfo instanceof ObjectArrayTypeInfo) { ObjectArrayTypeInfo<?, ?> objectArrayTypeInfo = (ObjectArrayTypeInfo<?, ?>) typeInfo; recursivelyRegisterType(objectArrayTypeInfo.getComponentInfo(), config, alreadySeen); } }
@SuppressWarnings("unchecked") @Internal private static <T extends SpecificRecordBase> List<PojoField> generateFieldsFromAvroSchema(Class<T> typeClass) { PojoTypeExtractor pte = new PojoTypeExtractor(); ArrayList<Type> typeHierarchy = new ArrayList<>(); typeHierarchy.add(typeClass); TypeInformation ti = pte.analyzePojo(typeClass, typeHierarchy, null, null, null); if (!(ti instanceof PojoTypeInfo)) { throw new IllegalStateException("Expecting type to be a PojoTypeInfo"); } PojoTypeInfo pti = (PojoTypeInfo) ti; List<PojoField> newFields = new ArrayList<>(pti.getTotalFields()); for (int i = 0; i < pti.getArity(); i++) { PojoField f = pti.getPojoFieldAt(i); TypeInformation newType = f.getTypeInformation(); // check if type is a CharSequence if (newType instanceof GenericTypeInfo) { if ((newType).getTypeClass().equals(CharSequence.class)) { // replace the type by a org.apache.avro.util.Utf8 newType = new GenericTypeInfo(org.apache.avro.util.Utf8.class); } } PojoField newField = new PojoField(f.getField(), newType); newFields.add(newField); } return newFields; }
@Override protected StateInternals createStateInternals() { MemoryStateBackend backend = new MemoryStateBackend(); try { AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend( new DummyEnvironment("test", 1, 0), new JobID(), "test_op", new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()), 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); keyedStateBackend.setCurrentKey( ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "Hello"))); return new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of()); } catch (Exception e) { throw new RuntimeException(e); } } }
public static void recursivelyRegisterType(TypeInformation<?> typeInfo, ExecutionConfig config, Set<Class<?>> alreadySeen) { if (typeInfo instanceof GenericTypeInfo) { GenericTypeInfo<?> genericTypeInfo = (GenericTypeInfo<?>) typeInfo; Serializers.recursivelyRegisterType(genericTypeInfo.getTypeClass(), config, alreadySeen); } else if (typeInfo instanceof CompositeType) { List<GenericTypeInfo<?>> genericTypesInComposite = new ArrayList<>(); getContainedGenericTypes((CompositeType<?>)typeInfo, genericTypesInComposite); for (GenericTypeInfo<?> gt : genericTypesInComposite) { Serializers.recursivelyRegisterType(gt.getTypeClass(), config, alreadySeen); } } else if (typeInfo instanceof ObjectArrayTypeInfo) { ObjectArrayTypeInfo<?, ?> objectArrayTypeInfo = (ObjectArrayTypeInfo<?, ?>) typeInfo; recursivelyRegisterType(objectArrayTypeInfo.getComponentInfo(), config, alreadySeen); } }
@Test(expected = InvalidProgramException.class) public void testGenericNonKeyType() { // Fail: GenericType cannot be used as key TypeInformation<GenericNonKeyType> genericType = new GenericTypeInfo<>(GenericNonKeyType.class); new ExpressionKeys<>("*", genericType); }