@SuppressWarnings("unchecked") public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossFunction, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) { validateInputType(CrossFunction.class, crossFunction.getClass(), 0, in1Type); validateInputType(CrossFunction.class, crossFunction.getClass(), 1, in2Type); if(crossFunction instanceof ResultTypeQueryable) { return ((ResultTypeQueryable<OUT>) crossFunction).getProducedType(); } return createTypeInfo(CrossFunction.class, crossFunction.getClass(), 2, in1Type, in2Type); }
@Override public TypeInformation<E> getProducedType() { return TypeExtractor.getForClass(this.avroValueType); }
@SuppressWarnings("unchecked") public static <IN1, IN2, OUT> TypeInformation<OUT> createTypeInfo(Class<?> baseClass, Class<?> clazz, int returnParamPos, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) { ArrayList<Type> typeHierarchy = new ArrayList<Type>(); Type returnType = getParameterType(baseClass, typeHierarchy, clazz, returnParamPos); TypeInformation<OUT> typeInfo = null; // return type is a variable -> try to get the type info from the input of the base class directly if (returnType instanceof TypeVariable<?>) { ParameterizedType immediateBaseChild = (ParameterizedType) typeHierarchy.get(typeHierarchy.size() - 1); typeInfo = (TypeInformation<OUT>) createTypeInfoWithImmediateBaseChildInput(immediateBaseChild, (TypeVariable<?>) returnType, in1Type, in2Type); if (typeInfo != null) { return typeInfo; } } // get info from hierarchy return (TypeInformation<OUT>) createTypeInfoWithTypeHierarchy(typeHierarchy, returnType, in1Type, in2Type); }
Type varContent = materializeTypeVariable(typeHierarchy, (TypeVariable<?>) tupleChild.getActualTypeArguments()[i]); tupleSubTypes[i] = createTypeInfoWithImmediateBaseChildInput(immediateBaseChild, (TypeVariable<?>) subtypes[i], in1Type, in2Type); tupleSubTypes[i] = createTypeInfoWithTypeHierarchy(new ArrayList<Type>(typeHierarchy), subtypes[i], in1Type, in2Type); Type typeVar = materializeTypeVariable(typeHierarchy, (TypeVariable<?>) t); return createTypeInfoWithTypeHierarchy(typeHierarchy, typeVar, in1Type, in2Type); TypeInformation<OUT> typeInfo = (TypeInformation<OUT>) createTypeInfoWithImmediateBaseChildInput(immediateBaseChild, (TypeVariable<?>) t, in1Type, in2Type); if (typeInfo != null) { GenericArrayType genericArray = (GenericArrayType) t; TypeInformation<?> componentInfo = createTypeInfoWithTypeHierarchy(typeHierarchy, genericArray.getGenericComponentType(), in1Type, in2Type); return ObjectArrayTypeInfo.getInfoFor(t, componentInfo); return getForClass((Class<OUT>) ((ParameterizedType) t).getRawType()); return getForClass((Class<OUT>) t);
@SuppressWarnings("unchecked") private ObjectArrayTypeInfo(Type arrayType, Type componentType) { this.arrayType = arrayType; this.componentType = componentType; this.componentInfo = (TypeInformation<C>) TypeExtractor.createTypeInfo(componentType); }
@SuppressWarnings({ "unchecked", "rawtypes" }) public static <X> TypeInformation<X> getForObject(X value) { Validate.notNull(value); // check if we can extract the types from tuples, otherwise work with the class if (value instanceof Tuple) { Tuple t = (Tuple) value; int numFields = t.getArity(); TypeInformation<?>[] infos = new TypeInformation[numFields]; for (int i = 0; i < numFields; i++) { Object field = t.getField(i); if (field == null) { throw new InvalidTypesException("Automatic type extraction is not possible on candidates with null values. " + "Please specify the types directly."); } infos[i] = getForObject(field); } return (TypeInformation<X>) new TupleTypeInfo(value.getClass(), infos); } else { return getForClass((Class<X>) value.getClass()); } }
/** * Constructor for a non-grouped reduce (all reduce). * * @param input The input data set to the groupReduce function. * @param function The user-defined GroupReduce function. */ public ReduceGroupOperator(DataSet<IN> input, GroupReduceFunction<IN, OUT> function) { super(input, TypeExtractor.getGroupReduceReturnTypes(function, input.getType())); this.function = function; this.grouper = null; checkCombinability(); }
private static <IN1, IN2> TypeInformation<?> createTypeInfoWithImmediateBaseChildInput(ParameterizedType baseChild, TypeVariable<?> typeVar, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) { Type[] baseChildArgs = baseChild.getActualTypeArguments(); TypeInformation<?> info = null; if (in1Type != null) { info = findCorrespondingInfo(typeVar, baseChildArgs[0], in1Type); } if (info == null && in2Type != null) { info = findCorrespondingInfo(typeVar, baseChildArgs[1], in2Type); } if (info != null) { return info; } return null; }
/** * Creates a new data set that contains the given elements. The elements must all be of the same type, * for example, all of the {@link String} or {@link Integer}. The sequence of elements must not be empty. * Furthermore, the elements must be serializable (as defined in {@link java.io.Serializable}, because the * execution environment may ship the elements into the cluster. * <p> * The framework will try and determine the exact type from the collection elements. * In case of generic elements, it may be necessary to manually supply the type information * via {@link #fromCollection(Collection, TypeInformation)}. * <p> * Note that this operation will result in a non-parallel data source, i.e. a data source with * a degree of parallelism of one. * * @param data The elements to make up the data set. * @return A DataSet representing the given list of elements. */ public <X> DataSource<X> fromElements(X... data) { if (data == null) { throw new IllegalArgumentException("The data must not be null."); } if (data.length == 0) { throw new IllegalArgumentException("The number of elements must not be zero."); } return fromCollection(Arrays.asList(data), TypeExtractor.getForObject(data[0])); }
public static TypeInformation<?> createTypeInfo(Type t) { ArrayList<Type> typeHierarchy = new ArrayList<Type>(); typeHierarchy.add(t); return createTypeInfoWithTypeHierarchy(typeHierarchy, t, null, null); }
public FlatMapOperator(DataSet<IN> input, FlatMapFunction<IN, OUT> function) { super(input, TypeExtractor.getFlatMapReturnTypes(function, input.getType())); this.function = function; extractSemanticAnnotationsFromUdf(function.getClass()); }
/** * Generic method to create an input DataSet with in {@link InputFormat}. The DataSet will not be * immediately created - instead, this method returns a DataSet that will be lazily created from * the input format once the program is executed. * <p> * Since all data sets need specific information about their types, this method needs to determine * the type of the data produced by the input format. It will attempt to determine the data type * by reflection, unless the the input format implements the {@link ResultTypeQueryable} interface. * In the latter case, this method will invoke the {@link ResultTypeQueryable#getProducedType()} * method to determine data type produced by the input format. * * @param inputFormat The input format used to create the data set. * @return A DataSet that represents the data created by the input format. * * @see #createInput(InputFormat, TypeInformation) */ public <X> DataSource<X> createInput(InputFormat<X, ?> inputFormat) { if (inputFormat == null) { throw new IllegalArgumentException("InputFormat must not be null."); } try { return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat)); } catch (Exception e) { throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " + "Please specify the TypeInformation of the produced type explicitly."); } }
/** * Finalizes a Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.<br/> * Each CrossFunction call returns exactly one element. * * @param function The CrossFunction that is called for each pair of crossed elements. * @return An CrossOperator that represents the crossed result DataSet * * @see CrossFunction * @see DataSet */ public <R> CrossOperator<I1, I2, R> with(CrossFunction<I1, I2, R> function) { if (function == null) { throw new NullPointerException("Cross function must not be null."); } TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, input1.getType(), input2.getType()); return new CrossOperator<I1, I2, R>(input1, input2, function, returnType); }
private TypeInformation<Message> getMessageType(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf) { return TypeExtractor.createTypeInfo(MessagingFunction.class, mf.getClass(), 2, null, null); }
/** * Constructor for a grouped reduce. * * @param input The grouped input to be processed group-wise by the groupReduce function. * @param function The user-defined GroupReduce function. */ public ReduceGroupOperator(Grouping<IN> input, GroupReduceFunction<IN, OUT> function) { super(input != null ? input.getDataSet() : null, TypeExtractor.getGroupReduceReturnTypes(function, input.getDataSet().getType())); this.function = function; this.grouper = input; checkCombinability(); extractSemanticAnnotationsFromUdf(function.getClass()); }
private static TypeInformation<?> findCorrespondingInfo(TypeVariable<?> typeVar, Type type, TypeInformation<?> corrInfo) { if (type instanceof TypeVariable) { TypeVariable<?> variable = (TypeVariable<?>) type; if (variable.getName().equals(typeVar.getName()) && variable.getGenericDeclaration().equals(typeVar.getGenericDeclaration())) { return corrInfo; } } else if (type instanceof ParameterizedType && Tuple.class.isAssignableFrom((Class<?>) ((ParameterizedType) type).getRawType())) { ParameterizedType tuple = (ParameterizedType) type; Type[] args = tuple.getActualTypeArguments(); for (int i = 0; i < args.length; i++) { TypeInformation<?> info = findCorrespondingInfo(typeVar, args[i], ((TupleTypeInfo<?>) corrInfo).getTypeAt(i)); if (info != null) { return info; } } } return null; }
return fromCollection(data, TypeExtractor.getForObject(firstValue));
@SuppressWarnings("unchecked") public static <IN1, IN2, OUT> TypeInformation<OUT> getJoinReturnTypes(JoinFunction<IN1, IN2, OUT> joinFunction, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) { validateInputType(JoinFunction.class, joinFunction.getClass(), 0, in1Type); validateInputType(JoinFunction.class, joinFunction.getClass(), 1, in2Type); if(joinFunction instanceof ResultTypeQueryable) { return ((ResultTypeQueryable<OUT>) joinFunction).getProducedType(); } return createTypeInfo(JoinFunction.class, joinFunction.getClass(), 2, in1Type, in2Type); }
@SuppressWarnings("unchecked") public static <IN> TypeInformation<IN> getInputFormatTypes(InputFormat<IN, ?> inputFormat) { if(inputFormat instanceof ResultTypeQueryable) { return ((ResultTypeQueryable<IN>) inputFormat).getProducedType(); } return createTypeInfo(InputFormat.class, inputFormat.getClass(), 0, null, null); }
/** * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the * framework to create a parallel data source that returns the elements in the iterator. * The iterator must be serializable (as defined in {@link java.io.Serializable}, because the * execution environment may ship the elements into the cluster. * <p> * Because the iterator will remain unmodified until the actual execution happens, the type of data * returned by the iterator must be given explicitly in the form of the type class (this is due to the * fact that the Java compiler erases the generic type information). * * @param iterator The iterator that produces the elements of the data set. * @param type The class of the data produced by the iterator. Must not be a generic class. * @return A DataSet representing the elements in the iterator. * * @see #fromParallelCollection(SplittableIterator, TypeInformation) */ public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, Class<X> type) { return fromParallelCollection(iterator, TypeExtractor.getForClass(type)); }