private TypeInformation<?>[] extractFieldTypes(int[] fields, Class<?>[] givenTypes, TypeInformation<?> inType) { TupleTypeInfo<?> inTupleType = (TupleTypeInfo<?>) inType; TypeInformation<?>[] fieldTypes = new TypeInformation[fields.length]; for(int i=0; i<fields.length; i++) { if(inTupleType.getTypeAt(fields[i]).getTypeClass() != givenTypes[i]) { throw new IllegalArgumentException("Given types do not match types of input data set."); } fieldTypes[i] = inTupleType.getTypeAt(fields[i]); } return fieldTypes; }
private static TypeInformation<?> findCorrespondingInfo(TypeVariable<?> typeVar, Type type, TypeInformation<?> corrInfo) { if (type instanceof TypeVariable) { TypeVariable<?> variable = (TypeVariable<?>) type; if (variable.getName().equals(typeVar.getName()) && variable.getGenericDeclaration().equals(typeVar.getGenericDeclaration())) { return corrInfo; } } else if (type instanceof ParameterizedType && Tuple.class.isAssignableFrom((Class<?>) ((ParameterizedType) type).getRawType())) { ParameterizedType tuple = (ParameterizedType) type; Type[] args = tuple.getActualTypeArguments(); for (int i = 0; i < args.length; i++) { TypeInformation<?> info = findCorrespondingInfo(typeVar, args[i], ((TupleTypeInfo<?>) corrInfo).getTypeAt(i)); if (info != null) { return info; } } } return null; }
public FieldPositionKeys(int[] groupingFields, TypeInformation<T> type, boolean allowEmpty) { if (!type.isTupleType()) { throw new InvalidProgramException("Specifying keys via field positions is only valid for tuple data types"); } if (!allowEmpty && (groupingFields == null || groupingFields.length == 0)) { throw new IllegalArgumentException("The grouping fields must not be empty."); } TupleTypeInfo<?> tupleType = (TupleTypeInfo<?>)type; this.groupingFields = makeFields(groupingFields, (TupleTypeInfo<?>) type); types = new TypeInformation[this.groupingFields.length]; for(int i = 0; i < this.groupingFields.length; i++) { types[i] = tupleType.getTypeAt(this.groupingFields[i]); } }
private VertexCentricIteration(VertexUpdateFunction<VertexKey, VertexValue, Message> uf, MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf, DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue, int maximumNumberOfIterations) { Validate.notNull(uf); Validate.notNull(mf); Validate.notNull(edgesWithoutValue); Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one."); // check that the edges are actually a valid tuple set of vertex key types TypeInformation<Tuple2<VertexKey, VertexKey>> edgesType = edgesWithoutValue.getType(); Validate.isTrue(edgesType.isTupleType() && edgesType.getArity() == 2, "The edges data set (for edges without edge values) must consist of 2-tuples."); TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) edgesType; Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1)) && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()), "Both tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface."); this.updateFunction = uf; this.messagingFunction = mf; this.edgesWithoutValue = edgesWithoutValue; this.edgesWithValue = null; this.maximumNumberOfIterations = maximumNumberOfIterations; this.aggregators = new HashMap<String, Class<? extends Aggregator<?>>>(); this.messageType = getMessageType(mf); }
Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1)) && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()), "The first two tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface.");
public AggregateOperator<IN> and(Aggregations function, int field) { Validate.notNull(function); TupleTypeInfo<?> inType = (TupleTypeInfo<?>) getType(); if (field < 0 || field >= inType.getArity()) { throw new IllegalArgumentException("Aggregation field position is out of range."); } AggregationFunctionFactory factory = function.getFactory(); AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass()); this.aggregationFunctions.add(aggFunct); this.fields.add(field); return this; }
/** * Sets the input data set for this operator. In the case of this operator this input data set represents * the set of vertices with their initial state. * * @param inputData The input data set, which in the case of this operator represents the set of * vertices with their initial state. * * @see eu.stratosphere.api.java.operators.CustomUnaryOperation#setInput(eu.stratosphere.api.java.DataSet) */ @Override public void setInput(DataSet<Tuple2<VertexKey, VertexValue>> inputData) { // sanity check that we really have two tuples TypeInformation<Tuple2<VertexKey, VertexValue>> inputType = inputData.getType(); Validate.isTrue(inputType.isTupleType() && inputType.getArity() == 2, "The input data set (the initial vertices) must consist of 2-tuples."); // check that the key type here is the same as for the edges TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) inputType).getTypeAt(0); TypeInformation<?> edgeType = edgesWithoutValue != null ? edgesWithoutValue.getType() : edgesWithValue.getType(); TypeInformation<VertexKey> edgeKeyType = ((TupleTypeInfo<?>) edgeType).getTypeAt(0); Validate.isTrue(keyType.equals(edgeKeyType), "The first tuple field (the vertex id) of the input data set (the initial vertices) " + "must be the same data type as the first fields of the edge data set (the source vertex id). " + "Here, the key type for the vertex ids is '%s' and the key type for the edges is '%s'.", keyType, edgeKeyType); this.initialVertices = inputData; }
private TypeInformation<?>[] extractFieldTypes(int[] fields, Class<?>[] givenTypes) { TypeInformation<?>[] fieldTypes = new TypeInformation[fields.length]; for(int i=0; i<fields.length; i++) { TypeInformation<?> typeInfo; if(isFieldInFirst[i]) { if(fields[i] >= 0) { typeInfo = ((TupleTypeInfo<?>)ds1.getType()).getTypeAt(fields[i]); } else { typeInfo = ds1.getType(); } } else { if(fields[i] >= 0) { typeInfo = ((TupleTypeInfo<?>)ds2.getType()).getTypeAt(fields[i]); } else { typeInfo = ds2.getType(); } } if(typeInfo.getTypeClass() != givenTypes[i]) { throw new IllegalArgumentException("Given types do not match types of input data set."); } fieldTypes[i] = typeInfo; } return fieldTypes; }
private TypeInformation<?>[] extractFieldTypes(int[] fields, Class<?>[] givenTypes) { TypeInformation<?>[] fieldTypes = new TypeInformation[fields.length]; for(int i=0; i<fields.length; i++) { TypeInformation<?> typeInfo; if(isFieldInFirst[i]) { if(fields[i] >= 0) { typeInfo = ((TupleTypeInfo<?>)ds1.getType()).getTypeAt(fields[i]); } else { typeInfo = ds1.getType(); } } else { if(fields[i] >= 0) { typeInfo = ((TupleTypeInfo<?>)ds2.getType()).getTypeAt(fields[i]); } else { typeInfo = ds2.getType(); } } if(typeInfo.getTypeClass() != givenTypes[i]) { throw new IllegalArgumentException("Given types do not match types of input data set."); } fieldTypes[i] = typeInfo; } return fieldTypes; } }
/** * <p> * Non grouped aggregation */ public AggregateOperator(DataSet<IN> input, Aggregations function, int field) { super(Validate.notNull(input), input.getType()); Validate.notNull(function); if (!input.getType().isTupleType()) { throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types."); } TupleTypeInfo<?> inType = (TupleTypeInfo<?>) input.getType(); if (field < 0 || field >= inType.getArity()) { throw new IllegalArgumentException("Aggregation field position is out of range."); } AggregationFunctionFactory factory = function.getFactory(); AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass()); // this is the first aggregation operator after a regular data set (non grouped aggregation) this.aggregationFunctions.add(aggFunct); this.fields.add(field); this.grouping = null; }
/** * Configures the reader to read the CSV data and parse it to the given type. The type must be a subclass of * {@link Tuple}. The type information for the fields is obtained from the type class. The type * consequently needs to specify all generic field types of the tuple. * * @param targetType The class of the target type, needs to be a subclass of Tuple. * @return The DataSet representing the parsed CSV data. */ public <T extends Tuple> DataSource<T> tupleType(Class<T> targetType) { Validate.notNull(targetType, "The target type class must not be null."); if (!Tuple.class.isAssignableFrom(targetType)) { throw new IllegalArgumentException("The target type must be a subclass of " + Tuple.class.getName()); } @SuppressWarnings("unchecked") TupleTypeInfo<T> typeInfo = (TupleTypeInfo<T>) TypeExtractor.createTypeInfo(targetType); CsvInputFormat<T> inputFormat = new CsvInputFormat<T>(path); Class<?>[] classes = new Class<?>[typeInfo.getArity()]; for (int i = 0; i < typeInfo.getArity(); i++) { classes[i] = typeInfo.getTypeAt(i).getTypeClass(); } configureInputFormat(inputFormat, classes); return new DataSource<T>(executionContext, inputFormat, typeInfo); }
AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());
validateInfo(new ArrayList<Type>(typeHierarchy), subTypes[i], ((TupleTypeInfo<?>) typeInfo).getTypeAt(i));
TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0); TypeInformation<Tuple2<VertexKey, Message>> messageTypeInfo = new TupleTypeInfo<Tuple2<VertexKey,Message>>(keyType, messageType);