public static <X extends Tuple> TupleTypeInfo<X> getBasicTupleTypeInfo(Class<?>... basicTypes) { if (basicTypes == null || basicTypes.length == 0) { throw new IllegalArgumentException(); } TypeInformation<?>[] infos = new TypeInformation<?>[basicTypes.length]; for (int i = 0; i < infos.length; i++) { Class<?> type = basicTypes[i]; if (type == null) { throw new IllegalArgumentException("Type at position " + i + " is null."); } TypeInformation<?> info = BasicTypeInfo.getInfoFor(type); if (info == null) { throw new IllegalArgumentException("Type at position " + i + " is not a basic type."); } infos[i] = info; } @SuppressWarnings("unchecked") TupleTypeInfo<X> tupleInfo = (TupleTypeInfo<X>) new TupleTypeInfo<Tuple>(infos); return tupleInfo; }
/** * Projects a pair of crossed elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0> ProjectCross<I1, I2, Tuple1<T0>> types(Class<T0> type0) { Class<?>[] types = {type0}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple1<T0>> tType = new TupleTypeInfo<Tuple1<T0>>(fTypes); return new ProjectCross<I1, I2, Tuple1<T0>>(this.ds1, this.ds2, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of joined elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0> ProjectJoin<I1, I2, Tuple1<T0>> types(Class<T0> type0) { Class<?>[] types = {type0}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple1<T0>> tType = new TupleTypeInfo<Tuple1<T0>>(fTypes); return new ProjectJoin<I1, I2, Tuple1<T0>>(this.ds1, this.ds2, this.keys1, this.keys2, this.hint, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of crossed elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @param type2 The class of field '2' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1, T2> ProjectCross<I1, I2, Tuple3<T0, T1, T2>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2) { Class<?>[] types = {type0, type1, type2}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple3<T0, T1, T2>> tType = new TupleTypeInfo<Tuple3<T0, T1, T2>>(fTypes); return new ProjectCross<I1, I2, Tuple3<T0, T1, T2>>(this.ds1, this.ds2, this.fieldIndexes, this.isFieldInFirst, tType); }
public DefaultCross(DataSet<I1> input1, DataSet<I2> input2) { super(input1, input2, (CrossFunction<I1, I2, Tuple2<I1, I2>>) new DefaultCrossFunction<I1, I2>(), new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType())); if (input1 == null || input2 == null) { throw new NullPointerException(); } this.input1 = input1; this.input2 = input2; }
/** * Projects a pair of joined elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1> ProjectJoin<I1, I2, Tuple2<T0, T1>> types(Class<T0> type0, Class<T1> type1) { Class<?>[] types = {type0, type1}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple2<T0, T1>> tType = new TupleTypeInfo<Tuple2<T0, T1>>(fTypes); return new ProjectJoin<I1, I2, Tuple2<T0, T1>>(this.ds1, this.ds2, this.keys1, this.keys2, this.hint, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of crossed elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1> ProjectCross<I1, I2, Tuple2<T0, T1>> types(Class<T0> type0, Class<T1> type1) { Class<?>[] types = {type0, type1}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple2<T0, T1>> tType = new TupleTypeInfo<Tuple2<T0, T1>>(fTypes); return new ProjectCross<I1, I2, Tuple2<T0, T1>>(this.ds1, this.ds2, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of joined elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @param type2 The class of field '2' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1, T2> ProjectJoin<I1, I2, Tuple3<T0, T1, T2>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2) { Class<?>[] types = {type0, type1, type2}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple3<T0, T1, T2>> tType = new TupleTypeInfo<Tuple3<T0, T1, T2>>(fTypes); return new ProjectJoin<I1, I2, Tuple3<T0, T1, T2>>(this.ds1, this.ds2, this.keys1, this.keys2, this.hint, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of joined elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @param type2 The class of field '2' of the result tuples. * @param type3 The class of field '3' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1, T2, T3> ProjectJoin<I1, I2, Tuple4<T0, T1, T2, T3>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3) { Class<?>[] types = {type0, type1, type2, type3}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple4<T0, T1, T2, T3>> tType = new TupleTypeInfo<Tuple4<T0, T1, T2, T3>>(fTypes); return new ProjectJoin<I1, I2, Tuple4<T0, T1, T2, T3>>(this.ds1, this.ds2, this.keys1, this.keys2, this.hint, this.fieldIndexes, this.isFieldInFirst, tType); }
/** * Projects a pair of crossed elements to a {@link Tuple} with the previously selected fields. * Requires the classes of the fields of the resulting tuples. * * @param type0 The class of field '0' of the result tuples. * @param type1 The class of field '1' of the result tuples. * @param type2 The class of field '2' of the result tuples. * @param type3 The class of field '3' of the result tuples. * @return The projected data set. * * @see Tuple * @see DataSet */ public <T0, T1, T2, T3> ProjectCross<I1, I2, Tuple4<T0, T1, T2, T3>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3) { Class<?>[] types = {type0, type1, type2, type3}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types); TupleTypeInfo<Tuple4<T0, T1, T2, T3>> tType = new TupleTypeInfo<Tuple4<T0, T1, T2, T3>>(fTypes); return new ProjectCross<I1, I2, Tuple4<T0, T1, T2, T3>>(this.ds1, this.ds2, this.fieldIndexes, this.isFieldInFirst, tType); }
protected DefaultJoin(DataSet<I1> input1, DataSet<I2> input2, Keys<I1> keys1, Keys<I2> keys2, JoinHint hint) { super(input1, input2, keys1, keys2, (JoinFunction<I1, I2, Tuple2<I1, I2>>) new DefaultJoinFunction<I1, I2>(), new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), hint); }
/** * Projects a {@link Tuple} {@link DataSet} to the previously selected fields. * Requires the classes of the fields of the resulting Tuples. * * @param type0 The class of field '0' of the result Tuples. * @return The projected DataSet. * * @see Tuple * @see DataSet */ public <T0> ProjectOperator<T, Tuple1<T0>> types(Class<T0> type0) { Class<?>[] types = {type0}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types, ds.getType()); TupleTypeInfo<Tuple1<T0>> tType = new TupleTypeInfo<Tuple1<T0>>(fTypes); return new ProjectOperator<T, Tuple1<T0>>(this.ds, this.fieldIndexes, tType); }
/** * Projects a {@link Tuple} {@link DataSet} to the previously selected fields. * Requires the classes of the fields of the resulting Tuples. * * @param type0 The class of field '0' of the result Tuples. * @param type1 The class of field '1' of the result Tuples. * @return The projected DataSet. * * @see Tuple * @see DataSet */ public <T0, T1> ProjectOperator<T, Tuple2<T0, T1>> types(Class<T0> type0, Class<T1> type1) { Class<?>[] types = {type0, type1}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types, ds.getType()); TupleTypeInfo<Tuple2<T0, T1>> tType = new TupleTypeInfo<Tuple2<T0, T1>>(fTypes); return new ProjectOperator<T, Tuple2<T0, T1>>(this.ds, this.fieldIndexes, tType); }
/** * Projects a {@link Tuple} {@link DataSet} to the previously selected fields. * Requires the classes of the fields of the resulting Tuples. * * @param type0 The class of field '0' of the result Tuples. * @param type1 The class of field '1' of the result Tuples. * @param type2 The class of field '2' of the result Tuples. * @return The projected DataSet. * * @see Tuple * @see DataSet */ public <T0, T1, T2> ProjectOperator<T, Tuple3<T0, T1, T2>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2) { Class<?>[] types = {type0, type1, type2}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types, ds.getType()); TupleTypeInfo<Tuple3<T0, T1, T2>> tType = new TupleTypeInfo<Tuple3<T0, T1, T2>>(fTypes); return new ProjectOperator<T, Tuple3<T0, T1, T2>>(this.ds, this.fieldIndexes, tType); }
/** * Projects a {@link Tuple} {@link DataSet} to the previously selected fields. * Requires the classes of the fields of the resulting Tuples. * * @param type0 The class of field '0' of the result Tuples. * @param type1 The class of field '1' of the result Tuples. * @param type2 The class of field '2' of the result Tuples. * @param type3 The class of field '3' of the result Tuples. * @return The projected DataSet. * * @see Tuple * @see DataSet */ public <T0, T1, T2, T3> ProjectOperator<T, Tuple4<T0, T1, T2, T3>> types(Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3) { Class<?>[] types = {type0, type1, type2, type3}; if(types.length != this.fieldIndexes.length) { throw new IllegalArgumentException("Numbers of projected fields and types do not match."); } TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, types, ds.getType()); TupleTypeInfo<Tuple4<T0, T1, T2, T3>> tType = new TupleTypeInfo<Tuple4<T0, T1, T2, T3>>(fTypes); return new ProjectOperator<T, Tuple4<T0, T1, T2, T3>>(this.ds, this.fieldIndexes, tType); }
@SuppressWarnings({ "unchecked", "rawtypes" }) public static <X> TypeInformation<X> getForObject(X value) { Validate.notNull(value); // check if we can extract the types from tuples, otherwise work with the class if (value instanceof Tuple) { Tuple t = (Tuple) value; int numFields = t.getArity(); TypeInformation<?>[] infos = new TypeInformation[numFields]; for (int i = 0; i < numFields; i++) { Object field = t.getField(i); if (field == null) { throw new InvalidTypesException("Automatic type extraction is not possible on candidates with null values. " + "Please specify the types directly."); } infos[i] = getForObject(field); } return (TypeInformation<X>) new TupleTypeInfo(value.getClass(), infos); } else { return getForClass((Class<X>) value.getClass()); } }
public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> getSmall5TupleDataSet(ExecutionEnvironment env) { List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<Tuple5<Integer, Long, Integer, String, Long>>(); data.add(new Tuple5<Integer, Long, Integer, String, Long>(1,1l,0,"Hallo",1l)); data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,2l,1,"Hallo Welt",2l)); data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,3l,2,"Hallo Welt wie",1l)); Collections.shuffle(data); TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>( BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); return env.fromCollection(data, type); }
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer( Keys.SelectorFunctionKeys<IN, ?> rawKeys, GroupReduceFunction<IN, OUT> function, TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input, boolean combinable) { @SuppressWarnings("unchecked") final Keys.SelectorFunctionKeys<IN, K> keys = (Keys.SelectorFunctionKeys<IN, K>) rawKeys; TypeInformation<Tuple2<K, IN>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, IN>>(keys.getKeyType(), inputType); KeyExtractingMapper<IN, K> extractor = new KeyExtractingMapper<IN, K>(keys.getKeyExtractor()); PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer = new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable); MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor"); reducer.setInput(mapper); mapper.setInput(input); // set the mapper's parallelism to the input parallelism to make sure it is chained mapper.setDegreeOfParallelism(input.getDegreeOfParallelism()); return reducer; } }
public static void main(String[] args) throws Exception { prepareTestDb(); ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5> source = environment.createInput(JDBCInputFormat.buildJDBCInputFormat() .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") .setDBUrl("jdbc:derby:memory:ebookshop") .setQuery("select * from books") .finish(), new TupleTypeInfo(Tuple5.class, INT_TYPE_INFO, STRING_TYPE_INFO, STRING_TYPE_INFO, DOUBLE_TYPE_INFO, INT_TYPE_INFO) ); source.output(JDBCOutputFormat.buildJDBCOutputFormat() .setDrivername("org.apache.derby.jdbc.EmbeddedDriver") .setDBUrl("jdbc:derby:memory:ebookshop") .setQuery("insert into newbooks (id,title,author,price,qty) values (?,?,?,?,?)") .finish()); environment.execute(); }
private static <T, K> MapOperatorBase<Tuple2<K, T>, T, ?> translateSelectorFunctionReducer(Keys.SelectorFunctionKeys<T, ?> rawKeys, ReduceFunction<T> function, TypeInformation<T> inputType, String name, Operator<T> input, int dop) { @SuppressWarnings("unchecked") final Keys.SelectorFunctionKeys<T, K> keys = (Keys.SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, T>>(keys.getKeyType(), inputType); KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper<T, K>(keys.getKeyExtractor()); PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<T, K>(function, keys, name, inputType, typeInfoWithKey); MapOperatorBase<T, Tuple2<K, T>, GenericMap<T, Tuple2<K, T>>> keyExtractingMap = new MapOperatorBase<T, Tuple2<K, T>, GenericMap<T, Tuple2<K, T>>>(extractor, new UnaryOperatorInformation<T, Tuple2<K, T>>(inputType, typeInfoWithKey), "Key Extractor"); MapOperatorBase<Tuple2<K, T>, T, GenericMap<Tuple2<K, T>, T>> keyRemovingMap = new MapOperatorBase<Tuple2<K, T>, T, GenericMap<Tuple2<K, T>, T>>(new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<Tuple2<K, T>, T>(typeInfoWithKey, inputType), "Key Extractor"); keyExtractingMap.setInput(input); reducer.setInput(keyExtractingMap); keyRemovingMap.setInput(reducer); // set dop keyExtractingMap.setDegreeOfParallelism(input.getDegreeOfParallelism()); reducer.setDegreeOfParallelism(dop); keyRemovingMap.setDegreeOfParallelism(dop); return keyRemovingMap; } }