/** * The purpose of this method is solely to check whether the data type to be processed * is in fact a tuple type. */ @Override public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) { if (!type.isTupleType()) { throw new InvalidProgramException("The " + CsvOutputFormat.class.getSimpleName() + " can only be used to write tuple data sets."); } } }
/** * The purpose of this method is solely to check whether the data type to be processed * is in fact a tuple type. */ @Override public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) { if (!type.isTupleType()) { throw new InvalidProgramException("The " + ScalaCsvOutputFormat.class.getSimpleName() + " can only be used to write tuple data sets."); } } }
@Override public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) { if (!type.isTupleType()) { throw new IllegalArgumentException("Input TypeInformation is not a tuple type."); } TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>) type; if (tupleType.getArity() != 2) { throw new IllegalArgumentException("Input TypeInformation must be a Tuple2 type."); } }
public AggregatingUdf(TypeInformation<T> typeInfo, AggregationFunction<Object>[] aggFunctions, int[] fieldPositions) { Preconditions.checkNotNull(typeInfo); Preconditions.checkNotNull(aggFunctions); Preconditions.checkArgument(aggFunctions.length == fieldPositions.length); Preconditions.checkArgument(typeInfo.isTupleType(), "TypeInfo for Scala Aggregate Operator must be a tuple TypeInfo."); this.typeInfo = typeInfo; this.aggFunctions = aggFunctions; this.fieldPositions = fieldPositions; }
@SuppressWarnings("unchecked") private <X extends Tuple> DataSink<T> internalWriteAsCsv(Path filePath, String rowDelimiter, String fieldDelimiter, WriteMode wm) { Preconditions.checkArgument(getType().isTupleType(), "The writeAsCsv() method can only be used on data sets of tuples."); CsvOutputFormat<X> of = new CsvOutputFormat<>(filePath, rowDelimiter, fieldDelimiter); if (wm != null) { of.setWriteMode(wm); } return output((OutputFormat<T>) of); }
@Test public void checkTypeInformation() throws Exception { HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<>( new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, new JobConf()); TypeInformation<Tuple2<Void, Long>> tupleType = hadoopInputFormat.getProducedType(); TypeInformation<Tuple2<Void, Long>> expectedType = new TupleTypeInfo<>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO); assertThat(tupleType.isTupleType(), is(true)); assertThat(tupleType, is(equalTo(expectedType))); }
@Test public void testMapPartitionLambda() { MapPartitionFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, String>> f = (i, o) -> {}; TypeInformation<?> ti = TypeExtractor.getMapPartitionReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testJoinLambda() { JoinFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, Double>, Tuple2<Tuple1<Integer>, String>> f = (i1, i2) -> null; TypeInformation<?> ti = TypeExtractor.getJoinReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, NESTED_TUPLE_DOUBLE_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testKeySelectorLambda() { KeySelector<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, String>> f = (i) -> null; TypeInformation<?> ti = TypeExtractor.getKeySelectorTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testMapLambda() { MapFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, String>> f = (i) -> null; TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testCoGroupLambda() { CoGroupFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, Double>, Tuple2<Tuple1<Integer>, String>> f = (i1, i2, o) -> {}; TypeInformation<?> ti = TypeExtractor.getCoGroupReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, NESTED_TUPLE_DOUBLE_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testFlatMapLambda() { FlatMapFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, String>> f = (i, out) -> out.collect(null); TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testPartitionerLambda() { Partitioner<Tuple2<Integer, String>> partitioner = (key, numPartitions) -> key.f1.length() % numPartitions; final TypeInformation<?> ti = TypeExtractor.getPartitionerTypes(partitioner, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(0), BasicTypeInfo.INT_TYPE_INFO); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
@Test public void testWithValueType() throws Exception { CsvReader reader = getCsvReader(); DataSource<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> items = reader.types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class); TypeInformation<?> info = items.getType(); Assert.assertEquals(true, info.isTupleType()); Assert.assertEquals(Tuple8.class, info.getTypeClass()); }
@Test public void testFunctionDependingOnInputWithFunctionHierarchy2() { IdentityMapper5<String> function = new IdentityMapper5<String>(); @SuppressWarnings({ "rawtypes", "unchecked" }) TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, new TupleTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO)); Assert.assertTrue(ti.isTupleType()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti; Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1)); }
@Test public void testFunctionInputInOutputMultipleTimes() { RichMapFunction<Float, ?> function = new FieldDuplicator<Float>(); TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, BasicTypeInfo.FLOAT_TYPE_INFO); Assert.assertTrue(ti.isTupleType()); Assert.assertEquals(2, ti.getArity()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti; Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, tti.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, tti.getTypeAt(1)); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Test public void testCustomArrayWithTypeVariable() { RichMapFunction<CustomArrayObject2<Boolean>[], ?> function = new IdentityMapper<CustomArrayObject2<Boolean>[]>(); TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple1<Boolean>[]>(){})); Assert.assertTrue(ti instanceof ObjectArrayTypeInfo<?, ?>); ObjectArrayTypeInfo<?, ?> oati = (ObjectArrayTypeInfo<?, ?>) ti; Assert.assertTrue(oati.getComponentInfo().isTupleType()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) oati.getComponentInfo(); Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, tti.getTypeAt(0)); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Test public void testDuplicateValueNested() { TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) new DuplicateValueNested<String>(), TypeInformation.of(new TypeHint<Tuple1<Tuple1<String>>>(){})); Assert.assertTrue(ti.isTupleType()); Assert.assertEquals(2, ti.getArity()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti; Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1)); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Test public void testFunctionDependingOnInputWithCustomTupleInput() { IdentityMapper<SameTypeVariable<String>> function = new IdentityMapper<SameTypeVariable<String>>(); TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple2<String, String>>(){})); Assert.assertTrue(ti.isTupleType()); Assert.assertEquals(2, ti.getArity()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti; Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1)); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Test public void testInputInference2() { EdgeMapper2<Boolean> em = new EdgeMapper2<Boolean>(); TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) em, Types.BOOLEAN); Assert.assertTrue(ti.isTupleType()); Assert.assertEquals(3, ti.getArity()); TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti; Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tti.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tti.getTypeAt(1)); Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, tti.getTypeAt(2)); }