/** * Creates SplitDataProperties for the given data types. * * @param source The DataSource for which the SplitDataProperties are created. */ public SplitDataProperties(DataSource<T> source) { this.type = source.getType(); }
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }
@Test public void testReturnType() throws Exception { CsvReader reader = getCsvReader(); DataSource<Item> items = reader.tupleType(Item.class); Assert.assertTrue(items.getType().getTypeClass() == Item.class); }
@Test public void testWithValueType() throws Exception { CsvReader reader = getCsvReader(); DataSource<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> items = reader.types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class); TypeInformation<?> info = items.getType(); Assert.assertEquals(true, info.isTupleType()); Assert.assertEquals(Tuple8.class, info.getTypeClass()); }
@Test public void testSubClassWithPartialsInHierarchie() throws Exception { CsvReader reader = getCsvReader(); DataSource<FinalItem> sitems = reader.tupleType(FinalItem.class); TypeInformation<?> info = sitems.getType(); Assert.assertEquals(true, info.isTupleType()); Assert.assertEquals(FinalItem.class, info.getTypeClass()); @SuppressWarnings("unchecked") TupleTypeInfo<SubItem> tinfo = (TupleTypeInfo<SubItem>) info; Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, tinfo.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tinfo.getTypeAt(1)); Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, tinfo.getTypeAt(2)); Assert.assertEquals(ValueTypeInfo.class, tinfo.getTypeAt(3).getClass()); Assert.assertEquals(ValueTypeInfo.class, tinfo.getTypeAt(4).getClass()); Assert.assertEquals(StringValue.class, ((ValueTypeInfo<?>) tinfo.getTypeAt(3)).getTypeClass()); Assert.assertEquals(LongValue.class, ((ValueTypeInfo<?>) tinfo.getTypeAt(4)).getTypeClass()); CsvInputFormat<?> inputFormat = (CsvInputFormat<?>) sitems.getInputFormat(); Assert.assertArrayEquals(new Class<?>[] {Integer.class, String.class, Double.class, StringValue.class, LongValue.class}, inputFormat.getFieldTypes()); }
@Test public void testFieldTypes() throws Exception { CsvReader reader = getCsvReader(); DataSource<Item> items = reader.tupleType(Item.class); TypeInformation<?> info = items.getType(); if (!info.isTupleType()) { Assert.fail(); } else { TupleTypeInfo<?> tinfo = (TupleTypeInfo<?>) info; Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, tinfo.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tinfo.getTypeAt(1)); Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, tinfo.getTypeAt(2)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tinfo.getTypeAt(3)); } CsvInputFormat<?> inputFormat = (CsvInputFormat<?>) items.getInputFormat(); Assert.assertArrayEquals(new Class<?>[]{Integer.class, String.class, Double.class, String.class}, inputFormat.getFieldTypes()); }
@Test public void testSubClass() throws Exception { CsvReader reader = getCsvReader(); DataSource<SubItem> sitems = reader.tupleType(SubItem.class); TypeInformation<?> info = sitems.getType(); Assert.assertEquals(true, info.isTupleType()); Assert.assertEquals(SubItem.class, info.getTypeClass()); @SuppressWarnings("unchecked") TupleTypeInfo<SubItem> tinfo = (TupleTypeInfo<SubItem>) info; Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, tinfo.getTypeAt(0)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tinfo.getTypeAt(1)); Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, tinfo.getTypeAt(2)); Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tinfo.getTypeAt(3)); CsvInputFormat<?> inputFormat = (CsvInputFormat<?>) sitems.getInputFormat(); Assert.assertArrayEquals(new Class<?>[]{Integer.class, String.class, Double.class, String.class}, inputFormat.getFieldTypes()); }
/** * Creates SplitDataProperties for the given data types. * * @param source The DataSource for which the SplitDataProperties are created. */ public SplitDataProperties(DataSource<T> source) { this.type = source.getType(); }
/** * Creates SplitDataProperties for the given data types. * * @param source The DataSource for which the SplitDataProperties are created. */ public SplitDataProperties(DataSource<T> source) { this.type = source.getType(); }
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }