/** * Creates a data stream from the given non-empty collection. * * <p>Note that this operation will result in a non-parallel data stream source, * i.e., a data stream source with parallelism one. * * @param data * The collection of elements to create the data stream from * @param typeInfo * The TypeInformation for the produced data stream * @param <OUT> * The type of the returned data stream * @return The data stream representing the given collection */ public <OUT> DataStreamSource<OUT> fromCollection(Collection<OUT> data, TypeInformation<OUT> typeInfo) { Preconditions.checkNotNull(data, "Collection must not be null"); // must not have null elements and mixed elements FromElementsFunction.checkCollection(data, typeInfo.getTypeClass()); SourceFunction<OUT> function; try { function = new FromElementsFunction<>(typeInfo.createSerializer(getConfig()), data); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } return addSource(function, "Collection Source", typeInfo).setParallelism(1); }
public static DataStream<Tuple2<String, Integer>> getSource(StreamExecutionEnvironment env, long rate) { return env.fromCollection(new ThrottledIterator<>(new GradeSource(), rate), TypeInformation.of(new TypeHint<Tuple2<String, Integer>>(){})); } }
public AggregatingUdf(TypeInformation<T> typeInfo, AggregationFunction<Object>[] aggFunctions, int[] fieldPositions) { Preconditions.checkNotNull(typeInfo); Preconditions.checkNotNull(aggFunctions); Preconditions.checkArgument(aggFunctions.length == fieldPositions.length); Preconditions.checkArgument(typeInfo.isTupleType(), "TypeInfo for Scala Aggregate Operator must be a tuple TypeInfo."); this.typeInfo = typeInfo; this.aggFunctions = aggFunctions; this.fieldPositions = fieldPositions; }
@SuppressWarnings("unchecked") @Override @PublicEvolving public TypeSerializer<T> createSerializer(ExecutionConfig executionConfig) { return (TypeSerializer<T>) new GenericArraySerializer<C>( componentInfo.getTypeClass(), componentInfo.createSerializer(executionConfig)); }
RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) { int arity = ((TupleTypeInfoBase) typeInfo).getArity(); if (pos < 0 || pos >= arity) { throw new CompositeType.InvalidFieldReferenceException( "Tried to select " + ((Integer) pos).toString() + ". field on \"" + typeInfo.toString() + "\", which is an invalid index."); } checkNotNull(typeInfo, "typeInfo must not be null."); checkNotNull(innerAccessor, "innerAccessor must not be null."); this.pos = pos; this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos); this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config); this.length = this.serializer.getArity(); this.fields = new Object[this.length]; this.innerAccessor = innerAccessor; }
private static <T> String getSerializerTree(TypeInformation<T> ti, int indent) { String ret = ""; if (ti instanceof CompositeType) { ret += StringUtils.repeat(' ', indent) + ti.getClass().getSimpleName() + "\n"; CompositeType<T> cti = (CompositeType<T>) ti; String[] fieldNames = cti.getFieldNames(); for (int i = 0; i < cti.getArity(); i++) { TypeInformation<?> fieldType = cti.getTypeAt(i); ret += StringUtils.repeat(' ', indent + 2) + fieldNames[i] + ":" + getSerializerTree(fieldType, indent); } } else { if (ti instanceof GenericTypeInfo) { ret += StringUtils.repeat(' ', indent) + "GenericTypeInfo (" + ti.getTypeClass().getSimpleName() + ")\n"; ret += getGenericTypeTree(ti.getTypeClass(), indent + 4); } else { ret += StringUtils.repeat(' ', indent) + ti.toString() + "\n"; } } return ret; }
inputType.createSerializer(new ExecutionConfig())); TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig()); assertTrue(keySerializer instanceof KryoSerializer); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999)); expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999)); expectedOutput.add(new Watermark(2999));
@Test @SuppressWarnings("unchecked") public void testTumblingEventTimeWindowsApply() throws Exception { closeCalled.set(0); final int windowSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()), EventTimeTrigger.create(), 0, null /* late data output tag */); testTumblingEventTimeWindows(operator); // we close once in the rest... Assert.assertEquals("Close was not called.", 2, closeCalled.get()); }
@Test @SuppressWarnings("unchecked") public void testTumblingEventTimeWindowsReduce() throws Exception { closeCalled.set(0); final int windowSize = 3; ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), EventTimeTrigger.create(), 0, null /* late data output tag */); testTumblingEventTimeWindows(operator); }
Event middle1Event3 = new Event(41, "a", 4.0); OutputTag<Event> lateDataTag = new OutputTag<Event>("late-data", TypeInformation.of(Event.class)); harness.processElement(new StreamRecord<>(startEvent, 6)); verifyWatermark(harness.getOutput().poll(), Long.MIN_VALUE); harness.processWatermark(new Watermark(6L)); verifyWatermark(harness.getOutput().poll(), 6L); while (!harness.getSideOutput(lateDataTag).isEmpty()) { StreamRecord<Event> eventStreamRecord = harness.getSideOutput(lateDataTag).poll(); late.add(eventStreamRecord.getValue());
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.getConfig().setGlobalJobParameters(params); return newDataPoint.f2 - oldDataPoint.f2; }, carData.getType().createSerializer(env.getConfig()))) .maxBy(1);
@Override protected <T> TypeSerializer<T> createSerializer(Class<T> type) { TypeInformation<T> typeInfo = TypeExtractor.getForClass(type); return typeInfo.createSerializer(new ExecutionConfig()); } }
public ArrayFieldAccessor(int pos, TypeInformation typeInfo) { if (pos < 0) { throw new CompositeType.InvalidFieldReferenceException("The " + ((Integer) pos).toString() + ". field selected on" + " an array, which is an invalid index."); } checkNotNull(typeInfo, "typeInfo must not be null."); this.pos = pos; this.fieldType = BasicTypeInfo.getInfoFor(typeInfo.getTypeClass().getComponentType()); }
/** * Creates a new de-/serialization schema for the given type. * * @param typeInfo The type information for the type de-/serialized by this schema. * @param ec The execution config, which is used to parametrize the type serializers. */ public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, ExecutionConfig ec) { this.typeInfo = checkNotNull(typeInfo, "typeInfo"); this.serializer = typeInfo.createSerializer(ec); }
public StreamTaskTestHarness( Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory, TypeInformation<OUT> outputType) { this.taskFactory = checkNotNull(taskFactory); this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE; this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE; this.jobConfig = new Configuration(); this.taskConfig = new Configuration(); this.executionConfig = new ExecutionConfig(); streamConfig = new StreamConfig(taskConfig); outputSerializer = outputType.createSerializer(executionConfig); outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer); this.taskStateManager = new TestTaskStateManager(); }
/** * Writes a DataStream into a Cassandra database. * * @param input input DataStream * @param <IN> input type * @return CassandraSinkBuilder, to further configure the sink */ public static <IN> CassandraSinkBuilder<IN> addSink(DataStream<IN> input) { TypeInformation<IN> typeInfo = input.getType(); if (typeInfo instanceof TupleTypeInfo) { DataStream<Tuple> tupleInput = (DataStream<Tuple>) input; return (CassandraSinkBuilder<IN>) new CassandraTupleSinkBuilder<>(tupleInput, tupleInput.getType(), tupleInput.getType().createSerializer(tupleInput.getExecutionEnvironment().getConfig())); } if (typeInfo instanceof RowTypeInfo) { DataStream<Row> rowInput = (DataStream<Row>) input; return (CassandraSinkBuilder<IN>) new CassandraRowSinkBuilder(rowInput, rowInput.getType(), rowInput.getType().createSerializer(rowInput.getExecutionEnvironment().getConfig())); } if (typeInfo instanceof PojoTypeInfo) { return new CassandraPojoSinkBuilder<>(input, input.getType(), input.getType().createSerializer(input.getExecutionEnvironment().getConfig())); } if (typeInfo instanceof CaseClassTypeInfo) { DataStream<Product> productInput = (DataStream<Product>) input; return (CassandraSinkBuilder<IN>) new CassandraScalaProductSinkBuilder<>(productInput, productInput.getType(), productInput.getType().createSerializer(input.getExecutionEnvironment().getConfig())); } throw new IllegalArgumentException("No support for the type of the given DataStream: " + input.getType()); }
ProcessJoinFunction<IN1, IN2, OUT> processJoinFunction, TypeInformation<OUT> outputType) { Preconditions.checkNotNull(processJoinFunction); Preconditions.checkNotNull(outputType); final ProcessJoinFunction<IN1, IN2, OUT> cleanedUdf = left.getExecutionEnvironment().clean(processJoinFunction); lowerBoundInclusive, upperBoundInclusive, left.getType().createSerializer(left.getExecutionConfig()), right.getType().createSerializer(right.getExecutionConfig()), cleanedUdf );
public TypeSerializerFormatTest(int numberOfTuples, long blockSize, int parallelism) { super(numberOfTuples, blockSize, parallelism); resultType = TypeExtractor.getForObject(getRecord(0)); serializer = resultType.createSerializer(new ExecutionConfig()); }
RecursiveTupleFieldAccessor(int pos, FieldAccessor<R, F> innerAccessor, TypeInformation<T> typeInfo) { checkNotNull(typeInfo, "typeInfo must not be null."); checkNotNull(innerAccessor, "innerAccessor must not be null."); int arity = ((TupleTypeInfo) typeInfo).getArity(); if (pos < 0 || pos >= arity) { throw new CompositeType.InvalidFieldReferenceException( "Tried to select " + ((Integer) pos).toString() + ". field on \"" + typeInfo.toString() + "\", which is an invalid index."); } this.pos = pos; this.innerAccessor = innerAccessor; this.fieldType = innerAccessor.fieldType; }
public static TypeSerializer<Event> createTypeSerializer() { TypeInformation<Event> typeInformation = (TypeInformation<Event>) TypeExtractor.createTypeInfo(Event.class); return typeInformation.createSerializer(new ExecutionConfig()); } }