public static void main(String[] args) throws Exception { // Checking input parameters final ParameterTool params = ParameterTool.fromArgs(args); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); DataStream<Integer> trainingData = env.addSource(new FiniteTrainingDataSource()); DataStream<Integer> newData = env.addSource(new FiniteNewDataSource()); // build new model on every second of new data DataStream<Double[]> model = trainingData .assignTimestampsAndWatermarks(new LinearTimestamp()) .timeWindowAll(Time.of(5000, TimeUnit.MILLISECONDS)) .apply(new PartialModelBuilder()); // use partial model for newData DataStream<Integer> prediction = newData.connect(model).map(new Predictor()); // emit result if (params.has("output")) { prediction.writeAsText(params.get("output")); } else { System.out.println("Printing result to stdout. Use --output to specify output path."); prediction.print(); } // execute program env.execute("Streaming Incremental Learning"); }
double triggerMeters = 50; DataStream<Tuple4<Integer, Integer, Double, Long>> topSpeeds = carData .assignTimestampsAndWatermarks(new CarTimestamp()) .keyBy(0) .window(GlobalWindows.create())
.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Integer>() {
.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
}); DataStream<Integer> extractOp = source1.assignTimestampsAndWatermarks( new AscendingTimestampExtractor<Integer>() { @Override
.assignTimestampsAndWatermarks(new TestWatermarkAssigner()) .timeWindowAll(Time.milliseconds(1), Time.milliseconds(1)) .process(new ProcessAllWindowFunction<Integer, Integer, TimeWindow>() {
@Test public void testProcessdWindowFunctionSideOutput() throws Exception { TestListResultSink<Integer> resultSink = new TestListResultSink<>(); TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>(); StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); see.setParallelism(3); see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); DataStream<Integer> dataStream = see.fromCollection(elements); OutputTag<String> sideOutputTag = new OutputTag<String>("side"){}; SingleOutputStreamOperator<Integer> windowOperator = dataStream .assignTimestampsAndWatermarks(new TestWatermarkAssigner()) .keyBy(new TestKeySelector()) .timeWindow(Time.milliseconds(1), Time.milliseconds(1)) .process(new ProcessWindowFunction<Integer, Integer, Integer, TimeWindow>() { private static final long serialVersionUID = 1L; @Override public void process(Integer integer, Context context, Iterable<Integer> elements, Collector<Integer> out) throws Exception { out.collect(integer); context.output(sideOutputTag, "sideout-" + String.valueOf(integer)); } }); windowOperator.getSideOutput(sideOutputTag).addSink(sideOutputResultSink); windowOperator.addSink(resultSink); see.execute(); assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-5"), sideOutputResultSink.getSortedResult()); assertEquals(Arrays.asList(1, 2, 5), resultSink.getSortedResult()); }
.assignTimestampsAndWatermarks(new TestWatermarkAssigner()) .timeWindowAll(Time.milliseconds(1), Time.milliseconds(1)) .sideOutputLateData(lateDataTag)
.assignTimestampsAndWatermarks(new TestWatermarkAssigner()) .keyBy(new TestKeySelector()) .timeWindow(Time.milliseconds(1), Time.milliseconds(1))
/** * Creates a graph from an edge stream operating in event time specified by timeExtractor . * * The time characteristic is set to event time. * * @see {@link org.apache.flink.streaming.api.TimeCharacteristic} * * @param edges a DataStream of edges. * @param timeExtractor the timestamp extractor. * @param context the execution environment. */ public SimpleEdgeStream(DataStream<Edge<K, EV>> edges, AscendingTimestampExtractor<Edge<K,EV>> timeExtractor, StreamExecutionEnvironment context) { context.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); this.edges = edges.assignTimestampsAndWatermarks(timeExtractor); this.context = context; }
return dataStream.assignTimestampsAndWatermarks(waterMarker);
@Test public void testUnboundedPojoStreamAndReturnPojo() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Event> input = env.addSource(new RandomEventSource(5)); input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Event>() { @Override public long extractAscendingTimestamp(Event element) { return element.getTimestamp(); } }); DataStream<Event> output = SiddhiCEP .define("inputStream", input, "id", "name", "price", "timestamp") .cql("from inputStream select timestamp, id, name, price insert into outputStream") .returns("outputStream", Event.class); String resultPath = tempFolder.newFile().toURI().toString(); output.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); assertEquals(5, getLineCount(resultPath)); }
public static void main(String[] args) throws Exception { // Checking input parameters final ParameterTool params = ParameterTool.fromArgs(args); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); DataStream<Integer> trainingData = env.addSource(new FiniteTrainingDataSource()); DataStream<Integer> newData = env.addSource(new FiniteNewDataSource()); // build new model on every second of new data DataStream<Double[]> model = trainingData .assignTimestampsAndWatermarks(new LinearTimestamp()) .timeWindowAll(Time.of(5000, TimeUnit.MILLISECONDS)) .apply(new PartialModelBuilder()); // use partial model for newData DataStream<Integer> prediction = newData.connect(model).map(new Predictor()); // emit result if (params.has("output")) { prediction.writeAsText(params.get("output")); } else { System.out.println("Printing result to stdout. Use --output to specify output path."); prediction.print(); } // execute program env.execute("Streaming Incremental Learning"); }
double triggerMeters = 50; DataStream<Tuple4<Integer, Integer, Double, Long>> topSpeeds = carData .assignTimestampsAndWatermarks(new CarTimestamp()) .keyBy(0) .window(GlobalWindows.create())
@Override @SuppressWarnings("unchecked") public DataStream<?> translate(FlinkOperator<FlatMap> operator, StreamingExecutorContext context) { Settings settings = context.getSettings(); FlinkAccumulatorFactory accumulatorFactory = context.getAccumulatorFactory(); DataStream input = context.getSingleInputStream(operator); UnaryFunctor mapper = operator.getOriginalOperator().getFunctor(); ExtractEventTime evtTimeFn = operator.getOriginalOperator().getEventTimeExtractor(); if (evtTimeFn != null) { input = input.assignTimestampsAndWatermarks( new EventTimeAssigner(context.getAllowedLateness(), evtTimeFn)) .returns((Class) StreamingElement.class); } return input .flatMap(new StreamingUnaryFunctorWrapper(mapper, accumulatorFactory, settings)) .returns((Class) StreamingElement.class) .name(operator.getName()) .setParallelism(operator.getParallelism()); } }
.assignTimestampsAndWatermarks(new Util.TimeAssigner<>(streamParams.getAllowedLateness()))