.connect(right) .keyBy(keySelector1, keySelector2) .transform("Interval Join", outputType, operator);
.connect(ds2.keyBy(i -> i)) .process(new CoProcessFunction<Integer, Integer, Integer>() { @Override
.connect(ds2.keyBy(i -> i)) .process(new CoProcessFunction<Integer, Integer, Integer>() { @Override
.connect(broadcastRulesStream) .process(new MatchFunction());
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); // Simulated trade stream DataStream<Trade> tradeStream = FinSources.tradeSource(env); // Simulated customer stream DataStream<Customer> customerStream = FinSources.customerSource(env); // Stream of enriched trades DataStream<EnrichedTrade> joinedStream = tradeStream .keyBy("customerId") .connect(customerStream.keyBy("customerId")) .process(new ProcessingTimeJoinFunction()); joinedStream.print(); env.execute("processing-time join"); }
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); // Simulated trade stream DataStream<Trade> tradeStream = FinSources.tradeSource(env); // Simulated customer stream DataStream<Customer> customerStream = FinSources.customerSource(env); // Stream of enriched trades DataStream<EnrichedTrade> joinedStream = tradeStream .keyBy("customerId") .connect(customerStream.keyBy("customerId")) .process(new EventTimeJoinFunction()); joinedStream.print(); env.execute("event-time join"); }
public DataStream<String> constructTestPipeline(DataStream<String> source) { OutputTag<Integer> filtered = new OutputTag<>("filter", BasicTypeInfo.INT_TYPE_INFO); OutputTag<Integer> process = new OutputTag<>("process", BasicTypeInfo.INT_TYPE_INFO); SingleOutputStreamOperator<String> input = source.process(new ProcessFunction<String, String>() { private static final long serialVersionUID = 1L; @Override public void processElement(String s, Context ctx, Collector<String> out) throws Exception { if (s.startsWith("filter ")) { ctx.output(filtered, Integer.parseInt(s.substring(7))); } else if (s.startsWith("process ")) { ctx.output(process, Integer.parseInt(s.substring(8))); } else { throw new RuntimeException("oOoO"); } } }); BroadcastStream<Integer> broadcast = input.getSideOutput(filtered).broadcast(bcstate); return input.getSideOutput(process).keyBy(i -> i).connect(broadcast).process(new BroadcastProcessor(bcstate)) .uid("stateful"); }
.connect(right) .keyBy(keySelector1, keySelector2) .transform("Interval Join", outputType, operator);
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // In this simple case we need a broadcast state descriptor, but aren't going to // use it to store anything. final MapStateDescriptor<Long, Long> dummyBroadcastState = new MapStateDescriptor<>( "dummy", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env.addSource(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)); // add a socket source BroadcastStream<String> queryStream = env.socketTextStream("localhost", 9999) // EXERCISE QUESTION: Is this needed? // .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(dummyBroadcastState); DataStream<TaxiRide> reports = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction()); printOrTest(reports); env.execute("Ongoing Rides"); }
.connect(right) .keyBy(keySelector1, keySelector2) .transform("Interval Join", outputType, operator);
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); // setup a stream of taxi rides DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor))); // add a socket source for the query stream BroadcastStream<String> queryStream = env .addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1))) .broadcast(queryDescriptor); // connect the two streams and process queries DataStream<Tuple2<String, String>> results = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryProcessor()); printOrTest(results); env.execute("Taxi Query"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // In this simple case we need a broadcast state descriptor, but aren't going to // use it to store anything. final MapStateDescriptor<Long, Long> dummyBroadcastState = new MapStateDescriptor<>( "dummy", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env.addSource(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)); // add a socket source BroadcastStream<String> queryStream = env.socketTextStream("localhost", 9999) .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(dummyBroadcastState); DataStream<TaxiRide> reports = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction()); printOrTest(reports); env.execute("Ongoing Rides"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); // setup a stream of taxi rides DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor))); // add a socket source for the query stream BroadcastStream<String> queryStream = env .addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1))) .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(queryDescriptor); // connect the two streams and process queries DataStream<Tuple2<String, String>> results = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryProcessor()); printOrTest(results); env.execute("Taxi Query"); }
.connect(queryStream) .process(new QueryFunction());
.connect(queryStream) .process(new QueryFunction());
.connect(queryStream) .process(new QueryFunction());
.connect(controlStream) .flatMap(new AmplifierFunction()) .addSink(new InfluxDBSink<>("amplifiedSensors"));