final DataStream<String> output = srcOne.connect(broadcast).process( new TestBroadcastProcessFunction());
final DataStream<String> output = srcOne.connect(broadcast).process( new TestKeyedBroadcastProcessFunction(100000L, expected));
/** * We connect two different data streams in a chain to a CoMap. */ @Test public void differentDataStreamSameChain() throws Exception { TestListResultSink<String> resultSink = new TestListResultSink<>(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataStream<Integer> src = env.fromElements(1, 3, 5); DataStream<String> stringMap = src.map(value -> "x " + value); stringMap.connect(src).map(new CoMapFunction<String, Integer, String>() { @Override public String map1(String value) { return value; } @Override public String map2(Integer value) { return String.valueOf(value + 1); } }).addSink(resultSink); env.execute(); List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6"); List<String> result = resultSink.getResult(); Collections.sort(expected); Collections.sort(result); assertEquals(expected, result); }
public static void main(String[] args) throws Exception { // Checking input parameters final ParameterTool params = ParameterTool.fromArgs(args); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); DataStream<Integer> trainingData = env.addSource(new FiniteTrainingDataSource()); DataStream<Integer> newData = env.addSource(new FiniteNewDataSource()); // build new model on every second of new data DataStream<Double[]> model = trainingData .assignTimestampsAndWatermarks(new LinearTimestamp()) .timeWindowAll(Time.of(5000, TimeUnit.MILLISECONDS)) .apply(new PartialModelBuilder()); // use partial model for newData DataStream<Integer> prediction = newData.connect(model).map(new Predictor()); // emit result if (params.has("output")) { prediction.writeAsText(params.get("output")); } else { System.out.println("Printing result to stdout. Use --output to specify output path."); prediction.print(); } // execute program env.execute("Streaming Incremental Learning"); }
.keyBy(Long::intValue); stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {
source.connect(source).transform( "Custom Operator", BasicTypeInfo.STRING_TYPE_INFO,
@Test public void testOutputTypeConfigurationWithTwoInputTransformation() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> source1 = env.fromElements(1, 10); DataStream<Integer> source2 = env.fromElements(2, 11); ConnectedStreams<Integer, Integer> connectedSource = source1.connect(source2); OutputTypeConfigurableOperationWithTwoInputs outputTypeConfigurableOperation = new OutputTypeConfigurableOperationWithTwoInputs(); DataStream<Integer> result = connectedSource.transform( "Two input and output type configurable operation", BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation); result.addSink(new DiscardingSink<>()); env.getStreamGraph(); assertEquals(BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation.getTypeInformation()); }
/** * Tests that the max parallelism is properly set for connected * streams. */ @Test public void testMaxParallelismWithConnectedKeyedStream() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128); DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult = input1 .connect(input2) .keyBy(value -> value, value -> value) .map(new NoOpIntCoMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner(); StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner(); }
.connect(ds2) .process(new CoProcessFunction<Integer, Integer, Integer>() { @Override
.connect(ds2) .process(new CoProcessFunction<Integer, Integer, Integer>() { @Override
BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
dataStream1.connect(dataStream2) .flatMap(new CoFlatMapFunction<Long, Long, Long>() {
assertEquals("a", newSplitEdge.getSelectedNames().get(0)); ConnectedStreams<Integer, Integer> connect = map.connect(flatMap); CoMapFunction<Integer, Integer, String> coMapper = new CoMapFunction<Integer, Integer, String>() { private static final long serialVersionUID = 1L;
source.connect(source).map(new TestCoMap<Long, Long, Integer>()).print(); fail(); } catch (Exception ignored) {} source.connect(source).flatMap(new TestCoFlatMap<Long, Long, Integer>()).print(); fail(); } catch (Exception ignored) {} source.connect(source).keyBy(new TestKeySelector<Long, String>(), new TestKeySelector<>()); fail(); } catch (Exception ignored) {} source.map(new TestMap<Long, Long>()).returns(Long.class).print(); source.flatMap(new TestFlatMap<Long, Long>()).returns(new TypeHint<Long>(){}).print(); source.connect(source).map(new TestCoMap<Long, Long, Integer>()).returns(BasicTypeInfo.INT_TYPE_INFO).print(); source.connect(source).flatMap(new TestCoFlatMap<Long, Long, Integer>()) .returns(BasicTypeInfo.INT_TYPE_INFO).print(); source.connect(source).keyBy(new TestKeySelector<>(), new TestKeySelector<>(), Types.STRING); source.coGroup(source).where(new TestKeySelector<>(), Types.STRING).equalTo(new TestKeySelector<>(), Types.STRING); source.join(source).where(new TestKeySelector<>(), Types.STRING).equalTo(new TestKeySelector<>(), Types.STRING);
opMethod.invoke(map2, minResource4, preferredResource4); DataStream<Long> connected = map1.connect(map2) .flatMap(new CoFlatMapFunction<Long, Long, Long>() { @Override
DataStream<String> connected = filter1.connect(filter2) .flatMap(new CoFlatMapFunction<Integer, Tuple2<Integer, Integer>, String>() {
ConnectedStreams<Tuple2<Long, Long>, Tuple2<Long, Long>> connected = src1.connect(src2);
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String ridesFile = params.get("rides", ExerciseBase.pathToRideData); final String faresFile = params.get("fares", ExerciseBase.pathToFareData); final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env .addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(ridesFile, servingSpeedFactor))) .filter((TaxiRide ride) -> (ride.isStart && (ride.rideId % 1000 != 0))) .keyBy(ride -> ride.rideId); DataStream<TaxiFare> fares = env .addSource(fareSourceOrTest(new CheckpointedTaxiFareSource(faresFile, servingSpeedFactor))) .keyBy(fare -> fare.rideId); SingleOutputStreamOperator processed = rides .connect(fares) .process(new EnrichmentFunction()); printOrTest(processed.getSideOutput(unmatchedFares)); env.execute("ExpiringStateExercise (java)"); }