/** * Applies the given {@link CoProcessFunction} on the connected input streams, * thereby creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, * this function can also query the time and set timers. When reacting to the firing of set * timers the function can directly emit elements and/or register yet more timers. * * @param coProcessFunction The {@link CoProcessFunction} that is called for each element * in the stream. * * @param <R> The type of elements emitted by the {@code CoProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( CoProcessFunction<IN1, IN2, R> coProcessFunction, TypeInformation<R> outputType) { TwoInputStreamOperator<IN1, IN2, R> operator; if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) { operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction)); } else { operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction)); } return transform("Co-Process", outputType, operator); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { CoProcessOperator<Integer, String, String> operator = new CoProcessOperator<>(new ProcessingTimeQueryingProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new TwoInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement1(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement2(new StreamRecord<>("6")); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5PT:17 TS:null")); expectedOutput.add(new StreamRecord<>("6PT:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { CoProcessOperator<Integer, String, String> operator = new CoProcessOperator<>(new WatermarkQueryingProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new TwoInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark1(new Watermark(17)); testHarness.processWatermark2(new Watermark(17)); testHarness.processElement1(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(42)); testHarness.processWatermark2(new Watermark(42)); testHarness.processElement2(new StreamRecord<>("6", 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
/** * Applies the given {@link CoProcessFunction} on the connected input streams, * thereby creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, * this function can also query the time and set timers. When reacting to the firing of set * timers the function can directly emit elements and/or register yet more timers. * * @param coProcessFunction The {@link CoProcessFunction} that is called for each element * in the stream. * * @param <R> The type of elements emitted by the {@code CoProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( CoProcessFunction<IN1, IN2, R> coProcessFunction, TypeInformation<R> outputType) { TwoInputStreamOperator<IN1, IN2, R> operator; if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) { operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction)); } else { operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction)); } return transform("Co-Process", outputType, operator); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
/** * Applies the given {@link CoProcessFunction} on the connected input streams, * thereby creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, * this function can also query the time and set timers. When reacting to the firing of set * timers the function can directly emit elements and/or register yet more timers. * * @param coProcessFunction The {@link CoProcessFunction} that is called for each element * in the stream. * * @param <R> The type of elements emitted by the {@code CoProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( CoProcessFunction<IN1, IN2, R> coProcessFunction, TypeInformation<R> outputType) { TwoInputStreamOperator<IN1, IN2, R> operator; if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) { operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction)); } else { operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction)); } return transform("Co-Process", outputType, operator); }
/** * Applies the given {@link CoProcessFunction} on the connected input streams, * thereby creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, * this function can also query the time and set timers. When reacting to the firing of set * timers the function can directly emit elements and/or register yet more timers. * * @param coProcessFunction The {@link CoProcessFunction} that is called for each element * in the stream. * * @param <R> The type of elements emitted by the {@code CoProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( CoProcessFunction<IN1, IN2, R> coProcessFunction, TypeInformation<R> outputType) { TwoInputStreamOperator<IN1, IN2, R> operator; if ((inputStream1 instanceof KeyedStream) && (inputStream2 instanceof KeyedStream)) { operator = new KeyedCoProcessOperator<>(inputStream1.clean(coProcessFunction)); } else { operator = new CoProcessOperator<>(inputStream1.clean(coProcessFunction)); } return transform("Co-Process", outputType, operator); }