/** * Verify that there is no late-data side output if the {@code WindowAssigner} does * not assign any windows. */ @Test public void testNoLateSideOutputForSkippedWindows() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processWatermark(0); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertTrue(testHarness.getSideOutput(lateOutputTag) == null || testHarness.getSideOutput(lateOutputTag).isEmpty()); }
@Test public void testLateSideOutput() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processWatermark(20); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L))); // we should also see side output if the WindowAssigner assigns no windows when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processElement(new StreamRecord<>(0, 10L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(10L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L), isStreamRecord(0, 10L))); }
expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.",
assertThat(testHarness.getSideOutput(integerOutputTag), contains(isStreamRecord(17, windowEnd - 1))); assertThat(testHarness.getSideOutput(longOutputTag), contains(isStreamRecord(17L, windowEnd - 1)));
expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.",
expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.",
while (!harness.getSideOutput(lateDataTag).isEmpty()) { StreamRecord<Event> eventStreamRecord = harness.getSideOutput(lateDataTag).poll(); late.add(eventStreamRecord.getValue());
TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput( lateOutputTag), new Tuple2ResultSortComparator()); testHarness.close();
"SideOutput was not correct.", lateExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator()); testHarness.close();
ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator()); assertEquals(null, sideActual); sideActual = testHarness.getSideOutput(lateOutputTag);
ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator()); assertEquals(null, sideActual); sideActual = testHarness.getSideOutput(lateOutputTag); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator()); assertEquals(null, sideActual);
ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput( lateOutputTag); sideActual = testHarness.getSideOutput(lateOutputTag); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator()); assertEquals(null, sideActual);
ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput( lateOutputTag); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag);
@Test public void testCurrentProcessingTimeForTimedOutInProcessingTime() throws Exception { OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); harness.processElement(event().withName("A").asStreamRecord()); harness.setProcessingTime(5); harness.processElement(event().withName("B").asStreamRecord()); harness.setProcessingTime(20); harness.processElement(event().withName("C").asStreamRecord()); assertOutput(harness.getOutput()) .nextElementEquals("5:A:B") .hasNoMoreElements(); // right now we time out only on next event in processing time, therefore the 20 assertOutput(harness.getSideOutput(sideOutputTag)) .nextElementEquals("20:B") .hasNoMoreElements(); } }
@Test public void testTimestampPassingForTimedOutInEventTime() throws Exception { OutputTag<String> timedOut = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("C").withTimestamp(20).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(3).asStreamRecord()); harness.processWatermark(22); assertOutput(harness.getOutput()) .nextElementEquals("5:B:A") .watermarkEquals(22) .hasNoMoreElements(); assertOutput(harness.getSideOutput(timedOut)) .nextElementEquals("15:A") .hasNoMoreElements(); } }
@Test public void testTimestampPassingForTimedOutInProcessingTime() throws Exception { OutputTag<String> timedOut = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); harness.processElement(event().withName("A").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(5); harness.processElement(event().withName("C").withTimestamp(5).asStreamRecord()); harness.setProcessingTime(20); harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); assertOutput(harness.getOutput()) .nextElementEquals("5:A:C") .hasNoMoreElements(); assertOutput(harness.getSideOutput(timedOut)) .nextElementEquals("15:C") .hasNoMoreElements(); } }
@Test public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception { OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); harness.processElement(event().withName("C").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(100); harness.processWatermark(22); assertOutput(harness.getOutput()) .nextElementEquals("100:C:A") .watermarkEquals(22) .hasNoMoreElements(); assertOutput(harness.getSideOutput(sideOutputTag)) .nextElementEquals("100:A") .hasNoMoreElements(); } }
this.stripStreamRecord(testHarness.getSideOutput(tagsToOutputTags.get(additionalOutput1))), contains( WindowedValue.valueInGlobalWindow("extra: one"), this.stripStreamRecord(testHarness.getSideOutput(tagsToOutputTags.get(additionalOutput2))), contains( WindowedValue.valueInGlobalWindow("extra: two"),