outputTimestamp, greaterThan((ReadableInstant) earlierEndingWindow.maxTimestamp()));
/** * Ensures that later sliding windows have an output time that is past the end of earlier windows. * * <p>If this is the earliest sliding window containing {@code inputTimestamp}, that's fine. * Otherwise, we pick the earliest time that doesn't overlap with earlier windows. */ @Experimental(Kind.OUTPUT_TIME) @Override public Instant getOutputTime(Instant inputTimestamp, IntervalWindow window) { Instant startOfLastSegment = window.maxTimestamp().minus(period); return startOfLastSegment.isBefore(inputTimestamp) ? inputTimestamp : startOfLastSegment.plus(1); }
/** Tests that the last hour of the universe in fact ends at the end of time. */ @Test public void testEndOfTime() { Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp(); FixedWindows windowFn = FixedWindows.of(Duration.standardHours(1)); IntervalWindow truncatedWindow = windowFn.assignWindow(endOfGlobalWindow.minus(1)); assertThat(truncatedWindow.maxTimestamp(), equalTo(endOfGlobalWindow)); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { assertThat( c.timestamp(), equalTo( new IntervalWindow( new Instant(0), new Instant(0).plus(Duration.standardMinutes(10))) .maxTimestamp())); } }));
@Test public void garbageCollectionTimeAfterEndOfGlobalWindow() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault().withWindowFn(windowFn); IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); assertThat(window.maxTimestamp(), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); }
@Test public void testTimerForEndOfWindow() throws Exception { tester = TriggerStateMachineTester.forTrigger( AfterWatermarkStateMachine.pastEndOfWindow(), FixedWindows.of(Duration.millis(100))); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), nullValue()); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), equalTo(window.maxTimestamp())); }
@Test public void beforeEndOfGlobalWindowSame() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.standardMinutes(2); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(10)); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(window.maxTimestamp().plus(allowedLateness))); }
@Test public void testTimerForEndOfWindowCompound() throws Exception { tester = TriggerStateMachineTester.forTrigger( AfterWatermarkStateMachine.pastEndOfWindow().withEarlyFirings(NeverStateMachine.ever()), FixedWindows.of(Duration.millis(100))); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), nullValue()); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), equalTo(window.maxTimestamp())); }
@Test public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.millis(Long.MAX_VALUE); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(-100)); assertThat( window.maxTimestamp().plus(allowedLateness), Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp())); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); } }
triggerShouldFinish(mockTriggerStateMachine); tester.advanceInputWatermark(new Instant(15)); tester.fireTimer(mergedWindow, mergedWindow.maxTimestamp(), TimeDomain.EVENT_TIME);
assertThat(tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp())); new Instant(expectedWindow.maxTimestamp().plus(Duration.standardHours(1)))); tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp().plus(allowedLateness)));
tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp().plus(allowedLateness))); injectElement(tester, 5); assertThat( tester.getWatermarkHold(), equalTo(expectedWindow.maxTimestamp().plus(allowedLateness))); tester.advanceInputWatermark(expectedWindow.maxTimestamp().plus(1)); tester.fireTimer(expectedWindow, expectedWindow.maxTimestamp(), TimeDomain.EVENT_TIME);
/** Ensure a closed trigger has its state recorded in the merge result window. */ @Test public void testMergingWithCloseTrigger() throws Exception { Duration allowedLateness = Duration.millis(50); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, allowedLateness, ClosingBehavior.FIRE_IF_NON_EMPTY); // Create a new merged session window. IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); // Force the trigger to be closed for the merged window. when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); // Fire and end-of-window timer as though the trigger set it tester.advanceInputWatermark(new Instant(13)); tester.fireTimer(mergedWindow, mergedWindow.maxTimestamp(), TimeDomain.EVENT_TIME); // Trigger is now closed. assertTrue(tester.isMarkedFinished(mergedWindow)); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false); // Revisit the same session window. tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); // Trigger is still closed. assertTrue(tester.isMarkedFinished(mergedWindow)); }
assertTrue(window.maxTimestamp().isBefore(GlobalWindow.INSTANCE.maxTimestamp())); assertTrue( window.maxTimestamp().plus(allowedLateness).isAfter(GlobalWindow.INSTANCE.maxTimestamp())); tester.advanceInputWatermark(window.maxTimestamp());
tester.fireTimer(expectedWindow, expectedWindow.maxTimestamp(), TimeDomain.EVENT_TIME);
.maxTimestamp() .plus(windowingStrategy.getAllowedLateness()) .plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS)
for (TimestampedValue<KV<String, Integer>> tv : expectedUnfiltered) { if (!window.start().isAfter(tv.getTimestamp()) && !tv.getTimestamp().isAfter(window.maxTimestamp())) { expected.add(tv);
.maxTimestamp() .plus(windowingStrategy.getAllowedLateness()) .plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS)
window1.maxTimestamp() .plus(windowingStrategy.getAllowedLateness()) .plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS)
timerInternals, WINDOW_1 .maxTimestamp() .plus(ALLOWED_LATENESS) .plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS) timerInternals, WINDOW_2 .maxTimestamp() .plus(ALLOWED_LATENESS) .plus(StatefulDoFnRunner.TimeInternalsCleanupTimer.GC_DELAY_MS)