AfterWatermarkEarlyAndLate trigger = AfterWatermark.pastEndOfWindow() .withEarlyFirings( (OnceTrigger) fromProto(triggerProto.getAfterEndOfWindow().getEarlyFirings())); .withLateFirings((OnceTrigger) fromProto(eowProto.getLateFirings()));
@Test public void testDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData displayData = DisplayData.from(window); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( displayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(displayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(displayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat(displayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(1)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(true);
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(2)))
WindowingStrategy.of((WindowFn<?, IntervalWindow>) windowFn) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(Never.ever())) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(allowedLateness);
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterPane.elementCountAtLeast(2)) .withLateFirings(Never.ever())) .discardingFiredPanes()
Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(1)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(false);
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(5))))
AfterFirst.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(3))), toProtoAndBackSpec( AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(3))), toProtoAndBackSpec( AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(3))), toProtoAndBackSpec( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(42))) .withLateFirings(AfterPane.elementCountAtLeast(3))),
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterPane.elementCountAtLeast(19))) .accumulatingFiredPanes() .withAllowedLateness(Duration.standardMinutes(3L)));
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterFirst.of( AfterPane.elementCountAtLeast(5),
Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(10)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(false);
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.ZERO) .discardingFiredPanes())
.withTimestampCombiner(TimestampCombiner.EARLIEST) .withTrigger( AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(1))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO)
.triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardSeconds(30))))
.withTrigger( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(2))) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100))
@Test @Category(ValidatesRunner.class) public void testPrimitiveDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData primitiveDisplayData = Iterables.getOnlyElement( DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window)); assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( primitiveDisplayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(primitiveDisplayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat( primitiveDisplayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
@Test @Category({NeedsRunner.class, UsesTestStream.class}) public void testProcessingTimeTrigger() { TestStream<Long> source = TestStream.create(VarLongCoder.of()) .addElements( TimestampedValue.of(1L, new Instant(1000L)), TimestampedValue.of(2L, new Instant(2000L))) .advanceProcessingTime(Duration.standardMinutes(12)) .addElements(TimestampedValue.of(3L, new Instant(3000L))) .advanceProcessingTime(Duration.standardMinutes(6)) .advanceWatermarkToInfinity(); PCollection<Long> sum = p.apply(source) .apply( Window.<Long>configure() .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(5)))) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO)) .apply(Sum.longsGlobally()); PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L); p.run(); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) { return infos .apply( "LeaderboardTeamFixedWindows", Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration)) // We will get early (speculative) results as well as cumulative // processing of late data. .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(FIVE_MINUTES)) .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(TEN_MINUTES))) .withAllowedLateness(allowedLateness) .accumulatingFiredPanes()) // Extract and sum teamname/score pairs from the event data. .apply("ExtractTeamScore", new ExtractAndSumScore("team")); } }
@Test public void testWatermarkHoldForLateNewWindow() throws Exception { Duration allowedLateness = Duration.standardMinutes(1); Duration gapDuration = Duration.millis(10); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(Sessions.withGapDuration(gapDuration)) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withTrigger( Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(1)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(false); assertEquals(null, tester.getWatermarkHold()); assertEquals(null, tester.getOutputWatermark()); tester.advanceInputWatermark(new Instant(40)); injectElements(tester, 1); assertThat(tester.getWatermarkHold(), nullValue()); injectElements(tester, 10); assertThat(tester.getWatermarkHold(), nullValue()); }