Duration processingTimeDelta = new Duration(lastProcessingTime, event.processingTime); if (processingTimeDelta.getMillis() > 0) { stream = stream.advanceProcessingTime(processingTimeDelta);
.addElements( elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class))
TimestampedValue.atMinimumTimestamp(5), TimestampedValue.atMinimumTimestamp(6)) .advanceProcessingTime(Duration.standardMinutes(10)) .advanceWatermarkToInfinity(); PCollection<Integer> streamVals = p.apply(new DirectTestStream<>(runner, testStream));
event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8))) .advanceProcessingTime(Duration.standardMinutes(10)) .advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))) .addElements( .advanceProcessingTime(Duration.standardMinutes(12))
.advanceProcessingTime(Duration.standardMinutes(10)) .addElements(event(TestUser.RED_TWO, 5, Duration.standardMinutes(3))) .advanceProcessingTime(Duration.standardMinutes(12)) .addElements(event(TestUser.BLUE_TWO, 3, Duration.standardSeconds(22))) .advanceProcessingTime(Duration.standardMinutes(10))
TestStream.create(VarLongCoder.of()) .addElements(TimestampedValue.of(1L, new Instant(1000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane .addElements(TimestampedValue.of(2L, new Instant(2000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane .addElements(TimestampedValue.of(3L, new Instant(3000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane
event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)) .advanceProcessingTime(Duration.standardMinutes(7)) .addElements( event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))) .advanceProcessingTime(Duration.standardMinutes(5)) .advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(Duration.standardHours(12))) event(TestUser.RED_ONE, 3, Duration.standardMinutes(7)), event(TestUser.RED_ONE, 2, (ALLOWED_LATENESS).plus(Duration.standardHours(13)))) .advanceProcessingTime(Duration.standardMinutes(6)) .addElements(event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(12))) .advanceProcessingTime(Duration.standardMinutes(20)) .advanceWatermarkToInfinity();
TimestampedValue.of(KV.of(2, "k2"), base.plus(Duration.standardSeconds(10))), TimestampedValue.of(KV.of(3, "k3"), base.plus(Duration.standardSeconds(20)))) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( TimestampedValue.of(KV.of(1, "k1"), base.plus(Duration.standardSeconds(30))),
.addElements(KV.of("key", "input1")) .addElements(KV.of("key", "input2")) .advanceProcessingTime(Duration.standardSeconds(2)) .advanceWatermarkToInfinity();
.advanceWatermarkTo(base.plus(Duration.standardSeconds(5))) .addElements(TimestampedValue.of("bb", base.minus(Duration.standardHours(1)))) .advanceProcessingTime(Duration.standardHours(1)) .advanceWatermarkToInfinity();
/** * Tests that when a processing time timers comes in after a window is expired it does not cause * a spurious output. */ @Test @Category({ValidatesRunner.class, UsesTestStream.class}) public void testCombiningAccumulatingProcessingTime() throws Exception { PCollection<Integer> triggeredSums = p.apply( TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(5, new Instant(5))) .advanceWatermarkTo(new Instant(100)) .advanceProcessingTime(Duration.millis(10)) .advanceWatermarkToInfinity()) .apply( Window.<Integer>into(FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO) .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.millis(10))))) .apply(Sum.integersGlobally().withoutDefaults()); PAssert.that(triggeredSums).containsInAnyOrder(7); p.run(); }
.advanceWatermarkTo(base) .addElements(TimestampedValue.of(KV.of(1, "k1"), base)) .advanceProcessingTime(Duration.standardMinutes(1)) .advanceWatermarkToInfinity();
TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))), TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20)))) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
@Test @Category({NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class}) public void testSimpleProcessingTimerTimer() throws Exception { final String timerId = "foo"; DoFn<KV<String, Integer>, Integer> fn = new DoFn<KV<String, Integer>, Integer>() { @TimerId(timerId) private final TimerSpec spec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME); @ProcessElement public void processElement(@TimerId(timerId) Timer timer, OutputReceiver<Integer> r) { timer.offset(Duration.standardSeconds(1)).setRelative(); r.output(3); } @OnTimer(timerId) public void onTimer(TimeDomain timeDomain, OutputReceiver<Integer> r) { if (timeDomain.equals(TimeDomain.PROCESSING_TIME)) { r.output(42); } } }; TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())) .addElements(KV.of("hello", 37)) .advanceProcessingTime(Duration.standardSeconds(2)) .advanceWatermarkToInfinity(); PCollection<Integer> output = pipeline.apply(stream).apply(ParDo.of(fn)); PAssert.that(output).containsInAnyOrder(3, 42); pipeline.run(); }
@Test @Category({NeedsRunner.class, UsesTestStream.class}) public void testProcessingTimeTrigger() { TestStream<Long> source = TestStream.create(VarLongCoder.of()) .addElements( TimestampedValue.of(1L, new Instant(1000L)), TimestampedValue.of(2L, new Instant(2000L))) .advanceProcessingTime(Duration.standardMinutes(12)) .addElements(TimestampedValue.of(3L, new Instant(3000L))) .advanceProcessingTime(Duration.standardMinutes(6)) .advanceWatermarkToInfinity(); PCollection<Long> sum = p.apply(source) .apply( Window.<Long>configure() .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(5)))) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO)) .apply(Sum.longsGlobally()); PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L); p.run(); }
@Test @Category({NeedsRunner.class, UsesTestStream.class}) public void streamingWrites() throws Exception { TestStream<Mutation> testStream = TestStream.create(SerializableCoder.of(Mutation.class)) .addElements(m(1L), m(2L)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements(m(3L), m(4L)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements(m(5L), m(6L)) .advanceWatermarkToInfinity(); pipeline .apply(testStream) .apply( SpannerIO.write() .withProjectId("test-project") .withInstanceId("test-instance") .withDatabaseId("test-database") .withServiceFactory(serviceFactory)); pipeline.run(); verifyBatches(batch(m(1L), m(2L)), batch(m(3L), m(4L)), batch(m(5L), m(6L))); }
@Parameters(name = "{index}: {0}") public static Iterable<TestStream<?>> data() { return ImmutableList.of( TestStream.create(VarIntCoder.of()).advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(42)) .advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .addElements(TimestampedValue.of(3, new Instant(17))) .advanceWatermarkToInfinity(), TestStream.create(StringUtf8Coder.of()) .advanceProcessingTime(Duration.millis(82)) .advanceWatermarkToInfinity()); }