@Override public PCollection<Row> buildIOReader(PBegin begin) { TestStream.Builder<Row> values = TestStream.create( schema, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Pair<Duration, List<Row>> pair : timestampedRows) { values = values.advanceWatermarkTo(new Instant(0).plus(pair.getKey())); for (int i = 0; i < pair.getValue().size(); i++) { values = values.addElements( TimestampedValue.of( pair.getValue().get(i), new Instant(pair.getValue().get(i).getDateTime(timestampField)))); } } return begin .apply( "MockedUnboundedTable_" + COUNTER.incrementAndGet(), values.advanceWatermarkToInfinity()) .setRowSchema(getSchema()); } }
/** * Builds an unbounded {@link PCollection} in {@link Pipeline} set by {@link * #inPipeline(Pipeline)}. * * <p>If timestamp field was set with {@link #withTimestampField(String)} then watermark will be * advanced to the values from that field. */ public PCollection<Row> buildUnbounded() { checkArgument(pipeline != null); checkArgument(rows.size() > 0); if (type == null) { type = rows.get(0).getSchema(); } TestStream.Builder<Row> values = TestStream.create( type, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rows) { if (timestampField != null) { values = values.advanceWatermarkTo(new Instant(row.getDateTime(timestampField))); } values = values.addElements(row); } return PBegin.in(pipeline).apply("unboundedPCollection", values.advanceWatermarkToInfinity()); } }
private PCollection<Row> prepareUnboundedPCollection2() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); Row row = rowsInTableA.get(0); values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); return PBegin.in(pipeline) .apply("unboundedInput2", values.advanceWatermarkToInfinity()) .apply( "unboundedInput2.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); } }
.advanceWatermarkTo(new Instant(0)) .addElements(KV.of("hello", "input1")) .addElements(KV.of("hello", "input2"))
.advanceWatermarkTo(base) .addElements("aa") .advanceWatermarkTo(base.plus(Duration.standardSeconds(5))) .addElements(TimestampedValue.of("bb", base.minus(Duration.standardHours(1)))) .advanceProcessingTime(Duration.standardHours(1))
TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))) .advanceWatermarkTo( baseTime.plus(TEAM_WINDOW_DURATION).minus(Duration.standardMinutes(1))) event(TestUser.BLUE_TWO, 2, Duration.standardSeconds(90)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))) .advanceWatermarkTo( baseTime.plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))) .advanceWatermarkToInfinity();
TestStream<String> values = TestStream.create(StringUtf8Coder.of()) .advanceWatermarkTo(base) .addElements( TimestampedValue.of("k1", base),
/** * Tests that when a processing time timers comes in after a window is expired it does not cause * a spurious output. */ @Test @Category({ValidatesRunner.class, UsesTestStream.class}) public void testCombiningAccumulatingProcessingTime() throws Exception { PCollection<Integer> triggeredSums = p.apply( TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(5, new Instant(5))) .advanceWatermarkTo(new Instant(100)) .advanceProcessingTime(Duration.millis(10)) .advanceWatermarkToInfinity()) .apply( Window.<Integer>into(FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO) .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.millis(10))))) .apply(Sum.integersGlobally().withoutDefaults()); PAssert.that(triggeredSums).containsInAnyOrder(7); p.run(); }
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly per window and per key. */ @Test public void testScoreUpdatesPerWindow() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_ONE, 50, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 50, Duration.standardMinutes(2)), event(TestUser.RED_ONE, 50, Duration.standardMinutes(3)), event(TestUser.RED_ONE, 60, Duration.standardMinutes(6)), event(TestUser.RED_TWO, 60, Duration.standardMinutes(7))) .advanceWatermarkToInfinity(); Duration teamWindowDuration = Duration.standardMinutes(5); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents) .apply(Window.<KV<String, GameActionInfo>>into(FixedWindows.of(teamWindowDuration))) .apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); String blueTeam = TestUser.BLUE_ONE.getTeam(); IntervalWindow window1 = new IntervalWindow(baseTime, teamWindowDuration); IntervalWindow window2 = new IntervalWindow(window1.end(), teamWindowDuration); PAssert.that(teamScores).inWindow(window1).containsInAnyOrder(KV.of(redTeam, 100)); PAssert.that(teamScores).inWindow(window2).containsInAnyOrder(KV.of(redTeam, 120)); p.run().waitUntilFinish(); }
.advanceWatermarkTo(new Instant(5)) .addElements(KV.of("hello", 37)) .advanceWatermarkTo(new Instant(0).plus(Duration.standardSeconds(1).plus(1))) .advanceWatermarkToInfinity();
TestStream.create(AvroCoder.of(GameActionInfo.class)) .advanceWatermarkTo(baseTime) event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(3))) .advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3)))
TestStream<KV<Integer, String>> values = TestStream.create(KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of())) .advanceWatermarkTo(base) .addElements(TimestampedValue.of(KV.of(1, "k1"), base)) .advanceProcessingTime(Duration.standardMinutes(1))
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly for multiple teams. */ @Test public void testScoreUpdatesPerTeam() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_ONE, 50, Duration.standardSeconds(10)), event(TestUser.RED_TWO, 50, Duration.standardSeconds(20)), event(TestUser.BLUE_ONE, 70, Duration.standardSeconds(30)), event(TestUser.BLUE_TWO, 80, Duration.standardSeconds(40)), event(TestUser.BLUE_TWO, 50, Duration.standardSeconds(50))) .advanceWatermarkToInfinity(); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); String blueTeam = TestUser.BLUE_ONE.getTeam(); PAssert.that(teamScores) .inWindow(GlobalWindow.INSTANCE) .containsInAnyOrder(KV.of(redTeam, 100), KV.of(blueTeam, 150), KV.of(blueTeam, 200)); p.run().waitUntilFinish(); }
@Test @Category({NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class}) public void testEventTimeTimerUnbounded() throws Exception { final String timerId = "foo"; DoFn<KV<String, Integer>, Integer> fn = new DoFn<KV<String, Integer>, Integer>() { @TimerId(timerId) private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME); @ProcessElement public void processElement(@TimerId(timerId) Timer timer, OutputReceiver<Integer> r) { timer.offset(Duration.standardSeconds(1)).setRelative(); r.output(3); } @OnTimer(timerId) public void onTimer(OutputReceiver<Integer> r) { r.output(42); } }; TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())) .advanceWatermarkTo(new Instant(0)) .addElements(KV.of("hello", 37)) .advanceWatermarkTo(new Instant(0).plus(Duration.standardSeconds(1))) .advanceWatermarkToInfinity(); PCollection<Integer> output = pipeline.apply(stream).apply(ParDo.of(fn)); PAssert.that(output).containsInAnyOrder(3, 42); pipeline.run(); }
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly for one team. */ @Test public void testScoreUpdatesOneTeam() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)), event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)), event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)), event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)), event(TestUser.RED_TWO, 201, Duration.standardSeconds(50))) .advanceWatermarkToInfinity(); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); PAssert.that(teamScores) .inWindow(GlobalWindow.INSTANCE) .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401)); p.run().waitUntilFinish(); }
@Test @Category({ValidatesRunner.class, UsesTestStream.class}) public void testReshuffleWithTimestampsStreaming() { TestStream<Long> stream = TestStream.create(VarLongCoder.of()) .advanceWatermarkTo(new Instant(0L).plus(Duration.standardDays(48L))) .addElements( TimestampedValue.of(0L, new Instant(0L)), TimestampedValue.of(1L, new Instant(0L).plus(Duration.standardDays(48L))), TimestampedValue.of( 2L, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(48L)))) .advanceWatermarkToInfinity(); PCollection<KV<String, Long>> input = pipeline .apply(stream) .apply(WithKeys.of("")) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10L)))); PCollection<KV<String, Long>> reshuffled = input.apply(Reshuffle.of()); PAssert.that(reshuffled.apply(Values.create())).containsInAnyOrder(0L, 1L, 2L); pipeline.run(); } }
private PCollection<Row> prepareUnboundedPCollection1() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rowsInTableA) { values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); } return PBegin.in(pipeline) .apply("unboundedInput1", values.advanceWatermarkToInfinity()) .apply( "unboundedInput1.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); }
@Parameters(name = "{index}: {0}") public static Iterable<TestStream<?>> data() { return ImmutableList.of( TestStream.create(VarIntCoder.of()).advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(42)) .advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .addElements(TimestampedValue.of(3, new Instant(17))) .advanceWatermarkToInfinity(), TestStream.create(StringUtf8Coder.of()) .advanceProcessingTime(Duration.millis(82)) .advanceWatermarkToInfinity()); }
@Test public void testAdvanceWatermarkNonMonotonicThrows() { Builder<Integer> stream = TestStream.create(VarIntCoder.of()).advanceWatermarkTo(new Instant(0L)); thrown.expect(IllegalArgumentException.class); stream.advanceWatermarkTo(new Instant(-1L)); }
@Test public void testAdvanceWatermarkEqualToPositiveInfinityThrows() { Builder<Integer> stream = TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)); thrown.expect(IllegalArgumentException.class); stream.advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE); }