.advanceWatermarkTo(new Instant(0)) .addElements( firstWindowArray[0], Arrays.copyOfRange(firstWindowArray, 1, firstWindowArray.length)) .advanceWatermarkTo(new Instant(0).plus(Duration.standardMinutes(1))) .addElements( secondWindowArray[0], Arrays.copyOfRange(secondWindowArray, 1, secondWindowArray.length)) .advanceWatermarkToInfinity();
TestStream<Integer> testStream = TestStream.create(VarIntCoder.of()) .addElements(1, 2, 3) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.atMinimumTimestamp(4), TimestampedValue.atMinimumTimestamp(5), TimestampedValue.atMinimumTimestamp(6)) .advanceProcessingTime(Duration.standardMinutes(10)) .advanceWatermarkToInfinity(); PCollection<Integer> streamVals = p.apply(new DirectTestStream<>(runner, testStream));
/** * Adds the specified elements to the source with the provided timestamps. * * @return A {@link TestStream.Builder} like this one that will add the provided elements after * all earlier events have completed. */ @SafeVarargs public final Builder<T> addElements( TimestampedValue<T> element, TimestampedValue<T>... elements) { checkArgument( element.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE), "Elements must have timestamps before %s. Got: %s", BoundedWindow.TIMESTAMP_MAX_VALUE, element.getTimestamp()); for (TimestampedValue<T> multiElement : elements) { checkArgument( multiElement.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE), "Elements must have timestamps before %s. Got: %s", BoundedWindow.TIMESTAMP_MAX_VALUE, multiElement.getTimestamp()); } ImmutableList<Event<T>> newEvents = ImmutableList.<Event<T>>builder() .addAll(events) .add(ElementEvent.add(element, elements)) .build(); return new Builder<>(coder, newEvents, currentWatermark); }
TestStream.Builder<KV<String, String>> streamBuilder = TestStream.create(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())) .advanceWatermarkTo(startInstant); long offset = 0L; for (KV<String, String> element : data) { streamBuilder = streamBuilder.addElements( TimestampedValue.of( element, TestStream<KV<String, String>> stream = streamBuilder .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(windowDuration - 1))) .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(windowDuration + 1))) .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(NUM_ELEMENTS))) .advanceWatermarkToInfinity();
TestStream<Integer> source = TestStream.create(VarIntCoder.of()) .addElements( TimestampedValue.of(1, instant), TimestampedValue.of(2, instant), TimestampedValue.of(3, instant)) .advanceWatermarkTo(instant.plus(Duration.standardMinutes(6))) .addElements(TimestampedValue.of(4, instant), TimestampedValue.of(5, instant)) .advanceWatermarkTo(instant.plus(Duration.standardMinutes(20))) .addElements( TimestampedValue.of(-1, instant), TimestampedValue.of(-2, instant), TimestampedValue.of(-3, instant)) .advanceWatermarkToInfinity();
TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8))) .advanceProcessingTime(Duration.standardMinutes(10)) .advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))) .addElements( event(TestUser.RED_ONE, 3, Duration.standardMinutes(1)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))) .advanceWatermarkTo(firstWindowCloses.minus(Duration.standardMinutes(1))) .addElements( event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), .advanceProcessingTime(Duration.standardMinutes(12)) .addElements( event(TestUser.RED_TWO, 9, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 1, Duration.standardMinutes(3))) .advanceWatermarkToInfinity();
TestStream<Long> source = TestStream.create(VarLongCoder.of()) .addElements(TimestampedValue.of(1L, new Instant(1000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane .addElements(TimestampedValue.of(2L, new Instant(2000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane .addElements(TimestampedValue.of(3L, new Instant(3000L))) .advanceProcessingTime(Duration.standardMinutes(6)) // Fire early pane .advanceWatermarkToInfinity(); // Fire on-time pane
TestStream.create(AvroCoder.of(GameActionInfo.class)) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_ONE, 2, Duration.standardMinutes(1))) .advanceProcessingTime(Duration.standardMinutes(10)) .addElements(event(TestUser.RED_TWO, 5, Duration.standardMinutes(3))) .advanceProcessingTime(Duration.standardMinutes(12)) .addElements(event(TestUser.BLUE_TWO, 3, Duration.standardSeconds(22))) .advanceProcessingTime(Duration.standardMinutes(10)) .addElements( event(TestUser.RED_ONE, 4, Duration.standardMinutes(4)), event(TestUser.BLUE_TWO, 2, Duration.standardMinutes(2))) .advanceWatermarkToInfinity();
TestStream.create( inputSchema, SerializableFunctions.identity(), SerializableFunctions.identity()) .addElements( Row.withSchema(inputSchema) .addValues(1, parseTimestampWithoutTimeZone("2017-01-01 01:01:01")) .addValues(2, parseTimestampWithoutTimeZone("2017-01-01 01:01:01")) .build()) .addElements( Row.withSchema(inputSchema) .addValues(3, parseTimestampWithoutTimeZone("2017-01-01 01:01:01")) .build()) .addElements( Row.withSchema(inputSchema) .addValues(4, parseTimestampWithoutTimeZone("2017-01-01 01:01:01")) .build()) .advanceWatermarkToInfinity());
Duration processingTimeDelta = new Duration(lastProcessingTime, event.processingTime); if (processingTimeDelta.getMillis() > 0) { stream = stream.advanceProcessingTime(processingTimeDelta); stream = stream.addElements(event.element); } else { stream = stream.advanceWatermarkTo(event.watermarkUpdate); return p.apply(name, stream.advanceWatermarkToInfinity());
TestStream<String> stream = TestStream.create(StringUtf8Coder.of()) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.of("firstPane", new Instant(100)), TimestampedValue.of("alsoFirstPane", new Instant(200))) .addElements(TimestampedValue.of("onTimePane", new Instant(500))) .advanceWatermarkTo(new Instant(1001L)) .addElements( TimestampedValue.of("finalLatePane", new Instant(750)), TimestampedValue.of("alsoFinalLatePane", new Instant(250))) .advanceWatermarkToInfinity();
.addElements( event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)) .advanceProcessingTime(Duration.standardMinutes(7)) .addElements( event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))) .advanceProcessingTime(Duration.standardMinutes(5)) .advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(Duration.standardHours(12))) .addElements( event(TestUser.RED_ONE, 3, Duration.standardMinutes(7)), event(TestUser.RED_ONE, 2, (ALLOWED_LATENESS).plus(Duration.standardHours(13)))) .advanceProcessingTime(Duration.standardMinutes(6)) .addElements(event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(12))) .advanceProcessingTime(Duration.standardMinutes(20)) .advanceWatermarkToInfinity();
.advanceWatermarkTo(START_TIME) .addElements(event(FIRST_WIN_WORDS.get(0), 2L)) .advanceWatermarkTo(START_TIME.plus(Duration.standardSeconds(27L))) .addElements( event(FIRST_WIN_WORDS.get(1), 25L), event(FIRST_WIN_WORDS.get(2), 18L), event(FIRST_WIN_WORDS.get(3), 28L)) .advanceWatermarkTo(START_TIME.plus(Duration.standardSeconds(65L))) .addElements(event(SECOND_WIN_WORDS.get(0), 61L), event(SECOND_WIN_WORDS.get(1), 63L)) .advanceWatermarkToInfinity();
TestStream<String> values = TestStream.create(StringUtf8Coder.of()) .advanceWatermarkTo(base) .addElements( TimestampedValue.of("k1", base), TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))), TimestampedValue.of("k5", base.plus(Duration.standardSeconds(70))), TimestampedValue.of("k6", base.plus(Duration.standardSeconds(80)))) .advanceWatermarkToInfinity();
TestStream<GameActionInfo> infos = TestStream.create(AvroCoder.of(GameActionInfo.class)) .addElements( event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)) .advanceWatermarkTo(window.maxTimestamp()) .addElements( event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))) .advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION)) .advanceWatermarkTo( baseTime .plus(ALLOWED_LATENESS) .addElements( event( TestUser.BLUE_TWO, 3, TEAM_WINDOW_DURATION.minus(Duration.standardSeconds(5))), event(TestUser.RED_ONE, 7, Duration.standardMinutes(4))) .advanceWatermarkToInfinity(); PCollection<KV<String, Integer>> teamScores = p.apply(infos).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))) .advanceWatermarkTo( baseTime.plus(TEAM_WINDOW_DURATION).minus(Duration.standardMinutes(1))) .addElements( event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.BLUE_TWO, 2, Duration.standardSeconds(90)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))) .advanceWatermarkTo( baseTime.plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))) .advanceWatermarkToInfinity(); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents)
.addElements( elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) .addElements( elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) .advanceWatermarkToInfinity();
.advanceWatermarkTo(new Instant(0)) .addElements(KV.of("hello", "input1")) .addElements(KV.of("hello", "input2")) .advanceWatermarkToInfinity();
.advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))) .addElements(KV.of("hello", 37)) .advanceWatermarkToInfinity();
.advanceWatermarkTo(new Instant(5)) .addElements(KV.of("hello", 37)) .advanceWatermarkTo(new Instant(0).plus(Duration.standardSeconds(1).plus(1))) .advanceWatermarkToInfinity();