/** * Runs the {@link WindowFn} over the provided input, returning a map of windows to the timestamps * in those windows. */ public static <T, W extends BoundedWindow> Map<W, Set<String>> runWindowFn( WindowFn<T, W> windowFn, List<Long> timestamps) throws Exception { List<TimestampedValue<T>> timestampedValues = new ArrayList<>(); for (Long timestamp : timestamps) { timestampedValues.add(TimestampedValue.of((T) null, new Instant(timestamp))); } return runWindowFnWithValue(windowFn, timestampedValues); }
@Test @Category(NeedsRunner.class) public void timestampsSucceeds() { PCollection<String> timestamped = pipeline.apply( Create.timestamped( TimestampedValue.of("foo", new Instant(0L)), TimestampedValue.of("bar", new Instant(1L)))); PCollection<TimestampedValue<String>> reified = timestamped.apply(Reify.timestamps()); PAssert.that(reified) .containsInAnyOrder( TimestampedValue.of("foo", new Instant(0)), TimestampedValue.of("bar", new Instant(1))); pipeline.run(); }
@Test public void testAddInputSameTimestamp() { TimestampedValue<Long> accum = TimestampedValue.of(100L, INSTANT); TimestampedValue<Long> input = TimestampedValue.of(200L, INSTANT); assertThat( "Latest for values with the same timestamp is chosen arbitrarily", fn.addInput(accum, input), isOneOf(accum, input)); }
@Test public void testEquality() { new EqualsTester() .addEqualityGroup( TimestampedValue.of("foo", new Instant(1000)), TimestampedValue.of("foo", new Instant(1000))) .addEqualityGroup(TimestampedValue.of("foo", new Instant(2000))) .addEqualityGroup(TimestampedValue.of("bar", new Instant(1000))) .addEqualityGroup( TimestampedValue.of("foo", BoundedWindow.TIMESTAMP_MIN_VALUE), TimestampedValue.atMinimumTimestamp("foo")) .testEquals(); }
@Test public void testCreateTimestampedDefaultOutputCoderUsingCoder() throws Exception { Coder<Record> coder = new RecordCoder(); Create.TimestampedValues<Record> values = Create.timestamped( TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.of(new Record2(), new Instant(0))) .withCoder(coder); assertThat(p.apply(values).getCoder(), equalTo(coder)); }
@Test @Category(NeedsRunner.class) public void testCreateParameterizedType() throws Exception { PCollection<TimestampedValue<String>> output = p.apply( Create.of( TimestampedValue.of("a", new Instant(0)), TimestampedValue.of("b", new Instant(0)))); PAssert.that(output) .containsInAnyOrder( TimestampedValue.of("a", new Instant(0)), TimestampedValue.of("b", new Instant(0))); p.run(); } /** An unserializable class to demonstrate encoding of elements. */
@Test public void processTimestampedElement() throws Exception { try (DoFnTester<Long, TimestampedValue<Long>> tester = DoFnTester.of(new ReifyTimestamps())) { TimestampedValue<Long> input = TimestampedValue.of(1L, new Instant(100)); tester.processTimestampedElement(input); assertThat(tester.takeOutputElements(), contains(input)); } }
@Test public void testCreateTimestampedPolymorphicType() throws Exception { thrown.expect(RuntimeException.class); thrown.expectMessage(Matchers.containsString("Unable to infer a coder")); // Create won't infer a default coder in this case. PCollection<Record> c = p.apply( Create.timestamped( TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.of(new Record2(), new Instant(0)))); p.run(); throw new RuntimeException("Coder: " + c.getCoder()); }
/** runs {@link WindowFn#assignWindows(WindowFn.AssignContext)}. */ public static <T, W extends BoundedWindow> Collection<W> assignedWindows( WindowFn<T, W> windowFn, long timestamp) throws Exception { return assignedWindowsWithValue( windowFn, TimestampedValue.of((T) null, new Instant(timestamp))); }
@Test @Category(NeedsRunner.class) public void testGloballyEventTimestamp() { PCollection<String> output = p.apply( Create.timestamped( TimestampedValue.of("foo", new Instant(100)), TimestampedValue.of("bar", new Instant(300)), TimestampedValue.of("baz", new Instant(200)))) .apply(Latest.globally()); PAssert.that(output).containsInAnyOrder("bar"); p.run(); }
@Test public void falseAfterEndOfWindow() throws Exception { triggerTester.injectElements(TimestampedValue.of(1, new Instant(1))); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0).plus(Duration.standardMinutes(5))); assertThat(triggerTester.shouldFire(window), is(false)); triggerTester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); assertThat(triggerTester.shouldFire(window), is(false)); } }
@Test public void testCreateTimestampedDefaultOutputCoderUsingTypeDescriptor() throws Exception { Coder<Record> coder = new RecordCoder(); p.getCoderRegistry().registerCoderForClass(Record.class, coder); Create.TimestampedValues<Record> values = Create.timestamped( TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.of(new Record2(), new Instant(0))) .withType(new TypeDescriptor<Record>() {}); assertThat(p.apply(values).getCoder(), equalTo(coder)); }
/** * Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the * proper bound. */ public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimes( WindowFn<T, W> windowFn, long timestamp) throws Exception { validateNonInterferingOutputTimesWithValue( windowFn, TimestampedValue.of((T) null, new Instant(timestamp))); } /**
@Test @Category(NeedsRunner.class) public void testCreateTimestamped() { List<TimestampedValue<String>> data = Arrays.asList( TimestampedValue.of("a", new Instant(1L)), TimestampedValue.of("b", new Instant(2L)), TimestampedValue.of("c", new Instant(3L))); PCollection<String> output = p.apply(Create.timestamped(data)).apply(ParDo.of(new PrintTimestamps())); PAssert.that(output).containsInAnyOrder("a:1", "b:2", "c:3"); p.run(); }
@Test public void peekValuesInWindow() throws Exception { try (DoFnTester<Long, String> tester = DoFnTester.of(new CounterDoFn())) { tester.startBundle(); tester.processElement(1L); tester.processElement(2L); tester.finishBundle(); assertThat( tester.peekOutputElementsInWindow(GlobalWindow.INSTANCE), containsInAnyOrder( TimestampedValue.of("1", new Instant(1000L)), TimestampedValue.of("2", new Instant(2000L)))); assertThat( tester.peekOutputElementsInWindow(new IntervalWindow(new Instant(0L), new Instant(10L))), Matchers.emptyIterable()); } }
/** * Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and * verifies that result of {@link WindowFn#getOutputTime windowFn.getOutputTime} for later windows * (as defined by {@code maxTimestamp} won't prevent the watermark from passing the end of earlier * windows. * * <p>This verifies that overlapping windows don't interfere at all. Depending on the {@code * windowFn} this may be stricter than desired. */ public static <T, W extends BoundedWindow> void validateGetOutputTimestamp( WindowFn<T, W> windowFn, long timestamp) throws Exception { validateGetOutputTimestampWithValue( windowFn, TimestampedValue.of((T) null, new Instant(timestamp))); }
@Test @Category(ValidatesRunner.class) public void testMergingWindowing() { PCollection<String> input = p.apply( Create.timestamped( TimestampedValue.of("a", new Instant(1)), TimestampedValue.of("a", new Instant(5)), TimestampedValue.of("a", new Instant(20)))); PCollection<String> output = input.apply(new WindowedCount(Sessions.withGapDuration(new Duration(10)))); PAssert.that(output).containsInAnyOrder(output("a", 2, 1, 1, 15), output("a", 1, 20, 20, 30)); p.run(); }