/** Returns the largest timestamp that can be included in this window. */ @Override public Instant maxTimestamp() { // end not inclusive return end.minus(1); }
@Override public Instant apply(Instant point) { long millisSinceStart = new Duration(offset, point).getMillis() % size.getMillis(); return millisSinceStart == 0 ? point : point.plus(size).minus(millisSinceStart); }
@VisibleForTesting Instant getWatermark(PartitionContext ctx, Instant now) { if (maxEventTimestamp.isAfter(now)) { return now.minus(maxDelay); // (a) above. } else if (ctx.getMessageBacklog() == 0 && ctx.getBacklogCheckTime().minus(maxDelay).isAfter(maxEventTimestamp) // Idle && maxEventTimestamp.getMillis() > 0) { // Read at least one record with positive timestamp. return ctx.getBacklogCheckTime().minus(maxDelay); } else { return maxEventTimestamp.minus(maxDelay); } } }
@ProcessElement public void processElement(ProcessContext c) { // Generate a timestamp that falls somewhere in the past RAND_RANGE hours. long randMillis = (long) (Math.random() * RAND_RANGE.getMillis()); // Instant randomTimestamp = minTimestamp.plus(randMillis); Instant randomTimestamp = minTimestamp.minus(randMillis); // Concept #2: Set the data element with that timestamp. c.outputWithTimestamp(c.element(), new Instant(randomTimestamp)); } }
public Collection<IntervalWindow> assignWindows(Instant timestamp) { List<IntervalWindow> windows = new ArrayList<>((int) (size.getMillis() / period.getMillis())); long lastStart = lastStartFor(timestamp); for (long start = lastStart; start > timestamp.minus(size).getMillis(); start -= period.getMillis()) { windows.add(new IntervalWindow(new Instant(start), size)); } return windows; }
/** Return the last start of a sliding window that contains the timestamp. */ private long lastStartFor(Instant timestamp) { return timestamp.getMillis() - timestamp.plus(period).minus(offset).getMillis() % period.getMillis(); }
@Override public IntervalWindow getSideInputWindow(BoundedWindow mainWindow) { if (mainWindow instanceof GlobalWindow) { throw new IllegalArgumentException( "Attempted to get side input window for GlobalWindow from non-global WindowFn"); } long lastStart = lastStartFor(mainWindow.maxTimestamp().minus(size)); return new IntervalWindow(new Instant(lastStart + period.getMillis()), size); } };
@Override public Instant getWatermark(PartitionContext context) { if (context.getMessageBacklog() == 0) { // The reader is caught up. May need to advance the watermark. Instant idleWatermark = context.getBacklogCheckTime().minus(IDLE_WATERMARK_DELTA); if (idleWatermark.isAfter(currentWatermark)) { currentWatermark = idleWatermark; } } // else, there is backlog (or is unknown). Do not advance the watermark. return currentWatermark; } }
@Override public IntervalWindow decode(InputStream inStream) throws IOException, CoderException { Instant end = instantCoder.decode(inStream); ReadableDuration duration = durationCoder.decode(inStream); return new IntervalWindow(end.minus(duration), end); }
@Override public Instant getWatermark(PartitionContext context) { if (context.getMessageBacklog() == 0) { // The reader is caught up. May need to advance the watermark. Instant idleWatermark = context.getBacklogCheckTime().minus(IDLE_WATERMARK_DELTA); if (idleWatermark.isAfter(currentWatermark)) { currentWatermark = idleWatermark; } } // else, there is backlog (or is unknown). Do not advance the watermark. return currentWatermark; } }
public Instant getCurrent(BooleanSupplier shardsUpToDate) { Instant now = Instant.now(); Instant readMin = getMinReadTimestamp(now); if (readMin == null) { if (shardsUpToDate.getAsBoolean()) { updateLastWatermark(now.minus(SAMPLE_PERIOD), now); } } else if (shouldUpdate(now)) { updateLastWatermark(readMin, now); } return lastWatermark; }
@Test public void watermarkStartsAtSamplePeriodBehindNowIfShardsUpToDate() { assertThat(watermark.getCurrent(SHARDS_UP_TO_DATE)) .isEqualTo(now.minus(KinesisWatermark.SAMPLE_PERIOD)); }
protected void checkTimestamp(WindowedValue<IN> ref, Instant timestamp) { if (timestamp.isBefore(ref.getTimestamp().minus(doFn.getAllowedTimestampSkew()))) { throw new IllegalArgumentException(String.format( "Cannot output with timestamp %s. Output timestamps must be no earlier than the " + "timestamp of the current input (%s) minus the allowed skew (%s). See the " + "DoFn#getAllowedTimestmapSkew() Javadoc for details on changing the allowed skew.", timestamp, ref.getTimestamp(), PeriodFormat.getDefault().print(doFn.getAllowedTimestampSkew().toPeriod()))); } }
@Test public void watermarkIsUpdatedToFirstRecordTimestamp() { Instant firstTimestamp = now.minus(Duration.standardHours(1)); watermark.update(firstTimestamp); assertThat(watermark.getCurrent(SHARDS_IRRELEVANT)).isEqualTo(firstTimestamp); }
@Test public void watermarkDoesNotGoBackward() { watermark.update(now); for (int i = 0; i <= KinesisWatermark.MIN_MESSAGES * 2; ++i) { watermark.update(now.minus(Duration.millis(i))); assertThat(watermark.getCurrent(SHARDS_IRRELEVANT)).isEqualTo(now); } }
@Test public void watermarkIsUpdatedToRecentRecordTimestampIfItIsOlderThanUpdateThreshold() { Instant firstTimestamp = now.minus(Duration.standardHours(1)); watermark.update(firstTimestamp); assertThat(watermark.getCurrent(SHARDS_IRRELEVANT)).isEqualTo(firstTimestamp); Instant timeAfterWatermarkUpdateThreshold = now.plus(KinesisWatermark.UPDATE_THRESHOLD.plus(Duration.millis(1))); setCurrentTimeTo(timeAfterWatermarkUpdateThreshold); Instant nextTimestamp = timeAfterWatermarkUpdateThreshold.plus(Duration.millis(1)); watermark.update(nextTimestamp); assertThat(watermark.getCurrent(SHARDS_IRRELEVANT)).isEqualTo(nextTimestamp); }
/** Tests that the last hour of the universe in fact ends at the end of time. */ @Test public void testEndOfTime() { Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp(); FixedWindows windowFn = FixedWindows.of(Duration.standardHours(1)); IntervalWindow truncatedWindow = windowFn.assignWindow(endOfGlobalWindow.minus(1)); assertThat(truncatedWindow.maxTimestamp(), equalTo(endOfGlobalWindow)); }
@Test public void testExplodeWindowsInOneWindowEquals() { Instant now = Instant.now(); BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L)); WindowedValue<String> value = WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING); assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value)); }
@Test public void testAdvanceWatermarkEqualToPositiveInfinityThrows() { Builder<Integer> stream = TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)); thrown.expect(IllegalArgumentException.class); stream.advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE); }
@Test public void testElementAtPositiveInfinityThrows() { Builder<Integer> stream = TestStream.create(VarIntCoder.of()) .addElements(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L))); thrown.expect(IllegalArgumentException.class); stream.addElements(TimestampedValue.of(1, BoundedWindow.TIMESTAMP_MAX_VALUE)); }