@Override public final Watermark getCurrentWatermark() { // this guarantees that the watermark never goes backwards. long potentialWM = currentMaxTimestamp - maxOutOfOrderness; if (potentialWM >= lastEmittedWatermark) { lastEmittedWatermark = potentialWM; } return new Watermark(lastEmittedWatermark); }
@Override protected boolean allowWatermark(Watermark mark) { // allow Long.MAX_VALUE since this is the special end-watermark that for example the Kafka source emits return mark.getTimestamp() == Long.MAX_VALUE && nextWatermarkTime != Long.MAX_VALUE; }
public void processWatermark1(Watermark mark) throws Exception { input1Watermark = mark.getTimestamp(); long newMin = Math.min(input1Watermark, input2Watermark); if (newMin > combinedWatermark) { combinedWatermark = newMin; processWatermark(new Watermark(combinedWatermark)); } }
/** * Verify no StreamRecord is equal to or later than any watermarks. This is checked over the * order of the elements * * @param elements An iterable containing StreamRecords and watermarks */ public static void assertNoLateRecords(Iterable<Object> elements) { // check that no watermark is violated long highestWatermark = Long.MIN_VALUE; for (Object elem : elements) { if (elem instanceof Watermark) { highestWatermark = ((Watermark) elem).asWatermark().getTimestamp(); } else if (elem instanceof StreamRecord) { boolean dataIsOnTime = highestWatermark < ((StreamRecord) elem).getTimestamp(); Assert.assertTrue("Late data was emitted after join", dataIsOnTime); } } } }
@Override protected void handleWatermark(Watermark mark) { if (mark.equals(Watermark.MAX_WATERMARK)) { output.emitWatermark(mark); lastWatermark = Long.MAX_VALUE; } } }
public void processWatermark2(Watermark mark) throws Exception { input2Watermark = mark.getTimestamp(); long newMin = Math.min(input1Watermark, input2Watermark); if (newMin > combinedWatermark) { combinedWatermark = newMin; processWatermark(new Watermark(combinedWatermark)); } }
@Override public Watermark checkAndGetNextWatermark(Integer lastElement, long extractedTimestamp) { return new Watermark(counter - 1); } }
@Override public void processWatermark(Watermark mark) throws Exception { // if we receive a Long.MAX_VALUE watermark we forward it since it is used // to signal the end of input and to not block watermark progress downstream if (mark.getTimestamp() == Long.MAX_VALUE && mark.getTimestamp() > currentWatermark) { currentWatermark = Long.MAX_VALUE; output.emitWatermark(mark); } } }
/** *Checks whether a new per-partition watermark is also a new cross-partition watermark. */ private void updateMinPunctuatedWatermark(Watermark nextWatermark) { if (nextWatermark.getTimestamp() > maxWatermarkSoFar) { long newMin = Long.MAX_VALUE; for (KafkaTopicPartitionState<?> state : subscribedPartitionStates) { @SuppressWarnings("unchecked") final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState = (KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) state; newMin = Math.min(newMin, withWatermarksState.getCurrentPartitionWatermark()); } // double-check locking pattern if (newMin > maxWatermarkSoFar) { synchronized (checkpointLock) { if (newMin > maxWatermarkSoFar) { maxWatermarkSoFar = newMin; sourceContext.emitWatermark(new Watermark(newMin)); } } } } }
@Override public final Watermark getCurrentWatermark() { return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); }
@Nullable public Watermark checkAndGetNewWatermark(T record, long timestamp) { Watermark mark = timestampsAndWatermarks.checkAndGetNextWatermark(record, timestamp); if (mark != null && mark.getTimestamp() > partitionWatermark) { partitionWatermark = mark.getTimestamp(); return mark; } else { return null; } }
@Override public void processWatermark(Watermark mark) throws Exception { output.emitWatermark(new Watermark(mark.getTimestamp() * 2)); } }
@Nullable @Override public Watermark getCurrentWatermark() { return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); } }
public void advanceWatermark(Watermark watermark) throws Exception { for (InternalTimerServiceImpl<?, ?> service : timerServices.values()) { service.advanceWatermark(watermark.getTimestamp()); } }
@Override public Watermark getCurrentWatermark() { // make sure timestamps are monotonously increasing, even when the system clock re-syncs final long now = Math.max(System.currentTimeMillis(), maxTimestamp); maxTimestamp = now; return new Watermark(now - 1); } }
@Override public void processWatermark(Watermark mark) throws Exception { super.processWatermark(mark); this.currentWatermark = mark.getTimestamp(); }
@Override public Watermark checkAndGetNextWatermark(Tuple3<String, String, Integer> lastElement, long extractedTimestamp) { return new Watermark(lastElement.f2 - 1); } }
public long getCurrentWatermarkTimestamp() { Watermark wm = timestampsAndWatermarks.getCurrentWatermark(); if (wm != null) { partitionWatermark = Math.max(partitionWatermark, wm.getTimestamp()); } return partitionWatermark; }
@Override public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) { return new Watermark(extractedTimestamp - 1); } })
@Override public void processWatermark(Watermark mark) throws Exception { super.processWatermark(mark); currentWatermark = mark.getTimestamp(); }