private static WatermarkUpdate updateAndTrace(String name, Instant oldTime, Instant currentTime) { WatermarkUpdate res = WatermarkUpdate.fromTimestamps(oldTime, currentTime); if (res.isAdvanced()) { WindowTracing.debug("Watermark {} advanced from {} to {}", name, oldTime, currentTime); } return res; }
WindowTracing.debug( "{}: Dropping element at {} for key: {}; " + "window: {} since it is too far behind inputWatermark: {}",
WindowTracing.debug( "{}: Dropping element at {} for key: {}; " + "window: {} since it is too far behind inputWatermark: {}",
WindowTracing.debug( "{}: Dropping element at {} for key:{}; window:{} " + "since too far behind inputWatermark:{}; outputWatermark:{}",
@Override public void processElement(WindowedValue<InputT> input) { // StatefulDoFnRunner always observes windows, so we need to explode for (WindowedValue<InputT> value : input.explodeWindows()) { BoundedWindow window = value.getWindows().iterator().next(); if (isLate(window)) { // The element is too late for this window. droppedDueToLateness.inc(); WindowTracing.debug( "StatefulDoFnRunner.processElement: Dropping element at {}; window:{} " + "since too far behind inputWatermark:{}", input.getTimestamp(), window, cleanupTimer.currentInputWatermarkTime()); } else { cleanupTimer.setForWindow(window); doFnRunner.processElement(value); } } }
@Override public void onTimer( String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) { if (cleanupTimer.isForWindow(timerId, window, timestamp, timeDomain)) { stateCleaner.clearForWindow(window); // There should invoke the onWindowExpiration of DoFn } else { // An event-time timer can never be late because we don't allow setting timers after GC time. // Ot can happen that a processing-time time fires for a late window, we need to ignore // this. if (!timeDomain.equals(TimeDomain.EVENT_TIME) && isLate(window)) { // don't increment the dropped counter, only do that for elements WindowTracing.debug( "StatefulDoFnRunner.onTimer: Ignoring processing-time timer at {}; window:{} " + "since window is too far behind inputWatermark:{}", timestamp, window, cleanupTimer.currentInputWatermarkTime()); } else { doFnRunner.onTimer(timerId, window, timestamp, timeDomain); } } }
/** Clear any remaining holds. */ public void clearHolds(ReduceFn<?, ?, ?, W>.Context context) { WindowTracing.debug( "WatermarkHold.clearHolds: For key:{}; window:{}; inputWatermark:{}; outputWatermark:{}", context.key(), context.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); context.state().access(elementHoldTag).clear(); context.state().access(EXTRA_HOLD_TAG).clear(); }
private void cancelEndOfWindowAndGarbageCollectionTimers( ReduceFn<?, ?, ?, W>.Context directContext) { WindowTracing.debug( "ReduceFnRunner.cancelEndOfWindowAndGarbageCollectionTimers: Deleting timers for " + "key:{}; window:{} where inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); Instant eow = directContext.window().maxTimestamp(); directContext.timers().deleteTimer(eow, TimeDomain.EVENT_TIME); Instant gc = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); if (gc.isAfter(eow)) { directContext.timers().deleteTimer(gc, TimeDomain.EVENT_TIME); } }
W window = windowNamespace.getWindow(); WindowTracing.debug( "{}: Received timer key:{}; window:{}; data:{} with " + "inputWatermark:{}; outputWatermark:{}", WindowTracing.debug( "{}: Cleaning up for key:{}; window:{} with inputWatermark:{}; outputWatermark:{}", ReduceFnRunner.class.getSimpleName(), WindowTracing.debug( "{}.onTimers: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); WindowTracing.debug( "ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}",
WindowTracing.debug( "WatermarkHold.extractAndRelease: for key:{}; window:{}; inputWatermark:{}; " + "outputWatermark:{}",
WindowTracing.debug( "WatermarkHold.onMerge: for key:{}; window:{}; inputWatermark:{}; " + "outputWatermark:{}", context.key(),
WindowTracing.debug( "describePane: {} pane (prev was {}) for key:{}; windowMaxTimestamp:{}; " + "inputWatermark:{}; outputWatermark:{}; isLateForOutput:{}",
WindowTracing.debug( "GroupAlsoByWindow: Dropping element at {} for key: {}; " + "window: {} since it is too far behind inputWatermark: {}",
WindowTracing.debug( "ReduceFnRunner.processElement: Dropping element at {} for key:{}; window:{} " + "since window is no longer active at inputWatermark:{}; outputWatermark:{}",