/** * This just forwards watermarks to the next operator because it has one data stream. * @param edgeIndex edge index * @param watermark watermark */ @Override public void trackAndEmitWatermarks(final int edgeIndex, final Watermark watermark) { watermarkCollector.emitWatermark(watermark); } }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
/** * This just forwards watermarks to the next operator because it has one data stream. * @param edgeIndex edge index * @param watermark watermark */ @Override public void trackAndEmitWatermarks(final int edgeIndex, final Watermark watermark) { watermarkCollector.emitWatermark(watermark); } }
private void processWatermark(final OutputCollector outputCollector, final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
private void processWatermark(final OutputCollector outputCollector, final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { outputCollector.emitWatermark(watermark); }
@Override public void onWatermark(final Watermark watermark) { checkAndInvokeBundle(); getOutputCollector().emitWatermark(watermark); checkAndFinishBundle(); }
LOG.debug("Emit watermark {}, {}", minWatermark, watermarks); watermarkCollector.emitWatermark(minWatermark);
LOG.debug("Emit watermark {}, {}", minWatermark, watermarks); watermarkCollector.emitWatermark(minWatermark);
@Override public void onWatermark(final Watermark inputWatermark) { // If no data, just forwards the watermark if (windowListMap.size() == 0 && currentOutputWatermark < inputWatermark.getTimestamp()) { currentOutputWatermark = inputWatermark.getTimestamp(); outputCollector.emitWatermark(inputWatermark); return; } final Iterator<Map.Entry<BoundedWindow, List<I>>> iterator = windowListMap.entrySet().iterator(); long minOutputTimestampOfEmittedWindows = Long.MAX_VALUE; while (iterator.hasNext()) { final Map.Entry<BoundedWindow, List<I>> entry = iterator.next(); if (entry.getKey().maxTimestamp().getMillis() <= inputWatermark.getTimestamp()) { // emit the windowed data if the watermark timestamp > the window max boundary final O output = viewFn.apply(new MultiView<>(entry.getValue())); outputCollector.emit(WindowedValue.of( output, entry.getKey().maxTimestamp(), entry.getKey(), PaneInfo.ON_TIME_AND_ONLY_FIRING)); iterator.remove(); minOutputTimestampOfEmittedWindows = Math.min(minOutputTimestampOfEmittedWindows, entry.getKey().maxTimestamp().getMillis()); } } if (minOutputTimestampOfEmittedWindows != Long.MAX_VALUE && currentOutputWatermark < minOutputTimestampOfEmittedWindows) { // update current output watermark and emit to next operators currentOutputWatermark = minOutputTimestampOfEmittedWindows; outputCollector.emitWatermark(new Watermark(currentOutputWatermark)); } }
/** * Output watermark * = max(prev output watermark, * min(input watermark, watermark holds)). * @param inputWatermark input watermark */ private void emitOutputWatermark(final Watermark inputWatermark) { // Find min watermark hold final Watermark minWatermarkHold = keyAndWatermarkHoldMap.isEmpty() ? new Watermark(Long.MAX_VALUE) // set this to MAX, in order to just use the input watermark. : Collections.min(keyAndWatermarkHoldMap.values()); final Watermark outputWatermarkCandidate = new Watermark( Math.max(prevOutputWatermark.getTimestamp(), Math.min(minWatermarkHold.getTimestamp(), inputWatermark.getTimestamp()))); if (LOG.isDebugEnabled()) { LOG.debug("Watermark hold: {}, " + "inputWatermark: {}, outputWatermark: {}", minWatermarkHold, inputWatermark, prevOutputWatermark); } if (outputWatermarkCandidate.getTimestamp() > prevOutputWatermark.getTimestamp()) { // progress! prevOutputWatermark = outputWatermarkCandidate; // emit watermark getOutputCollector().emitWatermark(outputWatermarkCandidate); // Remove minimum watermark holds if (minWatermarkHold.getTimestamp() == outputWatermarkCandidate.getTimestamp()) { keyAndWatermarkHoldMap.entrySet() .removeIf(entry -> entry.getValue().getTimestamp() == minWatermarkHold.getTimestamp()); } } }
@Override public void onWatermark(final Watermark watermark) { // TODO #298: Consider Processing DoFn PushBacks on Watermark checkAndInvokeBundle(); curInputWatermark = watermark.getTimestamp(); getSideInputReader().setCurrentWatermarkOfAllMainAndSideInputs(curInputWatermark); final long outputWatermarkCandidate = Math.min(curInputWatermark, curPushedBackWatermark); if (outputWatermarkCandidate > curOutputWatermark) { // Watermark advances! getOutputCollector().emitWatermark(new Watermark(outputWatermarkCandidate)); curOutputWatermark = outputWatermarkCandidate; } checkAndFinishBundle(); }