public void close() { this.shouldClose = true; } }
/** *Checks whether a new per-partition watermark is also a new cross-partition watermark. */ private void updateMinPunctuatedWatermark(Watermark nextWatermark) { if (nextWatermark.getTimestamp() > maxWatermarkSoFar) { long newMin = Long.MAX_VALUE; for (KafkaTopicPartitionState<?> state : subscribedPartitionStates) { @SuppressWarnings("unchecked") final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState = (KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) state; newMin = Math.min(newMin, withWatermarksState.getCurrentPartitionWatermark()); } // double-check locking pattern if (newMin > maxWatermarkSoFar) { synchronized (checkpointLock) { if (newMin > maxWatermarkSoFar) { maxWatermarkSoFar = newMin; sourceContext.emitWatermark(new Watermark(newMin)); } } } } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { for (int i = 0; i < numWatermarks; i++) { ctx.collectWithTimestamp(i, initialTime + i); ctx.emitWatermark(new Watermark(initialTime + i)); } while (running) { Thread.sleep(20); } }
ctx.emitWatermark(Watermark.MAX_WATERMARK);
@Override public void onProcessingTime(long timestamp) throws Exception { long minAcrossAll = Long.MAX_VALUE; boolean isEffectiveMinAggregation = false; for (KafkaTopicPartitionState<?> state : allPartitions) { // we access the current watermark for the periodic assigners under the state // lock, to prevent concurrent modification to any internal variables final long curr; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (state) { curr = ((KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>) state).getCurrentWatermarkTimestamp(); } minAcrossAll = Math.min(minAcrossAll, curr); isEffectiveMinAggregation = true; } // emit next watermark, if there is one if (isEffectiveMinAggregation && minAcrossAll > lastWatermarkTimestamp) { lastWatermarkTimestamp = minAcrossAll; emitter.emitWatermark(new Watermark(minAcrossAll)); } // schedule the next watermark timerService.registerTimer(timerService.getCurrentProcessingTime() + interval, this); } }
@Override public void run(SourceContext<SessionEvent<Integer, TestEventPayload>> ctx) { ParallelSessionsEventGenerator<Integer, SessionEvent<Integer, TestEventPayload>> generator = createGenerator(); this.isRunning = true; //main data source driver loop while (isRunning) { synchronized (ctx.getCheckpointLock()) { SessionEvent<Integer, TestEventPayload> evt = generator.nextEvent(); if (evt != null) { ctx.collectWithTimestamp(evt, evt.getEventTimestamp()); ctx.emitWatermark(new Watermark(generator.getWatermark())); } else { break; } } } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { if (i % getRuntimeContext().getNumberOfParallelSubtasks() == getRuntimeContext().getIndexOfThisSubtask()) { ctx.collect(new Tuple2<>(i, i)); } } } while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { ctx.emitWatermark(new Watermark(0)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { if (i % getRuntimeContext().getNumberOfParallelSubtasks() == getRuntimeContext().getIndexOfThisSubtask()) { ctx.collect(new Tuple2<>(i, i)); } } } // don't emit a final watermark so that we don't trigger the registered event-time // timers while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { ctx.emitWatermark(new Watermark(0)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } // don't emit a final watermark so that we don't trigger the registered event-time // timers while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { int index = 1; while (index <= numElements) { ctx.collectWithTimestamp(index, index); ctx.collectWithTimestamp(index - 1, index - 1); index++; ctx.emitWatermark(new Watermark(index - 2)); } // emit the final Long.MAX_VALUE watermark, do it twice and verify that // we only see one in the result ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { ctx.emitWatermark(new Watermark(0)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } // don't emit a final watermark so that we don't trigger the registered event-time // timers while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { int index = 1; while (index <= numElements) { ctx.collectWithTimestamp(index, index); ctx.collectWithTimestamp(index - 1, index - 1); index++; ctx.emitWatermark(new Watermark(index - 2)); } // emit the final Long.MAX_VALUE watermark, do it twice and verify that // we only see one in the result ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Tuple2<String, Integer>> ctx) { ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L); ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L); ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L); ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L); ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L); ctx.emitWatermark(new Watermark(5)); ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L); ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L); ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L); ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L); }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1); // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<String, Integer>> ctx) { ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L); ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L); ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L); ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L); ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L); ctx.emitWatermark(new Watermark(5)); ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L); ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L); ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L); ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L); }
@Override public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception { for (Tuple3<String, Long, Integer> value : input) { ctx.collectWithTimestamp(value, value.f1); ctx.emitWatermark(new Watermark(value.f1 - 1)); } ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Integer> ctx) throws Exception { ctx.collectWithTimestamp(1, 0); ctx.emitWatermark(new Watermark(0)); ctx.collectWithTimestamp(2, 1); ctx.collectWithTimestamp(5, 2); ctx.emitWatermark(new Watermark(2)); ctx.collectWithTimestamp(3, 3); ctx.collectWithTimestamp(4, 4); }
@Override public void emitEvent(SourceFunction.SourceContext<Tuple2<Long, IntType>> ctx, int eventSequenceNo) { final IntType intTypeNext = new IntType(eventSequenceNo); for (long i = 0; i < keyUniverseSize; i++) { final Tuple2<Long, IntType> generatedEvent = new Tuple2<>(i, intTypeNext); ctx.collectWithTimestamp(generatedEvent, eventSequenceNo); } ctx.emitWatermark(new Watermark(eventSequenceNo - watermarkTrailing)); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { for (int i = 0; i < numWatermarks; i++) { ctx.collectWithTimestamp(i, initialTime + i); ctx.emitWatermark(new Watermark(initialTime + i)); } }