@Override public void run(SourceContext<Long> ctx) throws Exception { while (isRunning && !this.valuesToEmit.isEmpty()) { synchronized (ctx.getCheckpointLock()) { ctx.collect(this.valuesToEmit.poll()); } } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { while (!cancel) { synchronized (ctx.getCheckpointLock()) { ctx.collect(1); } Thread.sleep(100L); } }
@Override public void run(SourceContext<String> ctx) throws Exception { while (isRunning) { if (readyToFail && getRuntimeContext().getIndexOfThisSubtask() == 0) { throw new Exception("Artificial failure."); } synchronized (ctx.getCheckpointLock()) { ctx.collect("test-element"); } Thread.sleep(1); } }
@Override public void run(SourceContext<Long> ctx) throws Exception { final Object lockingObject = ctx.getCheckpointLock(); while (isRunning && index < numElements) { long result = index % 10; synchronized (lockingObject) { index += step; ctx.collect(result); } } // if the program goes fast and no notifications come through, we // wait until all tasks had a chance to see a notification while (isRunning && numPostFailureNotifications.get() < notificationsToWaitFor) { Thread.sleep(50); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1); // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { while (running) { synchronized (ctx.getCheckpointLock()) { ctx.collect(emittedCount); } if (emittedCount < 100) { ++emittedCount; } else { emittedCount = 0; } Thread.sleep(1); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { ctx.emitWatermark(new Watermark(0)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } // don't emit a final watermark so that we don't trigger the registered event-time // timers while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { if (i % getRuntimeContext().getNumberOfParallelSubtasks() == getRuntimeContext().getIndexOfThisSubtask()) { ctx.collect(new Tuple2<>(i, i)); } } } while (isRunning) { Thread.sleep(20); } }
private void emitElementsAndWaitForCheckpoints(SourceContext<T> ctx, int checkpointsToWaitFor) throws InterruptedException { final Object lock = ctx.getCheckpointLock(); final int checkpointToAwait; synchronized (lock) { checkpointToAwait = numCheckpointsComplete + checkpointsToWaitFor; for (T t : elements) { ctx.collect(t); } } synchronized (lock) { while (running && numCheckpointsComplete < checkpointToAwait) { lock.wait(1); } } }
@Override public void run(SourceContext<Row> ctx) throws Exception { long offsetMS = offsetSeconds * 2000L; while (ms < durationMs) { synchronized (ctx.getCheckpointLock()) { for (int i = 0; i < numKeys; i++) { ctx.collect(Row.of(i, ms + offsetMS, "Some payload...")); } ms += sleepMs; } Thread.sleep(sleepMs); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { // immediately trigger any set timers ctx.emitWatermark(new Watermark(1000)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception { ctx.emitWatermark(new Watermark(0)); synchronized (ctx.getCheckpointLock()) { for (long i = 0; i < numElements; i++) { ctx.collect(new Tuple2<>(i, i)); } } // don't emit a final watermark so that we don't trigger the registered event-time // timers while (isRunning) { Thread.sleep(20); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { while ((start < counter || counter == -1) && isRunning) { synchronized (ctx.getCheckpointLock()) { ctx.collect(start); ++start; // loop back to 0 if (start == Integer.MAX_VALUE) { start = 0; } } Thread.sleep(10L); } }
@Override public void run(SourceContext<String> ctx) throws Exception { final Object lockingObject = ctx.getCheckpointLock(); while (isRunning && index < numElements) { char first = (char) ((index % 40) + 40); stringBuilder.setLength(0); stringBuilder.append(first); String result = randomString(stringBuilder, rnd); synchronized (lockingObject) { index += step; ctx.collect(result); } } }
@Override public void run(SourceContext<Tuple> ctx) throws Exception { long offsetMs = offsetSeconds * 1000L; while (ms < durationMs) { synchronized (ctx.getCheckpointLock()) { ctx.collect(new Tuple2<>(ms + offsetMs, "key")); } ms += sleepMs; Thread.sleep(sleepMs); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { while (running) { synchronized (ctx.getCheckpointLock()) { ctx.collect(1); } Thread.sleep(1); } }
@Override public void run(SourceContext<Tuple2<Integer, Integer>> out) throws Exception { Random random = new Random(42); while (--remainingEvents >= 0) { synchronized (out.getCheckpointLock()) { out.collect(new Tuple2<>(random.nextInt(numKeys), 1)); } } }
@Override public void run(SourceContext<Tuple3<Integer, Long, String>> ctx) throws Exception { while (ms < durationMs) { synchronized (ctx.getCheckpointLock()) { for (int i = 0; i < numKeys; i++) { ctx.collect(Tuple3.of(i, ms, "Some payload...")); } ms += idlenessMs; } Thread.sleep(idlenessMs); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { while (running && !stopJobSignal.isSignaled()) { synchronized (ctx.getCheckpointLock()) { ctx.collect(0); } Thread.sleep(5L); } }
@Override public void run(SourceContext<Tuple2<Integer, Long>> ctx) throws Exception { Object lock = ctx.getCheckpointLock(); while (count < NUM_INPUT){ synchronized (lock){ for (int i = 0; i < PARALLELISM; i++) { ctx.collect(Tuple2.of(i, count + 1)); } count++; } } }