@Override public void run(SourceContext<Long> ctx) throws Exception { for (Long processingTime : processingTimes) { if (cancelled) { break; } processingTimeService.setCurrentTime(processingTime); } }
public void setProcessingTime(long time) throws Exception { processingTimeService.setCurrentTime(time); }
@Test public void testCurrentProcessingTime() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); processingTimeService.setCurrentTime(17L); assertEquals(17, timerService.currentProcessingTime()); processingTimeService.setCurrentTime(42); assertEquals(42, timerService.currentProcessingTime()); }
processingTimeService.setCurrentTime(0);
assertEquals(2, timerService.numProcessingTimeTimers("ciao")); processingTimeService.setCurrentTime(10);
@Test public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception { final String testTopic = "test topic name"; Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>(); TestSourceContext<Long> sourceContext = new TestSourceContext<>(); TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService(); TestFetcher<Long> fetcher = new TestFetcher<>( sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), null, /* punctuated watermarks assigner*/ processingTimeProvider, 10); processingTimeProvider.setCurrentTime(10); // no partitions; when the periodic watermark emitter fires, no watermark should be emitted assertFalse(sourceContext.hasWatermark()); // counter-test that when the fetcher does actually have partitions, // when the periodic watermark emitter fires again, a watermark really is emitted fetcher.addDiscoveredPartitions(Collections.singletonList(new KafkaTopicPartition(testTopic, 0))); fetcher.emitRecord(100L, fetcher.subscribedPartitionStates().get(0), 3L); processingTimeProvider.setCurrentTime(20); assertEquals(100, sourceContext.getLatestWatermark().getTimestamp()); }
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception { final List<StreamElement> output = new ArrayList<>(); final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService(); testProcessingTimeService.setCurrentTime(0L); final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime); // regular stream source operator final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes)); operatorSetup.setupSourceOperator(operator, testProcessingTimeService); // run and wait to be stopped operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output)); assertEquals( numberLatencyMarkers + 1, // + 1 is the final watermark element output.size()); long timestamp = 0L; int i = 0; // verify that its only latency markers + a final watermark for (; i < numberLatencyMarkers; i++) { StreamElement se = output.get(i); Assert.assertTrue(se.isLatencyMarker()); Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId()); Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex()); Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp); timestamp += latencyMarkInterval; } Assert.assertTrue(output.get(i).isWatermark()); }
processingTimeService.setCurrentTime(initialTime); processingTimeService.setCurrentTime(initialTime + idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10); switch (testMethod) { case COLLECT: processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 2 * idleTimeout / 10); switch (testMethod) { case COLLECT: processingTimeService.setCurrentTime(initialTime + 4 * idleTimeout + idleTimeout / 10); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive()); processingTimeService.setCurrentTime(initialTime + 5 * idleTimeout + idleTimeout / 10); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
processingTimeService.setCurrentTime(10); processingTimeService.setCurrentTime(20);
queueFactory); processingTimeService1.setCurrentTime(10); timerService1.advanceWatermark(10); processingTimeService2.setCurrentTime(10); timerService2.advanceWatermark(10);
createQueueFactory()); processingTimeService.setCurrentTime(10); timerService.advanceWatermark(10);
assertEquals(1, timerService.numProcessingTimeTimers("ciao")); processingTimeService.setCurrentTime(10);
processingTimeService.setCurrentTime(0); processingTimeService.setCurrentTime(i);
processingTimeService.setCurrentTime(initialTime); processingTimeService.setCurrentTime(initialTime + watermarkInterval); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); processingTimeService.setCurrentTime(initialTime + 2 * watermarkInterval); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); processingTimeService.setCurrentTime(initialTime + idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); assertEquals(expectedOutput, output); processingTimeService.setCurrentTime(initialTime + 3 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 4 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout); processingTimeService.setCurrentTime(initialTime + 6 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 7 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); assertEquals(expectedOutput, output); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10); switch (testMethod) { case COLLECT: processingTimeService.setCurrentTime(initialTime + 8 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 3 * idleTimeout / 10); switch (testMethod) { case COLLECT: processingTimeService.setCurrentTime(initialTime + 9 * watermarkInterval);
tp.setCurrentTime(11); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 11); tp.setCurrentTime(15); tp.setCurrentTime(16); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 16); tp.setCurrentTime(35); assertEquals(1, tp.getNumActiveTimers()); tp.setCurrentTime(40); assertEquals(0, tp.getNumActiveTimers());
processingTimeProvider.setCurrentTime(10L); assertTrue(sourceContext.hasWatermark()); assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp()); processingTimeProvider.setCurrentTime(20L); assertFalse(sourceContext.hasWatermark());
assertEquals(102L, sourceContext.getLatestElement().getTimestamp()); processingTimeService.setCurrentTime(10); assertEquals(30L, sourceContext.getLatestElement().getTimestamp()); processingTimeService.setCurrentTime(20); fetcher.emitRecord(15L, part2, 3L); processingTimeService.setCurrentTime(30);
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L)); processingTimeService.setCurrentTime(10); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L)); processingTimeService.setCurrentTime(20); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L)); processingTimeService.setCurrentTime(30);