@Override protected ProcessingTimeService getProcessingTimeService() { if (testProcessingTimeService == null) { testProcessingTimeService = new TestProcessingTimeService(); } return testProcessingTimeService; } }
@SafeVarargs private MockFetcher(HashMap<KafkaTopicPartition, Long>... stateSnapshotsToReturn) throws Exception { super( new TestSourceContext<>(), new HashMap<>(), null, null, new TestProcessingTimeService(), 0, MockFetcher.class.getClassLoader(), new UnregisteredMetricsGroup(), false); this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn)); }
@SuppressWarnings("unchecked") private static <T> void setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, final ProcessingTimeService timeProvider) { ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setAutoWatermarkInterval(watermarkInterval); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateBackend(new MemoryStateBackend()); cfg.setTimeCharacteristic(timeChar); cfg.setOperatorID(new OperatorID()); Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0); StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class); when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE); StreamTask<?, ?> mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(new Object()); when(mockTask.getConfiguration()).thenReturn(cfg); when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap()); when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer); doAnswer(new Answer<ProcessingTimeService>() {
@Test public void testIgnorePartitionStateSentinelInSnapshot() throws Exception { final String testTopic = "test topic name"; Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>(); originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET); originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET); originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET); TestSourceContext<Long> sourceContext = new TestSourceContext<>(); TestFetcher<Long> fetcher = new TestFetcher<>( sourceContext, originalPartitions, null, null, new TestProcessingTimeService(), 0); synchronized (sourceContext.getCheckpointLock()) { HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState(); fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() { @Override public void onSuccess() { } @Override public void onException(Throwable cause) { throw new RuntimeException("Callback failed", cause); } }); assertTrue(fetcher.getLastCommittedOffsets().isPresent()); assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get()); } }
new TestProcessingTimeService(), 10, fetchLoopWaitLatch,
@Test public void testSkipCorruptedRecord() throws Exception { final String testTopic = "test topic name"; Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>(); originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET); TestSourceContext<Long> sourceContext = new TestSourceContext<>(); TestFetcher<Long> fetcher = new TestFetcher<>( sourceContext, originalPartitions, null, /* periodic watermark assigner */ null, /* punctuated watermark assigner */ new TestProcessingTimeService(), 0); final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0); fetcher.emitRecord(1L, partitionStateHolder, 1L); fetcher.emitRecord(2L, partitionStateHolder, 2L); assertEquals(2L, sourceContext.getLatestElement().getValue().longValue()); assertEquals(2L, partitionStateHolder.getOffset()); // emit null record fetcher.emitRecord(null, partitionStateHolder, 3L); assertEquals(2L, sourceContext.getLatestElement().getValue().longValue()); // the null record should be skipped assertEquals(3L, partitionStateHolder.getOffset()); // the offset in state still should have advanced }
@Test public void testCurrentProcessingTime() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); processingTimeService.setCurrentTime(17L); assertEquals(17, timerService.currentProcessingTime()); processingTimeService.setCurrentTime(42); assertEquals(42, timerService.currentProcessingTime()); }
@Test public void testCurrentEventTime() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); timerService.advanceWatermark(17); assertEquals(17, timerService.currentWatermark()); timerService.advanceWatermark(42); assertEquals(42, timerService.currentWatermark()); }
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception { final List<StreamElement> output = new ArrayList<>(); final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService(); testProcessingTimeService.setCurrentTime(0L); final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime); // regular stream source operator final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes)); operatorSetup.setupSourceOperator(operator, testProcessingTimeService); // run and wait to be stopped operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output)); assertEquals( numberLatencyMarkers + 1, // + 1 is the final watermark element output.size()); long timestamp = 0L; int i = 0; // verify that its only latency markers + a final watermark for (; i < numberLatencyMarkers; i++) { StreamElement se = output.get(i); Assert.assertTrue(se.isLatencyMarker()); Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId()); Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex()); Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp); timestamp += latencyMarkInterval; } Assert.assertTrue(output.get(i).isWatermark()); }
@Test public void testKeyGroupStartIndexSetting() { int startKeyGroupIdx = 7; int endKeyGroupIdx = 21; KeyGroupRange testKeyGroupList = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> service = createInternalTimerService( testKeyGroupList, keyContext, processingTimeService, IntSerializer.INSTANCE, StringSerializer.INSTANCE, createQueueFactory()); Assert.assertEquals(startKeyGroupIdx, service.getLocalKeyGroupRangeStartIdx()); }
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory());
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
@Test public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception { final String testTopic = "test topic name"; Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>(); TestSourceContext<Long> sourceContext = new TestSourceContext<>(); TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService(); TestFetcher<Long> fetcher = new TestFetcher<>( sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), null, /* punctuated watermarks assigner*/ processingTimeProvider, 10); processingTimeProvider.setCurrentTime(10); // no partitions; when the periodic watermark emitter fires, no watermark should be emitted assertFalse(sourceContext.hasWatermark()); // counter-test that when the fetcher does actually have partitions, // when the periodic watermark emitter fires again, a watermark really is emitted fetcher.addDiscoveredPartitions(Collections.singletonList(new KafkaTopicPartition(testTopic, 0))); fetcher.emitRecord(100L, fetcher.subscribedPartitionStates().get(0), 3L); processingTimeProvider.setCurrentTime(20); assertEquals(100, sourceContext.getLatestWatermark().getTimestamp()); }
/** * Verify that registering a processing-time timer that is earlier than the existing timers * removes the one physical timer and creates one for the earlier timestamp * {@link ProcessingTimeService}. */ @Test public void testRegisterEarlierProcessingTimerMovesPhysicalProcessingTimer() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism); keyContext.setCurrentKey(key); timerService.registerProcessingTimeTimer("ciao", 20); assertEquals(1, timerService.numProcessingTimeTimers()); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L)); timerService.registerProcessingTimeTimer("ciao", 10); assertEquals(2, timerService.numProcessingTimeTimers()); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L)); }
dummyEnvironment.setTaskStateManager(taskStateManager); ProcessingTimeService processingTimeService = new TestProcessingTimeService();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); processingTimeService.setCurrentTime(0);
TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();
TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();