@Override public OperatorID getOperatorID() { return new OperatorID(); } }
public AbstractStreamOperatorTestHarness( StreamOperator<OUT> operator, int maxParallelism, int parallelism, int subtaskIndex) throws Exception { this( operator, maxParallelism, parallelism, subtaskIndex, new OperatorID()); }
public OneInputStreamOperatorTestHarness( OneInputStreamOperator<IN, OUT> operator, int maxParallelism, int parallelism, int subtaskIndex) throws Exception { this(operator, maxParallelism, parallelism, subtaskIndex, new OperatorID()); }
@Override public OperatorID getOperatorID() { return new OperatorID(); } }
@Override public OperatorID getOperatorID() { return new OperatorID(); }
public OperatorID getOperatorID() { byte[] operatorIDBytes = config.getBytes(OPERATOR_ID, null); return new OperatorID(Preconditions.checkNotNull(operatorIDBytes)); }
@SuppressWarnings("unchecked") private static AbstractStreamOperator<?> createDescriptorCapturingMockOp( final AtomicReference<Object> ref, final ExecutionConfig config) throws Exception { AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class); KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class); DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config); when(operatorMock.getExecutionConfig()).thenReturn(config); doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { ref.set(invocationOnMock.getArguments()[2]); return null; } }).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(StateDescriptor.class)); when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore); when(operatorMock.getOperatorID()).thenReturn(new OperatorID()); return operatorMock; }
public MockStreamConfig() { super(new Configuration()); setOperatorID(new OperatorID()); } }
@Test public void testRestoreHeadWithNewId() throws Exception { OperatorID tailOperatorID = new OperatorID(44L, 44L); JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain( new OperatorID(42L, 42L), new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.empty()); TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot(); assertEquals(2, stateHandles.getSubtaskStateMappings().size()); createRunAndCheckpointOperatorChain( new OperatorID(4242L, 4242L), new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore)); assertEquals(Collections.singleton(tailOperatorID), RESTORED_OPERATORS); }
@Test public void testRestoreTailWithNewId() throws Exception { OperatorID headOperatorID = new OperatorID(42L, 42L); JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), new OperatorID(44L, 44L), new CounterOperator(), Optional.empty()); TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot(); assertEquals(2, stateHandles.getSubtaskStateMappings().size()); createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), new OperatorID(4444L, 4444L), new CounterOperator(), Optional.of(restore)); assertEquals(Collections.singleton(headOperatorID), RESTORED_OPERATORS); }
@Override public StreamElement deserialize(DataInputView source) throws IOException { int tag = source.readByte(); if (tag == TAG_REC_WITH_TIMESTAMP) { long timestamp = source.readLong(); return new StreamRecord<T>(typeSerializer.deserialize(source), timestamp); } else if (tag == TAG_REC_WITHOUT_TIMESTAMP) { return new StreamRecord<T>(typeSerializer.deserialize(source)); } else if (tag == TAG_WATERMARK) { return new Watermark(source.readLong()); } else if (tag == TAG_STREAM_STATUS) { return new StreamStatus(source.readInt()); } else if (tag == TAG_LATENCY_MARKER) { return new LatencyMarker(source.readLong(), new OperatorID(source.readLong(), source.readLong()), source.readInt()); } else { throw new IOException("Corrupt stream, found tag: " + tag); } }
@Test public void testRestore() throws Exception { OperatorID headOperatorID = new OperatorID(42L, 42L); OperatorID tailOperatorID = new OperatorID(44L, 44L); JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.empty()); TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot(); assertEquals(2, stateHandles.getSubtaskStateMappings().size()); createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore)); assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS); }
@Override public StreamElement deserialize(StreamElement reuse, DataInputView source) throws IOException { int tag = source.readByte(); if (tag == TAG_REC_WITH_TIMESTAMP) { long timestamp = source.readLong(); T value = typeSerializer.deserialize(source); StreamRecord<T> reuseRecord = reuse.asRecord(); reuseRecord.replace(value, timestamp); return reuseRecord; } else if (tag == TAG_REC_WITHOUT_TIMESTAMP) { T value = typeSerializer.deserialize(source); StreamRecord<T> reuseRecord = reuse.asRecord(); reuseRecord.replace(value); return reuseRecord; } else if (tag == TAG_WATERMARK) { return new Watermark(source.readLong()); } else if (tag == TAG_LATENCY_MARKER) { return new LatencyMarker(source.readLong(), new OperatorID(source.readLong(), source.readLong()), source.readInt()); } else { throw new IOException("Corrupt stream, found tag: " + tag); } }
@Test public void testRestoreWithoutState() throws Exception { OperatorID headOperatorID = new OperatorID(42L, 42L); OperatorID tailOperatorID = new OperatorID(44L, 44L); JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain( headOperatorID, new StatelessOperator(), tailOperatorID, new CounterOperator(), Optional.empty()); TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot(); assertEquals(2, stateHandles.getSubtaskStateMappings().size()); createRunAndCheckpointOperatorChain( headOperatorID, new StatelessOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore)); assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS); }
@Test public void testSerialization() throws Exception { final StreamElementSerializer<String> serializer = new StreamElementSerializer<String>(StringSerializer.INSTANCE); StreamRecord<String> withoutTimestamp = new StreamRecord<>("test 1 2 分享基督耶穌的愛給們,開拓雙贏!"); assertEquals(withoutTimestamp, serializeAndDeserialize(withoutTimestamp, serializer)); StreamRecord<String> withTimestamp = new StreamRecord<>("one more test 拓 們 分", 77L); assertEquals(withTimestamp, serializeAndDeserialize(withTimestamp, serializer)); StreamRecord<String> negativeTimestamp = new StreamRecord<>("他", Long.MIN_VALUE); assertEquals(negativeTimestamp, serializeAndDeserialize(negativeTimestamp, serializer)); Watermark positiveWatermark = new Watermark(13); assertEquals(positiveWatermark, serializeAndDeserialize(positiveWatermark, serializer)); Watermark negativeWatermark = new Watermark(-4647654567676555876L); assertEquals(negativeWatermark, serializeAndDeserialize(negativeWatermark, serializer)); LatencyMarker latencyMarker = new LatencyMarker(System.currentTimeMillis(), new OperatorID(-1, -1), 1); assertEquals(latencyMarker, serializeAndDeserialize(latencyMarker, serializer)); }
/** * This test checks that cancel calls that are issued before the operator is * instantiated still lead to proper canceling. */ @Test public void testEarlyCanceling() throws Exception { final StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setOperatorID(new OperatorID(4711L, 42L)); cfg.setStreamOperator(new SlowlyDeserializingOperator()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions()); final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions); final TaskExecutionState state = new TaskExecutionState( task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING); task.startTaskThread(); verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state)); // send a cancel. because the operator takes a long time to deserialize, this should // hit the task before the operator is deserialized task.cancelExecution(); task.getExecutingThread().join(); assertFalse("Task did not cancel", task.getExecutingThread().isAlive()); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testLifeCycleFull() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING); }
@Test public void testLifeCycleCancel() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // this should cancel the task even though it is blocked on runFinished task.cancelExecution(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING); }
/** * This test verifies that open() and close() are correctly called by the StreamTask. */ @Test @SuppressWarnings("unchecked") public void testOpenClose() throws Exception { final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>( SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); StreamConfig streamConfig = testHarness.getStreamConfig(); StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource()); streamConfig.setStreamOperator(sourceOperator); streamConfig.setOperatorID(new OperatorID()); testHarness.invoke(); testHarness.waitForTaskCompletion(); Assert.assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled); List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput()); Assert.assertEquals(10, resultElements.size()); }
@Test public void testQuiesceTimerServiceAfterOpClose() throws Exception { final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>( OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); StreamConfig streamConfig = testHarness.getStreamConfig(); streamConfig.setStreamOperator(new TestOperator()); streamConfig.setOperatorID(new OperatorID()); testHarness.invoke(); testHarness.waitForTaskRunning(); SystemProcessingTimeService timeService = (SystemProcessingTimeService) testHarness.getTask().getProcessingTimeService(); // verify that the timer service is running Assert.assertTrue(timeService.isAlive()); testHarness.endInput(); testHarness.waitForTaskCompletion(); timeService.shutdownService(); }