@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
@SafeVarargs private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain( OneInputStreamOperator<T, T>... operators) { checkNotNull(operators); checkArgument(operators.length > 0); try (MockEnvironment env = MockEnvironment.builder().build()) { final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env); final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class); final StreamConfig cfg = new StreamConfig(new Configuration()); final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length]; // initial output goes to nowhere @SuppressWarnings({"unchecked", "rawtypes"}) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>( new Output[0], statusProvider); // build the reverse operators array for (int i = 0; i < ops.length; i++) { OneInputStreamOperator<T, T> op = operators[ops.length - i - 1]; op.setup(containingTask, cfg, lastWriter); lastWriter = new ChainingOutput<>(op, statusProvider, null); ops[i] = op; } @SuppressWarnings("unchecked") final OP head = (OP) operators[0]; return new OperatorChain<>( ops, new RecordWriterOutput<?>[0], lastWriter, head); } }
(env) -> new OneInputStreamTask<>(env, tp), BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); mapTask.getProcessingTimeService().registerTimer(30, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) { mapTask.getProcessingTimeService().registerTimer(40, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) {
private void triggerCheckpoint( OneInputStreamTaskTestHarness<String, String> testHarness, OneInputStreamTask<String, String> streamTask) throws Exception { long checkpointId = 1L; CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 1L); testHarness.taskStateManager.setWaitForReportLatch(new OneShotLatch()); while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {} testHarness.taskStateManager.getWaitForReportLatch().await(); long reportedCheckpointId = testHarness.taskStateManager.getReportedCheckpointId(); assertEquals(checkpointId, reportedCheckpointId); }
final long t4 = System.currentTimeMillis() + 200; ProcessingTimeService timeService = mapTask.getProcessingTimeService(); timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0)); timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
task.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); restoredTask.triggerCheckpoint(new CheckpointMetaData(checkpointId, checkpointTimestamp), CheckpointOptions.forCheckpointWithDefaultLocation());
mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) {
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator); // make sure that stream tasks report their I/O statistics inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup()); } }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {}
@Test public void testQuiesceTimerServiceAfterOpClose() throws Exception { final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>( OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); StreamConfig streamConfig = testHarness.getStreamConfig(); streamConfig.setStreamOperator(new TestOperator()); streamConfig.setOperatorID(new OperatorID()); testHarness.invoke(); testHarness.waitForTaskRunning(); SystemProcessingTimeService timeService = (SystemProcessingTimeService) testHarness.getTask().getProcessingTimeService(); // verify that the timer service is running Assert.assertTrue(timeService.isAlive()); testHarness.endInput(); testHarness.waitForTaskCompletion(); timeService.shutdownService(); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }