LOG.debug("Initializing {}.", getName()); CheckpointExceptionHandlerFactory cpExceptionHandlerFactory = createCheckpointExceptionHandlerFactory(); getExecutionConfig().isFailTaskOnCheckpointError(), getEnvironment()); stateBackend = createStateBackend(); checkpointStorage = stateBackend.createCheckpointStorage(getEnvironment().getJobID()); "Time Trigger for " + getName(), getUserCodeClassLoader()); timerService = new SystemProcessingTimeService(this, getCheckpointLock(), timerThreadFactory); init(); LOG.debug("Invoking {}", getName()); initializeState(); openAllOperators(); run(); LOG.debug("Finished task {}", getName()); closeAllOperators(); LOG.debug("Closed operators for task {}", getName());
private static <T> void setupSourceOperator( StreamSource<T, ?> operator, ExecutionConfig executionConfig, Environment env, ProcessingTimeService timeProvider) { StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateBackend(new MemoryStateBackend()); cfg.setTimeCharacteristic(TimeCharacteristic.EventTime); cfg.setOperatorID(new OperatorID()); StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class); when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE); StreamTask<?, ?> mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(new Object()); when(mockTask.getConfiguration()).thenReturn(cfg); when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap()); when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer); doAnswer(new Answer<ProcessingTimeService>() { @Override public ProcessingTimeService answer(InvocationOnMock invocation) throws Throwable { if (timeProvider == null) { throw new RuntimeException("The time provider is null."); } return timeProvider; } }).when(mockTask).getProcessingTimeService(); operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class)); }
when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(checkpointLock); when(mockTask.getConfiguration()).thenReturn(config); when(mockTask.getTaskConfiguration()).thenReturn(underlyingConfig); when(mockTask.getEnvironment()).thenReturn(environment); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); when(mockTask.createStreamTaskStateInitializer()).thenReturn(streamTaskStateInitializer); when(mockTask.getUserCodeClassLoader()).thenReturn(cl); when(mockTask.getCancelables()).thenReturn(this.closableRegistry); when(mockTask.getStreamStatusMaintainer()).thenReturn(mockStreamStatusMaintainer); return null; }).when(mockTask).handleAsyncException(any(String.class), any(Throwable.class)); when(mockTask.getCheckpointStorage()).thenAnswer((invocationOnMock) -> this.checkpointStorage); return processingTimeService; }).when(mockTask).getProcessingTimeService();
@Override public void abortCheckpointOnBarrier(long checkpointId, Throwable cause) throws Exception { LOG.debug("Aborting checkpoint via cancel-barrier {} for task {}", checkpointId, getName()); // notify the coordinator that we decline this checkpoint getEnvironment().declineCheckpoint(checkpointId, cause); // notify all downstream operators that they should not wait for a barrier from us synchronized (lock) { operatorChain.broadcastCheckpointCancelMarker(checkpointId); } }
@Override public void triggerCheckpointOnBarrier( CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception { try { performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics); } catch (CancelTaskException e) { LOG.info("Operator {} was cancelled while performing checkpoint {}.", getName(), checkpointMetaData.getCheckpointId()); throw e; } catch (Exception e) { throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() + " for operator " + getName() + '.', e); } }
private StateBackend createStateBackend() throws Exception { final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader()); return StateBackendLoader.fromApplicationOrConfigOrDefault( fromApplication, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); }
private <IN, OT extends StreamOperator<IN>> StreamTask<IN, OT> createMockTask( StreamConfig streamConfig, Environment environment) { final Object checkpointLock = new Object(); @SuppressWarnings("unchecked") StreamTask<IN, OT> mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(checkpointLock); when(mockTask.getConfiguration()).thenReturn(streamConfig); when(mockTask.getEnvironment()).thenReturn(environment); when(mockTask.getExecutionConfig()).thenReturn(new ExecutionConfig().enableObjectReuse()); return mockTask; }
List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters) { final ClassLoader userCodeClassloader = containingTask.getUserCodeClassLoader(); final StreamConfig configuration = containingTask.getConfiguration(); outEdge, chainedConfigs.get(outEdge.getSourceId()), containingTask.getEnvironment());
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { final Environment environment = containingTask.getEnvironment(); this.container = containingTask; this.config = config; this.latencyStats = new LatencyStats(jobMetricGroup.addGroup("latency"), historySize, container.getIndexInSubtaskGroup(), getOperatorID(), granularity); this.runtimeContext = new StreamingRuntimeContext(this, environment, container.getAccumulatorMap());
when(containingTask.getEnvironment()).thenReturn(environment); when(containingTask.getCheckpointLock()).thenReturn(lock); when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);
@Override public void open() throws Exception { super.open(); checkState(this.reader == null, "The reader is already initialized."); checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it."); this.format.setRuntimeContext(getRuntimeContext()); this.format.configure(new Configuration()); this.checkpointLock = getContainingTask().getCheckpointLock(); // set the reader context based on the time characteristic final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.readerContext = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1); // and initialize the split reading thread this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState); this.restoredReaderState = null; this.reader.start(); }
public Object getCheckpointLock() { return mockTask.getCheckpointLock(); }
public OperatorChain(StreamTask<OUT, OP> containingTask) { final ClassLoader userCodeClassloader = containingTask.getUserCodeClassLoader(); final StreamConfig configuration = containingTask.getConfiguration(); containingTask.getEnvironment(), containingTask.getName());
operator.getContainingTask().getProcessingTimeService(), operator.getContainingTask().getCheckpointLock(), operator.getContainingTask().getStreamStatusMaintainer(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(),
/** * Constructor for initialization, possibly with initial state (recovery / savepoint / etc). * * <p>This constructor accepts a special {@link ProcessingTimeService}. By default (and if * null is passes for the time provider) a {@link SystemProcessingTimeService DefaultTimerService} * will be used. * * @param environment The task environment for this task. * @param timeProvider Optionally, a specific time provider to use. */ protected StreamTask( Environment environment, @Nullable ProcessingTimeService timeProvider) { super(environment); this.timerService = timeProvider; this.configuration = new StreamConfig(getTaskConfiguration()); this.accumulatorMap = getEnvironment().getAccumulatorRegistry().getUserMap(); this.recordWriters = createRecordWriters(configuration, environment); }
public ClassLoader getUserCodeClassloader() { return container.getUserCodeClassLoader(); }
/** * Gets the execution config defined on the execution environment of the job to which this * operator belongs. * * @return The job's execution config. */ public ExecutionConfig getExecutionConfig() { return container.getExecutionConfig(); }
public StreamTaskStateInitializer createStreamTaskStateInitializer() { return new StreamTaskStateInitializerImpl( getEnvironment(), stateBackend, timerService); }
Preconditions.checkNotNull(getContainingTask()); final CloseableRegistry streamTaskCloseableRegistry = Preconditions.checkNotNull(containingTask.getCancelables()); final StreamTaskStateInitializer streamTaskStateManager = Preconditions.checkNotNull(containingTask.createStreamTaskStateInitializer());
factory, keyGroupRange, getContainingTask().getCancelables())) { getOperatorName() + "."; if (!getContainingTask().isCanceled()) { LOG.info(snapshotFailMessage, snapshotException);