String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0]; ensureRocksDBIsLoaded(tempDir);
private void tryShutdownTimerService() { if (timerService != null && !timerService.isTerminated()) { try { final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration(). getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS); if (!timerService.shutdownServiceUninterruptible(timeoutMs)) { LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " + "timers. Will continue with shutdown procedure.", timeoutMs); } } catch (Throwable t) { // catch and log the exception to not replace the original exception LOG.error("Could not shut down timer service", t); } } }
getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else if (groupSize > 1){ getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) { (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError()))
getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else if (groupSize > 1){ getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) { (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) {
private StateBackend createStateBackend() throws Exception { final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader()); return StateBackendLoader.fromApplicationOrConfigOrDefault( fromApplication, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); }
@Override public void invoke() throws Exception { RecordReader<SpeedTestRecord> reader = new RecordReader<>( getEnvironment().getInputGate(0), SpeedTestRecord.class, getEnvironment().getTaskManagerInfo().getTmpDirectories()); try { boolean isSlow = getTaskConfiguration().getBoolean(IS_SLOW_RECEIVER_CONFIG_KEY, false); int numRecords = 0; while (reader.next() != null) { if (isSlow && (numRecords++ % IS_SLOW_EVERY_NUM_RECORDS) == 0) { Thread.sleep(IS_SLOW_SLEEP_MS); } } } finally { reader.clearBuffers(); } } }
getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else if (groupSize > 1){ getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) { (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) {
getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator,
@Override public void invoke() throws Exception { RecordReader<SpeedTestRecord> reader = new RecordReader<>( getEnvironment().getInputGate(0), SpeedTestRecord.class, getEnvironment().getTaskManagerInfo().getTmpDirectories()); RecordWriter<SpeedTestRecord> writer = new RecordWriter<>(getEnvironment().getWriter(0)); try { SpeedTestRecord record; while ((record = reader.next()) != null) { writer.emit(record); } } finally { reader.clearBuffers(); writer.clearBuffers(); writer.flushAll(); } } }
this.headEventReader = new MutableRecordReader<>( getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration());
if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) { (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) {
Configuration taskManagerConfig = environment.getTaskManagerInfo().getConfiguration(); int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE); if (historySize <= 0) {
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry) throws IOException { // first, make sure that the RocksDB JNI library is loaded // we do this explicitly here to have better error handling String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0]; ensureRocksDBIsLoaded(tempDir); lazyInitializeForJob(env, operatorIdentifier); File instanceBasePath = new File(getNextStoragePath(), "job-" + jobId + "_op-" + operatorIdentifier + "_uuid-" + UUID.randomUUID()); return new RocksDBKeyedStateBackend<>( operatorIdentifier, env.getUserClassLoader(), instanceBasePath, getDbOptions(), getColumnOptions(), kvStateRegistry, keySerializer, numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), enableIncrementalCheckpointing); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
getEnvironment().getTaskManagerInfo().getTmpDirectories()); } else if (groupSize > 1){ getEnvironment().getTaskManagerInfo().getTmpDirectories()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval()
getEnvironment().getTaskManagerInfo().getTmpDirectories()); } else if (groupSize > 1){ getEnvironment().getTaskManagerInfo().getTmpDirectories()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);