@SuppressWarnings("unchecked") private static String getLogicalScope(MetricGroup group) { return ((FrontMetricGroup<AbstractMetricGroup<?>>) group).getLogicalScope(CHARACTER_FILTER, SCOPE_SEPARATOR); }
@Override public TaskMetricGroup getMetricGroup() { return UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(); } }
@Override public OperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) { return new OperatorMetricGroup(NoOpMetricRegistry.INSTANCE, this, operatorID, name); } };
this.config = config; try { OperatorMetricGroup operatorMetricGroup = environment.getMetricGroup().getOrAddOperator(config.getOperatorID(), config.getOperatorName()); this.output = new CountingOutput(output, operatorMetricGroup.getIOMetricGroup().getNumRecordsOutCounter()); if (config.isChainStart()) { operatorMetricGroup.getIOMetricGroup().reuseInputMetricsForTask(); operatorMetricGroup.getIOMetricGroup().reuseOutputMetricsForTask(); this.metrics = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup(); this.output = output; granularity); TaskManagerJobMetricGroup jobMetricGroup = this.metrics.parent().parent(); this.latencyStats = new LatencyStats(jobMetricGroup.addGroup("latency"), historySize, container.getIndexInSubtaskGroup(), LOG.warn("An error occurred while instantiating latency metrics.", e); this.latencyStats = new LatencyStats( UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup().addGroup("latency"), 1, 0,
public ChainingOutput( OneInputStreamOperator<T, ?> operator, StreamStatusProvider streamStatusProvider, @Nullable OutputTag<T> outputTag) { this.operator = operator; { Counter tmpNumRecordsIn; try { OperatorIOMetricGroup ioMetricGroup = ((OperatorMetricGroup) operator.getMetricGroup()).getIOMetricGroup(); tmpNumRecordsIn = ioMetricGroup.getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e); tmpNumRecordsIn = new SimpleCounter(); } numRecordsIn = tmpNumRecordsIn; } this.streamStatusProvider = streamStatusProvider; this.outputTag = outputTag; }
final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter(); final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();
getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), input1WatermarkGauge, input2WatermarkGauge); headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_2_WATERMARK, input2WatermarkGauge); getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
private static void testLatencyStats( final LatencyStats.Granularity granularity, final Consumer<List<Tuple2<String, Histogram>>> verifier) { final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup(); final TestMetricRegistry registry = new TestMetricRegistry(); final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME); final LatencyStats latencyStats = new LatencyStats( parentGroup, MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(), OPERATOR_SUBTASK_INDEX, OPERATOR_ID, granularity); latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0)); latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0)); latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1)); latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2)); latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3)); verifier.accept(registry.latencyHistograms); }
environment, new NoOpTaskActions(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());
private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter( StreamEdge edge, int outputIndex, Environment environment, String taskName, long bufferTimeout) { @SuppressWarnings("unchecked") StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = RecordWriter.createRecordWriter(bufferWriter, outputPartitioner, bufferTimeout, taskName); output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup()); return output; } }
super.open(); this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME); timestampedCollector = new TimestampedCollector<>(output);
@Override public void register(Metric metric, String metricName, AbstractMetricGroup group) { if (metric instanceof Histogram) { latencyHistograms.add(Tuple2.of(group.getMetricIdentifier(metricName), (Histogram) metric)); } }
metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e);
final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter(); final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public OperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) { return new OperatorMetricGroup(NoOpMetricRegistry.INSTANCE, this, operatorID, name); } };
static String generateJmxDomain(String metricName, MetricGroup group) { return JMX_DOMAIN_PREFIX + ((FrontMetricGroup<AbstractMetricGroup<?>>) group).getLogicalScope(CHARACTER_FILTER, '.') + '.' + metricName; }
metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e);