void addMetric(CruiseControlMetric metric) { int brokerId = metric.brokerId(); LOG.trace("Adding cruise control metric {}", metric); _maxMetricTimestamp = Math.max(metric.time(), _maxMetricTimestamp); _brokerLoad.compute(brokerId, (bid, load) -> { BrokerLoad brokerLoad = load == null ? new BrokerLoad() : load; brokerLoad.recordMetric(metric); return brokerLoad; }); }
/** * Send a CruiseControlMetric to the Kafka topic. * @param ccm the Cruise Control metric to send. */ public void sendCruiseControlMetric(CruiseControlMetric ccm) { // Use topic name as key if existing so that the same sampler will be able to collect all the information // of a topic. String key = ccm.metricClassId() == CruiseControlMetric.MetricClassId.TOPIC_METRIC ? ((TopicMetric) ccm).topic() : Integer.toString(ccm.brokerId()); ProducerRecord<String, CruiseControlMetric> producerRecord = new ProducerRecord<>(_cruiseControlMetricsTopic, null, ccm.time(), key, ccm); LOG.debug("Sending Cruise Control metric {}.", ccm); _producer.send(producerRecord, new Callback() { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { LOG.warn("Failed to send Cruise Control metric {}", ccm); _numMetricSendFailure++; } } }); }
@Test public void testMissingOtherBrokerMetrics() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); for (CruiseControlMetric metric : metrics) { if (metric.rawMetricType() == RawMetricType.BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX && metric.brokerId() == BROKER_ID_0) { // Do nothing and skip the metric. } else { processor.addMetric(metric); } } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals("Should have all 4 partition metrics.", 4, samples.partitionMetricSamples().size()); assertEquals("Should have ignored broker 0", 1, samples.brokerMetricSamples().size()); }
@Test public void testMissingBrokerCpuUtilization() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); for (CruiseControlMetric metric : metrics) { if (metric.rawMetricType() == RawMetricType.BROKER_CPU_UTIL && metric.brokerId() == BROKER_ID_0) { // Do nothing and skip the metric. } else { processor.addMetric(metric); } } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals("Should have ignored partitions on broker 0", 1, samples.partitionMetricSamples().size()); assertEquals("Should have ignored broker 0", 1, samples.brokerMetricSamples().size()); }
@Test public void testBrokerMetricSerde() throws UnknownVersionException { BrokerMetric brokerMetric = new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, 123L, 0, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(brokerMetric)); assertEquals(CruiseControlMetric.MetricClassId.BROKER_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.ALL_TOPIC_BYTES_IN.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(VALUE, deserialized.value(), 0.000001); }
@Test public void testTopicMetricSerde() throws UnknownVersionException { TopicMetric topicMetric = new TopicMetric(RawMetricType.TOPIC_BYTES_IN, 123L, 0, TOPIC, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(topicMetric)); assertEquals(CruiseControlMetric.MetricClassId.TOPIC_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.TOPIC_BYTES_IN.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(TOPIC, ((TopicMetric) deserialized).topic()); assertEquals(VALUE, deserialized.value(), 0.000001); }
@Test public void testPartitionMetricSerde() throws UnknownVersionException { PartitionMetric partitionMetric = new PartitionMetric(RawMetricType.PARTITION_SIZE, 123L, 0, TOPIC, PARTITION, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(partitionMetric)); assertEquals(CruiseControlMetric.MetricClassId.PARTITION_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.PARTITION_SIZE.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(TOPIC, ((PartitionMetric) deserialized).topic()); assertEquals(PARTITION, ((PartitionMetric) deserialized).partition()); assertEquals(VALUE, deserialized.value(), 0.000001); } }
/** * Send a CruiseControlMetric to the Kafka topic. * @param ccm the Cruise Control metric to send. */ public void sendCruiseControlMetric(CruiseControlMetric ccm) { // Use topic name as key if existing so that the same sampler will be able to collect all the information // of a topic. String key = ccm.metricClassId() == CruiseControlMetric.MetricClassId.TOPIC_METRIC ? ((TopicMetric) ccm).topic() : Integer.toString(ccm.brokerId()); ProducerRecord<String, CruiseControlMetric> producerRecord = new ProducerRecord<>(_cruiseControlMetricsTopic, null, ccm.time(), key, ccm); LOG.debug("Sending Cruise Control metric {}.", ccm); _producer.send(producerRecord, new Callback() { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { LOG.warn("Failed to send Cruise Control metric {}", ccm); _numMetricSendFailure++; } } }); }