From 8c125b8537f6279f3e2fb99715520617a9312d23 Mon Sep 17 00:00:00 2001
From: Jakub Stejskal <xstejs24@gmail.com>
Date: Mon, 17 Jun 2024 12:40:00 +0200
Subject: [PATCH] Update metrics configuration for Kafka, MM2 and Connect
 (#168)

Signed-off-by: Jakub Stejskal <xstejs24@gmail.com>
---
 amq-streams/kafka/00-kafka-route.yaml         | 201 +++++++++-------
 debezium/connector/030-Metrics.yaml           |  71 ++++--
 .../00-cruise-control-dashboard.yaml          |   2 +-
 .../dashboards/00-debezium-dashboard.yaml     |   2 +-
 .../00-kafka-connect-dashboard.yaml           |   2 +-
 .../dashboards/00-kafka-dashboard.yaml        |   2 +-
 .../dashboards/00-kafka-kraft-dashboard.yaml  |   2 +-
 .../00-kafka-mirror-maker-2-dashboard.yaml    |   2 +-
 .../dashboards/00-zookeeper-dashboard.yaml    |   2 +-
 strimzi/kraft-mode/kafka/00-kafka-route.yaml  | 219 ++++++++---------
 .../00-kafka-mirror-maker-2-tls.yaml          |  57 ++++-
 .../zookeeper-mode/kafka/00-kafka-route.yaml  | 225 ++++++++++--------
 .../00-kafka-mirror-maker-2-tls.yaml          |  57 ++++-
 13 files changed, 509 insertions(+), 335 deletions(-)

diff --git a/amq-streams/kafka/00-kafka-route.yaml b/amq-streams/kafka/00-kafka-route.yaml
index 756e7aae..0111f0c4 100755
--- a/amq-streams/kafka/00-kafka-route.yaml
+++ b/amq-streams/kafka/00-kafka-route.yaml
@@ -255,121 +255,162 @@ metadata:
   name: heimdall-kafka-jmx-exporter-configuration
 data:
   heimdall-kafka-jmx-exporter-configuration.yaml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
     lowercaseOutputName: true
     rules:
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    # Special cases and very specific rules
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         topic: "$4"
         partition: "$5"
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         broker: "$4:$5"
-    - pattern: "kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_tls_info"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_tls_info
+      type: GAUGE
       labels:
-        listener: "$2"
-        networkProcessor: "$3"
-        protocol: "$4"
-        cipher: "$5"
-    - pattern: "kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+),\
-        \ listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_software"
-      type: "GAUGE"
+        cipher: "$2"
+        protocol: "$3"
+        listener: "$4"
+        networkProcessor: "$5"
+    - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_software
+      type: GAUGE
       labels:
         clientSoftwareName: "$2"
         clientSoftwareVersion: "$3"
         listener: "$4"
         networkProcessor: "$5"
+    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total):"
+      name: kafka_server_$1_$4
+      type: COUNTER
+      labels:
+        listener: "$2"
+        networkProcessor: "$3"
     - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total)
+      name: kafka_server_$1_$4
+      type: COUNTER
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>MeanRate"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*, (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        listener: "$2"
+        networkProcessor: "$3"
+    # Some percent metrics use MeanRate attribute
+    # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    # Generic gauges for percents
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        "$4": "$5"
+    # Generic per-second counters with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+    # Generic gauges with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+    # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+    # Note that these are missing the '_sum' metric!
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        "$6": "$7"
         quantile: "0.$8"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
+        "$4": "$5"
         quantile: "0.$6"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
         quantile: "0.$4"
+    # KRaft mode: uncomment the following lines to export KRaft related metrics
+    # KRaft overall related metrics
+    # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics
+    #- pattern: "kafka.server<type=raft-metrics><>(.+-total|.+-max):"
+    #  name: kafka_server_raftmetrics_$1
+    #  type: COUNTER
+    #- pattern: "kafka.server<type=raft-metrics><>(.+):"
+    #  name: kafka_server_raftmetrics_$1
+    #  type: GAUGE
+    # KRaft "low level" channels related metrics
+    # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics
+    #- pattern: "kafka.server<type=raft-channel-metrics><>(.+-total|.+-max):"
+    #  name: kafka_server_raftchannelmetrics_$1
+    #  type: COUNTER
+    #- pattern: "kafka.server<type=raft-channel-metrics><>(.+):"
+    #  name: kafka_server_raftchannelmetrics_$1
+    #  type: GAUGE
+    # Broker metrics related to fetching metadata topic records in KRaft mode
+    #- pattern: "kafka.server<type=broker-metadata-metrics><>(.+):"
+    #  name: kafka_server_brokermetadatametrics_$1
+    #  type: GAUGE
 ---
 apiVersion: v1
 kind: ConfigMap
diff --git a/debezium/connector/030-Metrics.yaml b/debezium/connector/030-Metrics.yaml
index 0541a934..59b42f54 100755
--- a/debezium/connector/030-Metrics.yaml
+++ b/debezium/connector/030-Metrics.yaml
@@ -30,10 +30,18 @@ data:
         clientId: "$2"
         $3: "$4"
       help: "Kafka $1 JMX metric info version and commit-id"
-      type: GAUGE
+      type: UNTYPED
 
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
-    - pattern: kafka.consumer<type=consumer-fetch-manager-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total|compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
+    - pattern: kafka.consumer<type=consumer-fetch-manager-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total)
+      name: kafka_consumer_fetch_manager_$4
+      labels:
+        clientId: "$1"
+        topic: "$2"
+        partition: "$3"
+      help: "Kafka Consumer JMX metric type consumer-fetch-manager-metrics"
+      type: COUNTER
+    - pattern: kafka.consumer<type=consumer-fetch-manager-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
       name: kafka_consumer_fetch_manager_$4
       labels:
         clientId: "$1"
@@ -43,7 +51,14 @@ data:
       type: GAUGE
 
     #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"
-    - pattern: kafka.producer<type=producer-topic-metrics, client-id=(.+), topic=(.+)><>(.+-total|compression-rate|.+-avg|.+rate)
+    - pattern: kafka.producer<type=producer-topic-metrics, client-id=(.+), topic=(.+)><>(.+-total)
+      name: kafka_producer_topic_$3
+      labels:
+        clientId: "$1"
+        topic: "$2"
+      help: "Kafka Producer JMX metric type producer-topic-metrics"
+      type: COUNTER
+    - pattern: kafka.producer<type=producer-topic-metrics, client-id=(.+), topic=(.+)><>(compression-rate|.+-avg|.+rate)
       name: kafka_producer_topic_$3
       labels:
         clientId: "$1"
@@ -53,7 +68,14 @@ data:
 
     #kafka.connect:type=connect-node-metrics,client-id="{clientid}",node-id="{nodeid}"
     #kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id="{nodeid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total|.+-avg|.+-rate)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total)
+      name: kafka_$2_$5
+      labels:
+        clientId: "$3"
+        nodeId: "$4"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-avg|.+-rate)
       name: kafka_$2_$5
       labels:
         clientId: "$3"
@@ -65,7 +87,13 @@ data:
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-coordinator-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-metrics,client-id="{clientid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total|.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total)
+      name: kafka_$2_$4
+      labels:
+        clientId: "$3"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
       name: kafka_$2_$4
       labels:
         clientId: "$3"
@@ -97,7 +125,14 @@ data:
     #kafka.connect:type=source-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=sink-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}"
-    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total|.+-count|.+-ms|.+-ratio|.+-seq-no|.+-rate|.+-max|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total)
+      name: kafka_connect_$1_$4
+      labels:
+        connector: "$2"
+        task: "$3"
+      help: "Kafka Connect JMX metric type $1"
+      type: COUNTER
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-count|.+-ms|.+-ratio|.+-seq-no|.+-rate|.+-max|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
       name: kafka_connect_$1_$4
       labels:
         connector: "$2"
@@ -114,6 +149,10 @@ data:
       type: GAUGE
 
     #kafka.connect:type=connect-worker-metrics
+    - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_$1
+      help: "Kafka Connect JMX metric worker"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+)
       name: kafka_connect_worker_$1
       help: "Kafka Connect JMX metric worker"
@@ -129,6 +168,10 @@ data:
       type: UNTYPED
 
     #kafka.connect:type=connect-worker-rebalance-metrics
+    - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_rebalance_$1
+      help: "Kafka Connect JMX metric rebalance information"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+)
       name: kafka_connect_worker_rebalance_$1
       help: "Kafka Connect JMX metric rebalance information"
@@ -139,19 +182,3 @@ data:
       name: kafka_connect_coordinator_$1
       help: "Kafka Connect JMX metric assignment information"
       type: GAUGE
-    
-    # Debezium specific metrics
-    - pattern: "debezium.([^:]+)<type=connector-metrics, context=([^,]+), server=([^,]+), key=([^>]+)><>RowsScanned"
-      name: "debezium_metrics_RowsScanned"
-      labels:
-        plugin: "$1"
-        name: "$3"
-        context: "$2"
-        table: "$4"
-
-    - pattern: "debezium.([^:]+)<type=connector-metrics, context=([^,]+), server=([^>]+)>([^:]+)"
-      name: "debezium_metrics_$4"
-      labels:
-        plugin: "$1"
-        name: "$3"
-        context: "$2"
diff --git a/metrics/grafana/dashboards/00-cruise-control-dashboard.yaml b/metrics/grafana/dashboards/00-cruise-control-dashboard.yaml
index 4ec4d478..20b3c73a 100755
--- a/metrics/grafana/dashboards/00-cruise-control-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-cruise-control-dashboard.yaml
@@ -1584,7 +1584,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-cruise-control-.*\",strimzi_io_name=\"$strimzi_cluster_name-cruise-control\"}) by (kubernetes_pod_name)",
+              "expr": "sum(jvm_memory_used_bytes{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-cruise-control-.*\",strimzi_io_name=\"$strimzi_cluster_name-cruise-control\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{kubernetes_pod_name}}",
diff --git a/metrics/grafana/dashboards/00-debezium-dashboard.yaml b/metrics/grafana/dashboards/00-debezium-dashboard.yaml
index 6ed36dae..5740805c 100755
--- a/metrics/grafana/dashboards/00-debezium-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-debezium-dashboard.yaml
@@ -1327,7 +1327,7 @@ spec:
                         "steppedLine": false,
                         "targets": [
                             {
-                                "expr": "jvm_memory_bytes_used{area='heap', clusterName=~\"$OPENSHIFT_CLUSTER\"}",
+                                "expr": "jvm_memory_used_bytes{area='heap', clusterName=~\"$OPENSHIFT_CLUSTER\"}",
                                 "format": "time_series",
                                 "intervalFactor": 1,
                                 "legendFormat": "Used",
diff --git a/metrics/grafana/dashboards/00-kafka-connect-dashboard.yaml b/metrics/grafana/dashboards/00-kafka-connect-dashboard.yaml
index 6d86388c..7fe58376 100755
--- a/metrics/grafana/dashboards/00-kafka-connect-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-kafka-connect-dashboard.yaml
@@ -1496,7 +1496,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum (jvm_memory_bytes_used{strimzi_io_kind=~\"KafkaConnect.*\",strimzi_io_cluster=\"$strimzi_connect_cluster_name\"}) by (kubernetes_pod_name)",
+              "expr": "sum (jvm_memory_used_bytes{strimzi_io_kind=~\"KafkaConnect.*\",strimzi_io_cluster=\"$strimzi_connect_cluster_name\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
diff --git a/metrics/grafana/dashboards/00-kafka-dashboard.yaml b/metrics/grafana/dashboards/00-kafka-dashboard.yaml
index 5046d431..2de3f36e 100755
--- a/metrics/grafana/dashboards/00-kafka-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-kafka-dashboard.yaml
@@ -1137,7 +1137,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)",
+              "expr": "sum(jvm_memory_used_bytes{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kafka_broker\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{kubernetes_pod_name}}",
diff --git a/metrics/grafana/dashboards/00-kafka-kraft-dashboard.yaml b/metrics/grafana/dashboards/00-kafka-kraft-dashboard.yaml
index f654409c..0706fe65 100755
--- a/metrics/grafana/dashboards/00-kafka-kraft-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-kafka-kraft-dashboard.yaml
@@ -506,7 +506,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kraft_node\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)",
+              "expr": "sum(jvm_memory_used_bytes{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$kraft_node\",strimzi_io_name=\"$strimzi_cluster_name-kafka\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{kubernetes_pod_name}}",
diff --git a/metrics/grafana/dashboards/00-kafka-mirror-maker-2-dashboard.yaml b/metrics/grafana/dashboards/00-kafka-mirror-maker-2-dashboard.yaml
index 257e35d4..04abd706 100755
--- a/metrics/grafana/dashboards/00-kafka-mirror-maker-2-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-kafka-mirror-maker-2-dashboard.yaml
@@ -2140,7 +2140,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum(jvm_memory_bytes_used{strimzi_io_kind=~\"KafkaMirrorMaker2\",strimzi_io_cluster=\"$strimzi_mirror_maker_cluster_name\", clusterName=~\"$OPENSHIFT_CLUSTER\"}) by (kubernetes_pod_name)",
+              "expr": "sum(jvm_memory_used_bytes{strimzi_io_kind=~\"KafkaMirrorMaker2\",strimzi_io_cluster=\"$strimzi_mirror_maker_cluster_name\", clusterName=~\"$OPENSHIFT_CLUSTER\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 2,
diff --git a/metrics/grafana/dashboards/00-zookeeper-dashboard.yaml b/metrics/grafana/dashboards/00-zookeeper-dashboard.yaml
index 4e8d5a82..403d0acb 100755
--- a/metrics/grafana/dashboards/00-zookeeper-dashboard.yaml
+++ b/metrics/grafana/dashboards/00-zookeeper-dashboard.yaml
@@ -1001,7 +1001,7 @@ spec:
           "targets": [
             {
               "datasource": "${DS_THANOS}",
-              "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}) by (kubernetes_pod_name)",
+              "expr": "sum(jvm_memory_used_bytes{namespace=\"$kubernetes_namespace\",kubernetes_pod_name=~\"$strimzi_cluster_name-$zk_node\",strimzi_io_name=\"$strimzi_cluster_name-zookeeper\"}) by (kubernetes_pod_name)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{kubernetes_pod_name}}",
diff --git a/strimzi/kraft-mode/kafka/00-kafka-route.yaml b/strimzi/kraft-mode/kafka/00-kafka-route.yaml
index e5b07d84..4345b905 100755
--- a/strimzi/kraft-mode/kafka/00-kafka-route.yaml
+++ b/strimzi/kraft-mode/kafka/00-kafka-route.yaml
@@ -326,119 +326,139 @@ metadata:
   name: horus-kafka-jmx-exporter-configuration
 data:
   horus-kafka-jmx-exporter-configuration.yaml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
     lowercaseOutputName: true
     rules:
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    # Special cases and very specific rules
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         topic: "$4"
         partition: "$5"
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         broker: "$4:$5"
-    - pattern: "kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_tls_info"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_tls_info
+      type: GAUGE
       labels:
-        listener: "$2"
-        networkProcessor: "$3"
-        protocol: "$4"
-        cipher: "$5"
-    - pattern: "kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+),\
-        \ listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_software"
-      type: "GAUGE"
+        cipher: "$2"
+        protocol: "$3"
+        listener: "$4"
+        networkProcessor: "$5"
+    - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_software
+      type: GAUGE
       labels:
         clientSoftwareName: "$2"
         clientSoftwareVersion: "$3"
         listener: "$4"
         networkProcessor: "$5"
+    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total):"
+      name: kafka_server_$1_$4
+      type: COUNTER
+      labels:
+        listener: "$2"
+        networkProcessor: "$3"
     - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total)
+      name: kafka_server_$1_$4
+      type: COUNTER
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>MeanRate"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*, (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        listener: "$2"
+        networkProcessor: "$3"
+    # Some percent metrics use MeanRate attribute
+    # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    # Generic gauges for percents
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        "$4": "$5"
+    # Generic per-second counters with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+    # Generic gauges with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+    # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+    # Note that these are missing the '_sum' metric!
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        "$6": "$7"
         quantile: "0.$8"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
+        "$4": "$5"
         quantile: "0.$6"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
         quantile: "0.$4"
     # KRaft mode: uncomment the following lines to export KRaft related metrics
@@ -465,45 +485,6 @@ data:
 ---
 apiVersion: v1
 kind: ConfigMap
-metadata:
-  name: horus-zookeeper-jmx-exporter-configuration
-data:
-  horus-zookeeper-jmx-exporter-configuration.yaml: |
-    lowercaseOutputName: true
-    rules:
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)"
-      name: "zookeeper_$2"
-      type: "GAUGE"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+)><>(\\w+)"
-      name: "zookeeper_$3"
-      type: "GAUGE"
-      labels:
-        replicaId: "$2"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+)><>(Packets\\w+)"
-      name: "zookeeper_$4"
-      type: "COUNTER"
-      labels:
-        replicaId: "$2"
-        memberType: "$3"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+)><>(\\w+)"
-      name: "zookeeper_$4"
-      type: "GAUGE"
-      labels:
-        replicaId: "$2"
-        memberType: "$3"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
-      name: "zookeeper_$4_$5"
-      type: "GAUGE"
-      labels:
-        replicaId: "$2"
-        memberType: "$3"
----
-apiVersion: v1
-kind: ConfigMap
 metadata:
   name: horus-cruise-control-jmx-exporter-configuration
 data:
diff --git a/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml b/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
index d66d704e..9ccc7788 100755
--- a/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
+++ b/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
@@ -127,11 +127,19 @@ data:
         clientId: "$2"
         $3: "$4"
       help: "Kafka $1 JMX metric info version and commit-id"
-      type: GAUGE
+      type: UNTYPED
 
     #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total|compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total)
+      name: kafka_$2_$6
+      labels:
+        clientId: "$3"
+        topic: "$4"
+        partition: "$5"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
       name: kafka_$2_$6
       labels:
         clientId: "$3"
@@ -142,7 +150,14 @@ data:
 
     #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(.+-total|compression-rate|.+-avg)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(.+-total)
+      name: kafka_$2_$5
+      labels:
+        clientId: "$3"
+        topic: "$4"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(compression-rate|.+-avg)
       name: kafka_$2_$5
       labels:
         clientId: "$3"
@@ -152,19 +167,32 @@ data:
 
     #kafka.connect:type=connect-node-metrics,client-id="{clientid}",node-id="{nodeid}"
     #kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id="{nodeid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total|.+-avg)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total)
       name: kafka_$2_$5
       labels:
         clientId: "$3"
         nodeId: "$4"
       help: "Kafka $1 JMX metric type $2"
-      type: UNTYPED
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-avg)
+      name: kafka_$2_$5
+      labels:
+        clientId: "$3"
+        nodeId: "$4"
+      help: "Kafka $1 JMX metric type $2"
+      type: GAUGE
 
     #kafka.connect:type=kafka-metrics-count,client-id="{clientid}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-coordinator-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-metrics,client-id="{clientid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total|.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total)
+      name: kafka_$2_$4
+      labels:
+        clientId: "$3"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
       name: kafka_$2_$4
       labels:
         clientId: "$3"
@@ -186,7 +214,14 @@ data:
     #kafka.connect:type=source-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=sink-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}"
-    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total|.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total)
+      name: kafka_connect_$1_$4
+      labels:
+        connector: "$2"
+        task: "$3"
+      help: "Kafka Connect JMX metric type $1"
+      type: COUNTER
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
       name: kafka_connect_$1_$4
       labels:
         connector: "$2"
@@ -204,12 +239,20 @@ data:
       type: GAUGE
 
     #kafka.connect:type=connect-worker-metrics
+    - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_$1
+      help: "Kafka Connect JMX metric worker"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+)
       name: kafka_connect_worker_$1
       help: "Kafka Connect JMX metric worker"
       type: GAUGE
 
     #kafka.connect:type=connect-worker-rebalance-metrics
+    - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_rebalance_$1
+      help: "Kafka Connect JMX metric rebalance information"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+)
       name: kafka_connect_worker_rebalance_$1
       help: "Kafka Connect JMX metric rebalance information"
diff --git a/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml b/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml
index d7ecf618..cbe94e92 100755
--- a/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml
+++ b/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml
@@ -301,121 +301,162 @@ metadata:
   name: anubis-kafka-jmx-exporter-configuration
 data:
   anubis-kafka-jmx-exporter-configuration.yaml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
     lowercaseOutputName: true
     rules:
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    # Special cases and very specific rules
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         topic: "$4"
         partition: "$5"
-    - pattern: "kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value"
-      name: "kafka_server_$1_$2"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
       labels:
         clientId: "$3"
         broker: "$4:$5"
-    - pattern: "kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_tls_info"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_tls_info
+      type: GAUGE
       labels:
-        listener: "$2"
-        networkProcessor: "$3"
-        protocol: "$4"
-        cipher: "$5"
-    - pattern: "kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+),\
-        \ listener=(.+), networkProcessor=(.+)><>connections"
-      name: "kafka_server_$1_connections_software"
-      type: "GAUGE"
+        cipher: "$2"
+        protocol: "$3"
+        listener: "$4"
+        networkProcessor: "$5"
+    - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_software
+      type: GAUGE
       labels:
         clientSoftwareName: "$2"
         clientSoftwareVersion: "$3"
         listener: "$4"
         networkProcessor: "$5"
+    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total):"
+      name: kafka_server_$1_$4
+      type: COUNTER
+      labels:
+        listener: "$2"
+        networkProcessor: "$3"
     - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)"
-      name: "kafka_server_$1_$4"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+-total)
+      name: kafka_server_$1_$4
+      type: COUNTER
       labels:
         listener: "$2"
         networkProcessor: "$3"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>MeanRate"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)Percent\\w*, (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3_percent"
-      type: "GAUGE"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
+      name: kafka_server_$1_$4
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        listener: "$2"
+        networkProcessor: "$3"
+    # Some percent metrics use MeanRate attribute
+    # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    # Generic gauges for percents
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*, (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
+        "$4": "$5"
+    # Generic per-second counters with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)PerSec\\w*><>Count"
-      name: "kafka_$1_$2_$3_total"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+    # Generic gauges with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Value"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
-        $6: "$7"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+    # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+    # Note that these are missing the '_sum' metric!
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-        $6: "$7"
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        "$6": "$7"
         quantile: "0.$8"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
       labels:
-        $4: "$5"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
-        $4: "$5"
+        "$4": "$5"
         quantile: "0.$6"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>Count"
-      name: "kafka_$1_$2_$3_count"
-      type: "COUNTER"
-    - pattern: "kafka.(\\w+)<type=(.+), name=(.+)><>(\\d+)thPercentile"
-      name: "kafka_$1_$2_$3"
-      type: "GAUGE"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
       labels:
         quantile: "0.$4"
+    # KRaft mode: uncomment the following lines to export KRaft related metrics
+    # KRaft overall related metrics
+    # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics
+    #- pattern: "kafka.server<type=raft-metrics><>(.+-total|.+-max):"
+    #  name: kafka_server_raftmetrics_$1
+    #  type: COUNTER
+    #- pattern: "kafka.server<type=raft-metrics><>(.+):"
+    #  name: kafka_server_raftmetrics_$1
+    #  type: GAUGE
+    # KRaft "low level" channels related metrics
+    # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics
+    #- pattern: "kafka.server<type=raft-channel-metrics><>(.+-total|.+-max):"
+    #  name: kafka_server_raftchannelmetrics_$1
+    #  type: COUNTER
+    #- pattern: "kafka.server<type=raft-channel-metrics><>(.+):"
+    #  name: kafka_server_raftchannelmetrics_$1
+    #  type: GAUGE
+    # Broker metrics related to fetching metadata topic records in KRaft mode
+    #- pattern: "kafka.server<type=broker-metadata-metrics><>(.+):"
+    #  name: kafka_server_brokermetadatametrics_$1
+    #  type: GAUGE
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -423,35 +464,33 @@ metadata:
   name: anubis-zookeeper-jmx-exporter-configuration
 data:
   anubis-zookeeper-jmx-exporter-configuration.yaml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
     lowercaseOutputName: true
     rules:
+    # replicated Zookeeper
     - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)"
       name: "zookeeper_$2"
-      type: "GAUGE"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+)><>(\\w+)"
+      type: GAUGE
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)"
       name: "zookeeper_$3"
-      type: "GAUGE"
+      type: GAUGE
       labels:
         replicaId: "$2"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+)><>(Packets\\w+)"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(Packets\\w+)"
       name: "zookeeper_$4"
-      type: "COUNTER"
+      type: COUNTER
       labels:
         replicaId: "$2"
         memberType: "$3"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+)><>(\\w+)"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)"
       name: "zookeeper_$4"
-      type: "GAUGE"
+      type: GAUGE
       labels:
         replicaId: "$2"
         memberType: "$3"
-    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\\
-        d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
       name: "zookeeper_$4_$5"
-      type: "GAUGE"
+      type: GAUGE
       labels:
         replicaId: "$2"
         memberType: "$3"
diff --git a/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml b/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
index 36d337c0..1eb7d189 100755
--- a/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
+++ b/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml
@@ -127,11 +127,19 @@ data:
         clientId: "$2"
         $3: "$4"
       help: "Kafka $1 JMX metric info version and commit-id"
-      type: GAUGE
+      type: UNTYPED
 
     #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total|compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total)
+      name: kafka_$2_$6
+      labels:
+        clientId: "$3"
+        topic: "$4"
+        partition: "$5"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
       name: kafka_$2_$6
       labels:
         clientId: "$3"
@@ -142,7 +150,14 @@ data:
 
     #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(.+-total|compression-rate|.+-avg)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(.+-total)
+      name: kafka_$2_$5
+      labels:
+        clientId: "$3"
+        topic: "$4"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(compression-rate|.+-avg)
       name: kafka_$2_$5
       labels:
         clientId: "$3"
@@ -152,19 +167,32 @@ data:
 
     #kafka.connect:type=connect-node-metrics,client-id="{clientid}",node-id="{nodeid}"
     #kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id="{nodeid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total|.+-avg)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total)
       name: kafka_$2_$5
       labels:
         clientId: "$3"
         nodeId: "$4"
       help: "Kafka $1 JMX metric type $2"
-      type: UNTYPED
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-avg)
+      name: kafka_$2_$5
+      labels:
+        clientId: "$3"
+        nodeId: "$4"
+      help: "Kafka $1 JMX metric type $2"
+      type: GAUGE
 
     #kafka.connect:type=kafka-metrics-count,client-id="{clientid}"
     #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-coordinator-metrics,client-id="{clientid}"
     #kafka.consumer:type=consumer-metrics,client-id="{clientid}"
-    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total|.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total)
+      name: kafka_$2_$4
+      labels:
+        clientId: "$3"
+      help: "Kafka $1 JMX metric type $2"
+      type: COUNTER
+    - pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
       name: kafka_$2_$4
       labels:
         clientId: "$3"
@@ -186,7 +214,14 @@ data:
     #kafka.connect:type=source-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=sink-task-metrics,connector="{connector}",task="{task}"
     #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}"
-    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total|.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total)
+      name: kafka_connect_$1_$4
+      labels:
+        connector: "$2"
+        task: "$3"
+      help: "Kafka Connect JMX metric type $1"
+      type: COUNTER
+    - pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
       name: kafka_connect_$1_$4
       labels:
         connector: "$2"
@@ -204,12 +239,20 @@ data:
       type: GAUGE
 
     #kafka.connect:type=connect-worker-metrics
+    - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_$1
+      help: "Kafka Connect JMX metric worker"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+)
       name: kafka_connect_worker_$1
       help: "Kafka Connect JMX metric worker"
       type: GAUGE
 
     #kafka.connect:type=connect-worker-rebalance-metrics
+    - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+-total)
+      name: kafka_connect_worker_rebalance_$1
+      help: "Kafka Connect JMX metric rebalance information"
+      type: COUNTER
     - pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+)
       name: kafka_connect_worker_rebalance_$1
       help: "Kafka Connect JMX metric rebalance information"