Remove hardcoded kafka string in Kafka alerts and add Alert for criti…

…cal filling of the PVs (#164)
skodjob · Apr 16, 2024 · b256f9b · b256f9b
1 parent 28604fe
commit b256f9b
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/metrics/alerts/kafka-kraft/kafka-kraft.yaml b/metrics/alerts/kafka-kraft/kafka-kraft.yaml
@@ -332,14 +332,23 @@ spec:
     - name: kafka
       rules:
         - alert: KafkaRunningOutOfSpace
-          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-kafka-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-kafka-[0-9]+"} < 30
-          for: 2m
+          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} < 20
+          for: 5m
           labels:
             severity: warning
             app: strimzi
           annotations:
             summary: 'Kafka ({{ $labels.namespace }}/{{ $labels.pod }}) is running out of free disk space'
             description: 'There are only {{ $value }} percent available at {{ $labels.persistentvolumeclaim }} PVC'
+        - alert: KafkaStorageGettingFull
+          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} < 50
+          for: 5m
+          labels:
+            severity: critical
+            app: strimzi
+          annotations:
+            summary: 'Kafka ({{ $labels.namespace }}/{{ $labels.pod }}) is running out of free disk space'
+            description: 'There are only {{ $value }} percent available at {{ $labels.persistentvolumeclaim }} PVC'
         - alert: UnderReplicatedPartitions
           expr: kafka_server_replicamanager_underreplicatedpartitions > 0
           for: 20m

diff --git a/metrics/alerts/kafka/kafka.yaml b/metrics/alerts/kafka/kafka.yaml
@@ -332,14 +332,23 @@ spec:
     - name: kafka
       rules:
         - alert: KafkaRunningOutOfSpace
-          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-kafka-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-kafka-[0-9]+"} < 30
-          for: 2m
+          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} < 20
+          for: 5m
           labels:
             severity: warning
             app: strimzi
           annotations:
             summary: 'Kafka ({{ $labels.namespace }}/{{ $labels.pod }}) is running out of free disk space'
             description: 'There are only {{ $value }} percent available at {{ $labels.persistentvolumeclaim }} PVC'
+        - alert: KafkaStorageGettingFull
+          expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} * 100 / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data(-[0-9]+)?-(.+)-[0-9]+"} < 5
+          for: 5m
+          labels:
+            severity: critical
+            app: strimzi
+          annotations:
+            summary: 'Kafka ({{ $labels.namespace }}/{{ $labels.pod }}) is running out of free disk space'
+            description: 'There are only {{ $value }} percent available at {{ $labels.persistentvolumeclaim }} PVC'
         - alert: UnderReplicatedPartitions
           expr: kafka_server_replicamanager_underreplicatedpartitions > 0
           for: 20m