*: add grafana mixin

monitoring-mixins · Dec 9, 2022 · a51b164 · a51b164
1 parent 314aaa6
commit a51b164
Show file tree

Hide file tree

Showing 6 changed files with 642 additions and 0 deletions.
diff --git a/assets/grafana/alerts.yaml b/assets/grafana/alerts.yaml
@@ -0,0 +1,15 @@
+groups:
+- name: GrafanaAlerts
+  rules:
+  - alert: GrafanaRequestsFailing
+    annotations:
+      message: '{{ $labels.namespace }}/{{ $labels.job }}/{{ $labels.handler }} is
+        experiencing {{ $value | humanize }}% errors'
+    expr: |
+      100 * namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."}
+      / ignoring (status_code)
+      sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query"})
+      > 50
+    for: 5m
+    labels:
+      severity: warning