Skip to content
This repository has been archived by the owner on Sep 22, 2023. It is now read-only.

Commit

Permalink
update alerts
Browse files Browse the repository at this point in the history
  • Loading branch information
Ulf Lilleengen committed Sep 20, 2023
1 parent 833a652 commit 6eda6ee
Showing 1 changed file with 51 additions and 13 deletions.
64 changes: 51 additions & 13 deletions alerts/prometheusrules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,48 @@ metadata:
name: trustification-alerts
spec:
groups:
# Uptime errors
- name: scraping-errors
rules:
- alert: TrustificationDown
expr: |
absent(up{job="trustification-monitoring",namespace="{{ $labels.namespace }}"} == 1)
for: 5m
labels:
service: trustification
severity: critical
annotations:
message: "{{ $labels.namespace }} monitoring down"

# API errors
- name: api-errors
rules:

- alert: BombasticApiErrors
expr: "(sum(rate(bombastic_api_http_requests_total{status !~ \"5[0-9]+\"}[10m])) / sum(rate(bombastic_api_http_requests_total[10m]))) < 0.99"
expr: |
(sum(rate(bombastic_api_http_requests_total{status !~ "5.."}[10m]))
/
sum(rate(bombastic_api_http_requests_total[10m]))) < 0.99
labels:
service: trustification-bombastic-api
severity: high
annotations:
message: "{{ $labels.namespace }} request success rate below 0.99 for bombastic API"
- alert: VexinationApiErrors
expr: "(sum(rate(vexination_api_http_requests_total{status !~ \"5[0-9]+\"}[10m])) / sum(rate(vexination_api_http_requests_total[10m]))) < 0.99"
expr: |
(sum(rate(vexination_api_http_requests_total{status !~ "5.."}[10m]))
/
sum(rate(vexination_api_http_requests_total[10m]))) < 0.99
labels:
service: trustification-vexination-api
severity: high
annotations:
message: "{{ $labels.namespace }} request success rate below 0.99 for vexination API"
- alert: SpogApiErrors
expr: "(sum(rate(spog_api_http_requests_total{status !~ \"5[0-9]+\"}[10m])) / sum(rate(spog_api_http_requests_total[10m]))) < 0.99"
expr: |
(sum(rate(spog_api_http_requests_total{status !~ "5.."}[10m]))
/
sum(rate(spog_api_http_requests_total[10m]))) < 0.99
labels:
service: trustification-spog-api
severity: high
Expand All @@ -34,40 +57,55 @@ spec:
- name: api-latency
rules:
- alert: BombasticLatency
expr: "(sum(rate(bombastic_api_http_requests_duration_seconds_bucket{exported_endpoint=\"/api/v1/sbom/search\", le=\"1\"}[10m])) / sum(rate(bombastic_api_http_requests_duration_seconds_count{exported_endpoint=\"/api/v1/sbom/search\"}[10m]))) < 0.99"
expr: |
(sum(rate(bombastic_api_http_requests_duration_seconds_bucket{exported_endpoint="/api/v1/sbom/search", status!~"5..", le="1"}[10m]))
/
sum(rate(bombastic_api_http_requests_duration_seconds_count{exported_endpoint="/api/v1/sbom/search", status!~"5.."}[10m]))) < 0.99
labels:
service: trustification-bombastic-api
severity: medium
annotations:
message: "{{ $labels.namespace }} requests within expected response time for API"
message: "{{ $labels.namespace }} bombastic requests above expected response time for API"
- alert: VexinationLatency
expr: "(sum(rate(vexination_api_http_requests_duration_seconds_bucket{exported_endpoint=\"/api/v1/sbom/search\", le=\"1\"}[10m])) / sum(rate(vexination_api_http_requests_duration_seconds_count{exported_endpoint=\"/api/v1/sbom/search\"}[10m]))) < 0.99"
expr: |
(sum(rate(vexination_api_http_requests_duration_seconds_bucket{exported_endpoint="/api/v1/sbom/search", status !~ "5..", le="1"}[10m]))
/
sum(rate(vexination_api_http_requests_duration_seconds_count{exported_endpoint="/api/v1/sbom/search", status !~ "5.."}[10m]))) < 0.99
labels:
service: trustification-vexination-api
severity: medium
annotations:
message: "{{ $labels.namespace }} requests within expected response time for API"
message: "{{ $labels.namespace }} vexination requests within expected response time for API"
- alert: SpogLatency
expr: "(sum(rate(spog_api_http_requests_duration_seconds_bucket{exported_endpoint=~\"/api/v1/.*/search\", le=\"1\"}[10m])) / sum(rate(spog_api_http_requests_duration_seconds_count{exported_endpoint=~\"/api/v1/.*/search\"}[10m]))) < 0.99"
expr: |
(sum(rate(spog_api_http_requests_duration_seconds_bucket{exported_endpoint=~"/api/v1/.*/search", status !~ "5..", le="1"}[10m]))
/
sum(rate(spog_api_http_requests_duration_seconds_count{exported_endpoint=~"/api/v1/.*/search", status !~ "5.."}[10m]))) < 0.99
labels:
service: trustification-spog-api
severity: medium
annotations:
message: "{{ $labels.namespace }} requests within expected response time for API"
message: "{{ $labels.namespace }} spog requests above expected response time for API"
# Indexer errors
- name: indexer-errors
rules:
- alert: BombasticIndexerErrors
expr: "(sum(rate(index_failed_total{pod=~\"bombastic-indexer.* \"}[10m])) / sum(rate(index_indexed_total{pod=~\"bombastic-indexer.*\"}[10m]))) > 0.05"
expr: |
(sum(rate(index_failed_total{pod=~"bombastic-indexer.*"}[10m]))
/
sum(rate(index_indexed_total{pod=~"bombastic-indexer.*"}[10m]))) > 0.01
labels:
service: trustification-bombastic-indexer
severity: low
annotations:
message: "{{ $labels.namespace }} indexer failure rate below 5%"
message: "{{ $labels.namespace }} indexer failure rate above 1%"
- alert: VexinationIndexerErrors
expr: "(sum(rate(index_failed_total{pod=~\"vexination-indexer.* \"}[10m])) / sum(rate(index_indexed_total{pod=~\"vexination-indexer.*\"}[10m]))) > 0.05"
expr: |
(sum(rate(index_failed_total{pod=~"vexination-indexer.*"}[10m]))
/
sum(rate(index_indexed_total{pod=~"vexination-indexer.*"}[10m]))) > 0.01
labels:
service: trustification-vexination-indexer
severity: low
annotations:
message: "{{ $labels.namespace }} indexer failure rate below 5%"
message: "{{ $labels.namespace }} indexer failure rate above 1%"

0 comments on commit 6eda6ee

Please sign in to comment.