Merge pull request #49 from K-Phoen/yaml-alerts

Yaml alerts

alert/alert.go

@@ -62,7 +62,7 @@ func New(name string, options ...Option) *Alert {
 	return alert
 }
 
-// Notify adds a notification for this alert.
+// Notify adds a notification for this alert given a channel.
 func Notify(channel *Channel) Option {
 	return func(alert *Alert) {
 		alert.Builder.Notifications = append(alert.Builder.Notifications, sdk.AlertNotification{
@@ -72,6 +72,15 @@ func Notify(channel *Channel) Option {
 	}
 }
 
+// NotifyChannel adds a notification for this alert given a channel ID.
+func NotifyChannel(channel int64) Option {
+	return func(alert *Alert) {
+		alert.Builder.Notifications = append(alert.Builder.Notifications, sdk.AlertNotification{
+			ID: channel,
+		})
+	}
+}
+
 // Message sets the message associated to the alert.
 func Message(content string) Option {
 	return func(alert *Alert) {

alert/alert_test.go

@@ -35,6 +35,16 @@ func TestNotificationCanBeSet(t *testing.T) {
 	req.Equal(int64(1), a.Builder.Notifications[0].ID)
 }
 
+func TestNotificationCanBeSetByChannelID(t *testing.T) {
+	req := require.New(t)
+
+	a := New("", NotifyChannel(1))
+
+	req.Len(a.Builder.Notifications, 1)
+	req.Empty(a.Builder.Notifications[0].UID)
+	req.Equal(int64(1), a.Builder.Notifications[0].ID)
+}
+
 func TestForIntervalCanBeSet(t *testing.T) {
 	req := require.New(t)
 

cmd/yaml/example.yaml

@@ -63,6 +63,19 @@ rows:
                 query: "go_memstats_heap_alloc_bytes"
                 legend: "{{job}}"
                 ref: A
+          alert:
+            title: Too many heap allocations
+            evaluate_every: 1m
+            for: 1m
+            notify: 1
+            message: "Wow, a we're allocating a lot."
+            on_no_data: alerting
+            on_execution_error: alerting
+            if:
+              - operand: and
+                value: {func: avg, ref: A, from: 1m, to: now}
+                threshold: {above: 23000000}
+
       - table:
           title: Threads
           datasource: prometheus-default

decoder/dashboard_test.go

@@ -164,6 +164,56 @@ rows:
 	require.Equal(t, ErrTargetNotConfigured, err)
 }
 
+func TestUnmarshalYAMLWithNoAlertThresholdGraph(t *testing.T) {
+	payload := `
+rows:
+  - name: Test row
+    panels:
+      - graph:
+          title: Heap allocations
+          alert:
+            title: Too many heap allocations
+            evaluate_every: 1m
+            for: 1m
+            if:
+              - operand: and
+                value: {func: avg, ref: A, from: 1m, to: now}
+
+          targets:
+            - prometheus: { query: "go_memstats_heap_alloc_bytes" }
+`
+
+	_, err := UnmarshalYAML(bytes.NewBufferString(payload))
+
+	require.Error(t, err)
+	require.Equal(t, ErrNoAlertThresholdDefined, err)
+}
+
+func TestUnmarshalYAMLWithInvalidAlertValueFunctionGraph(t *testing.T) {
+	payload := `
+rows:
+  - name: Test row
+    panels:
+      - graph:
+          title: Heap allocations
+          alert:
+            title: Too many heap allocations
+            evaluate_every: 1m
+            for: 1m
+            if:
+              - operand: and
+                value: {func: BLOOPER, ref: A, from: 1m, to: now}
+                threshold: {above: 23000000}
+          targets:
+            - prometheus: { query: "go_memstats_heap_alloc_bytes" }
+`
+
+	_, err := UnmarshalYAML(bytes.NewBufferString(payload))
+
+	require.Error(t, err)
+	require.Equal(t, ErrInvalidAlertValueFunc, err)
+}
+
 func generalOptions() testCase {
 	yaml := `title: Awesome dashboard
 
@@ -536,6 +586,18 @@ rows:
           height: 400px
           span: 4
           datasource: prometheus-default
+          alert:
+            title: Too many heap allocations
+            evaluate_every: 1m
+            for: 1m
+            notify: 1
+            message: "Wow, a we're allocating a lot."
+            on_no_data: alerting
+            on_execution_error: alerting
+            if:
+              - operand: and
+                value: {func: avg, ref: A, from: 1m, to: now}
+                threshold: {above: 23000000}
           axes:
             left: { unit: short, min: 0, max: 100, label: Requests }
             right: { hidden: true }
@@ -583,6 +645,39 @@ rows:
 					"fill": 1,
 					"title": "Heap allocations",
 					"aliasColors": {},
+					"alert": {
+						"conditions": [
+							{
+								"evaluator": {
+									"params": [23000000],
+									"type": "gt"
+								},
+								"operator": {"type": "and"},
+								"query": {"params": ["A", "1m", "now"]},
+								"reducer": {"type": "avg"},
+								"type": "query"
+							}
+						],
+						"executionErrorState": "alerting",
+						"for": "1m",
+						"frequency": "1m",
+						"handler": 1,
+						"message": "Wow, a we're allocating a lot.",
+						"name": "Too many heap allocations",
+						"noDataState": "alerting",
+						"notifications": [
+							{
+								"disableResolveMessage": false,
+								"frequency": "",
+								"id": 1,
+								"isDefault": false,
+								"name": "",
+								"sendReminder": false,
+								"settings": null,
+								"type": ""
+							}
+						]
+					},
 					"bars": false,
 					"points": false,
 					"stack": false,

decoder/graph.go

@@ -1,18 +1,25 @@
 package decoder
 
 import (
+	"fmt"
+
+	"github.com/K-Phoen/grabana/alert"
 	"github.com/K-Phoen/grabana/axis"
 	"github.com/K-Phoen/grabana/graph"
 	"github.com/K-Phoen/grabana/row"
 )
 
+var ErrNoAlertThresholdDefined = fmt.Errorf("no threshold defined")
+var ErrInvalidAlertValueFunc = fmt.Errorf("invalid alert value function")
+
 type dashboardGraph struct {
 	Title      string
 	Span       float32
 	Height     string
 	Datasource string
 	Targets    []target
 	Axes       graphAxes
+	Alert      *graphAlert
 }
 
 func (graphPanel dashboardGraph) toOption() (row.Option, error) {
@@ -33,6 +40,14 @@ func (graphPanel dashboardGraph) toOption() (row.Option, error) {
 	if graphPanel.Axes.Bottom != nil {
 		opts = append(opts, graph.XAxis(graphPanel.Axes.Bottom.toOptions()...))
 	}
+	if graphPanel.Alert != nil {
+		alertOpts, err := graphPanel.Alert.toOptions()
+		if err != nil {
+			return nil, err
+		}
+
+		opts = append(opts, graph.Alert(graphPanel.Alert.Title, alertOpts...))
+	}
 
 	for _, t := range graphPanel.Targets {
 		opt, err := graphPanel.target(t)
@@ -90,3 +105,160 @@ type graphAxes struct {
 	Right  *graphAxis
 	Bottom *graphAxis
 }
+
+type graphAlert struct {
+	Title            string
+	EvaluateEvery    string `yaml:"evaluate_every"`
+	For              string
+	If               []alertCondition
+	Notify           *int64
+	Message          string
+	OnNoData         string `yaml:"on_no_data"`
+	OnExecutionError string `yaml:"on_execution_error"`
+}
+
+func (a graphAlert) toOptions() ([]alert.Option, error) {
+	opts := []alert.Option{
+		alert.EvaluateEvery(a.EvaluateEvery),
+		alert.For(a.For),
+	}
+
+	if a.OnNoData != "" {
+		var mode alert.NoDataMode
+
+		switch a.OnNoData {
+		case "no_data":
+			mode = alert.NoData
+		case "alerting":
+			mode = alert.Error
+		case "keep_state":
+			mode = alert.KeepLastState
+		case "ok":
+			mode = alert.OK
+		default:
+			return nil, fmt.Errorf("unknown on_no_data mode '%s'", a.OnNoData)
+		}
+
+		opts = append(opts, alert.OnNoData(mode))
+	}
+	if a.OnExecutionError != "" {
+		var mode alert.ErrorMode
+
+		switch a.OnExecutionError {
+		case "alerting":
+			mode = alert.Alerting
+		case "keep_state":
+			mode = alert.LastState
+		default:
+			return nil, fmt.Errorf("unknown on_execution_error mode '%s'", a.OnExecutionError)
+		}
+
+		opts = append(opts, alert.OnExecutionError(mode))
+	}
+	if a.Notify != nil {
+		opts = append(opts, alert.NotifyChannel(*a.Notify))
+	}
+	if a.Message != "" {
+		opts = append(opts, alert.Message(a.Message))
+	}
+
+	for _, condition := range a.If {
+		conditionOpt, err := condition.toOption()
+		if err != nil {
+			return nil, err
+		}
+
+		opts = append(opts, conditionOpt)
+	}
+
+	return opts, nil
+}
+
+type alertThreshold struct {
+	HasNoValue   bool
+	Above        *float64
+	Below        *float64
+	OutsideRange [2]float64
+	WithinRange  [2]float64
+}
+
+func (threshold alertThreshold) toOption() (alert.ConditionOption, error) {
+	if threshold.HasNoValue {
+		return alert.HasNoValue(), nil
+	}
+	if threshold.Above != nil {
+		return alert.IsAbove(*threshold.Above), nil
+	}
+	if threshold.Below != nil {
+		return alert.IsBelow(*threshold.Below), nil
+	}
+	if threshold.OutsideRange[0] != 0 && threshold.OutsideRange[1] != 0 {
+		return alert.IsOutsideRange(threshold.OutsideRange[0], threshold.OutsideRange[1]), nil
+	}
+	if threshold.WithinRange[0] != 0 && threshold.WithinRange[1] != 0 {
+		return alert.IsWithinRange(threshold.WithinRange[0], threshold.WithinRange[1]), nil
+	}
+
+	return nil, ErrNoAlertThresholdDefined
+}
+
+type alertValue struct {
+	Func     string
+	QueryRef string `yaml:"ref"`
+	From     string
+	To       string
+}
+
+func (v alertValue) toOption() (alert.ConditionOption, error) {
+	var alertFunc func(refID string, from string, to string) alert.ConditionOption
+
+	switch v.Func {
+	case "avg":
+		alertFunc = alert.Avg
+	case "sum":
+		alertFunc = alert.Sum
+	case "count":
+		alertFunc = alert.Count
+	case "last":
+		alertFunc = alert.Last
+	case "min":
+		alertFunc = alert.Min
+	case "max":
+		alertFunc = alert.Max
+	case "median":
+		alertFunc = alert.Median
+	case "diff":
+		alertFunc = alert.Diff
+	case "percent_diff":
+		alertFunc = alert.PercentDiff
+	default:
+		return nil, ErrInvalidAlertValueFunc
+	}
+
+	return alertFunc(v.QueryRef, v.From, v.To), nil
+}
+
+type alertCondition struct {
+	Operand   string
+	Value     alertValue
+	Threshold alertThreshold
+}
+
+func (c alertCondition) toOption() (alert.Option, error) {
+	operand := alert.And
+	if c.Operand == "or" {
+		operand = alert.Or
+	}
+
+	threshold, err := c.Threshold.toOption()
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := c.Value.toOption()
+	if err != nil {
+		return nil, err
+	}
+
+	return alert.If(operand, value, threshold), nil
+}