feat: display muted alerts on Grafana (#32)

* chore: moved filter string generation to function to reuse

* feat: refactor + display muted alerts on Grafana

* chore: fixed Grafana template

* chore: fixed Grafana template, v2

* chore: fixed Grafana alerts querying

pkg/alertmanager/alertmanager.go

@@ -7,10 +7,7 @@ import (
 	"io"
 	"main/pkg/config"
 	"main/pkg/types"
-	"main/pkg/utils/generic"
 	"net/http"
-	"net/url"
-	"strings"
 
 	"github.com/rs/zerolog"
 )
@@ -39,15 +36,9 @@ func (g *Alertmanager) CreateSilence(silence types.Silence) (types.SilenceCreate
 }
 
 func (g *Alertmanager) GetSilenceMatchingAlerts(silence types.Silence) ([]types.AlertmanagerAlert, error) {
-	filtersParts := generic.Map(silence.Matchers, func(m types.SilenceMatcher) string {
-		return fmt.Sprintf("filter=%s", url.QueryEscape(m.SerializeQueryString()))
-	})
-
-	filtersString := strings.Join(filtersParts, "&")
-
 	relativeUrl := fmt.Sprintf(
 		"/api/v2/alerts?%s&silenced=true&inhibited=true&active=true",
-		filtersString,
+		silence.GetFilterQueryString(),
 	)
 	url := g.RelativeLink(relativeUrl)
 	var res []types.AlertmanagerAlert

pkg/app/silences_create.go

@@ -30,13 +30,18 @@ func (a *App) HandleNewSilence(c tele.Context) error {
 		return c.Reply(fmt.Sprintf("Error getting created silence: %s", silenceErr))
 	}
 
+	alerts, alertsErr := a.Grafana.GetSilenceMatchingAlerts(silence)
+	if alertsErr != nil {
+		return c.Reply(fmt.Sprintf("Error getting alerts for silence: %s", alertsErr))
+	}
+
 	template, renderErr := a.TemplateManager.Render("silences_create", render.RenderStruct{
 		Grafana:      a.Grafana,
 		Alertmanager: a.Alertmanager,
-		Data: render.SilenceRender{
+		Data: types.SilenceWithAlerts{
 			Silence:       silence,
-			AlertsPresent: false,
-			Alerts:        []types.AlertmanagerAlert{},
+			AlertsPresent: true,
+			Alerts:        alerts,
 		},
 	})
 	if renderErr != nil {
@@ -80,7 +85,7 @@ func (a *App) HandleAlertmanagerNewSilence(c tele.Context) error {
 	template, renderErr := a.TemplateManager.Render("silences_create", render.RenderStruct{
 		Grafana:      a.Grafana,
 		Alertmanager: a.Alertmanager,
-		Data: render.SilenceRender{
+		Data: types.SilenceWithAlerts{
 			Silence:       silence,
 			AlertsPresent: alerts != nil,
 			Alerts:        alerts,

pkg/app/silences_list.go

@@ -4,9 +4,6 @@ import (
 	"fmt"
 	"main/pkg/types"
 	"main/pkg/types/render"
-	"main/pkg/utils/generic"
-	"sync"
-	"time"
 
 	tele "gopkg.in/telebot.v3"
 )
@@ -17,28 +14,15 @@ func (a *App) HandleListSilences(c tele.Context) error {
 		Str("text", c.Text()).
 		Msg("Got list silence query")
 
-	silences, err := a.Grafana.GetSilences()
+	silencesWithAlerts, err := types.GetSilencesWithAlerts(a.Grafana)
 	if err != nil {
-		return c.Reply(fmt.Sprintf("Error listing silence: %s", err))
-	}
-
-	silences = generic.Filter(silences, func(s types.Silence) bool {
-		return s.EndsAt.After(time.Now())
-	})
-
-	silencesRender := make([]render.SilenceRender, len(silences))
-	for index, silence := range silences {
-		silencesRender[index] = render.SilenceRender{
-			Silence:       silence,
-			AlertsPresent: false,
-			Alerts:        make([]types.AlertmanagerAlert, 0),
-		}
+		return c.Reply(err)
 	}
 
 	template, err := a.TemplateManager.Render("silences_list", render.RenderStruct{
 		Grafana:      a.Grafana,
 		Alertmanager: a.Alertmanager,
-		Data:         silencesRender,
+		Data:         silencesWithAlerts,
 	})
 	if err != nil {
 		a.Logger.Error().Err(err).Msg("Error rendering silences_list template")
@@ -58,54 +42,15 @@ func (a *App) HandleAlertmanagerListSilences(c tele.Context) error {
 		return c.Reply("Alertmanager is disabled.")
 	}
 
-	silences, err := a.Alertmanager.GetSilences()
+	silencesWithAlerts, err := types.GetSilencesWithAlerts(a.Alertmanager)
 	if err != nil {
-		return c.Reply(fmt.Sprintf("Error listing silence: %s", err))
-	}
-
-	silences = generic.Filter(silences, func(s types.Silence) bool {
-		return s.EndsAt.After(time.Now())
-	})
-
-	silencesRender := make([]render.SilenceRender, len(silences))
-
-	var wg sync.WaitGroup
-	var mutex sync.Mutex
-	var errs []error
-
-	for index, silence := range silences {
-		wg.Add(1)
-		go func(index int, silence types.Silence) {
-			defer wg.Done()
-
-			alerts, alertsErr := a.Alertmanager.GetSilenceMatchingAlerts(silence)
-			if alertsErr != nil {
-				mutex.Lock()
-				errs = append(errs, alertsErr)
-				mutex.Unlock()
-				return
-			}
-
-			mutex.Lock()
-			silencesRender[index] = render.SilenceRender{
-				Silence:       silence,
-				AlertsPresent: true,
-				Alerts:        alerts,
-			}
-			mutex.Unlock()
-		}(index, silence)
-	}
-
-	wg.Wait()
-
-	if len(errs) > 0 {
-		return c.Reply(fmt.Sprintf("Error getting alerts for silence on %d silences", len(errs)))
+		return c.Reply(err)
 	}
 
 	template, err := a.TemplateManager.Render("silences_list", render.RenderStruct{
 		Grafana:      a.Grafana,
 		Alertmanager: a.Alertmanager,
-		Data:         silencesRender,
+		Data:         silencesWithAlerts,
 	})
 	if err != nil {
 		a.Logger.Error().Err(err).Msg("Error rendering silences_list template")

pkg/grafana/grafana.go

@@ -224,6 +224,17 @@ func (g *Grafana) DeleteSilence(silenceID string) error {
 	return g.QueryDelete(url)
 }
 
+func (g *Grafana) GetSilenceMatchingAlerts(silence types.Silence) ([]types.AlertmanagerAlert, error) {
+	relativeUrl := fmt.Sprintf(
+		"/api/alertmanager/grafana/api/v2/alerts?%s&silenced=true&inhibited=true&active=true",
+		silence.GetFilterQueryString(),
+	)
+	url := g.RelativeLink(relativeUrl)
+	var res []types.AlertmanagerAlert
+	err := g.QueryAndDecode(url, &res)
+	return res, err
+}
+
 /* Query functions */
 
 func (g *Grafana) Query(url string) (io.ReadCloser, error) {

pkg/types/render/render.go

@@ -3,17 +3,10 @@ package render
 import (
 	"main/pkg/alertmanager"
 	"main/pkg/grafana"
-	"main/pkg/types"
 )
 
 type RenderStruct struct {
 	Grafana      *grafana.Grafana
 	Alertmanager *alertmanager.Alertmanager
 	Data         interface{}
 }
-
-type SilenceRender struct {
-	Silence       types.Silence
-	AlertsPresent bool
-	Alerts        []types.AlertmanagerAlert
-}

pkg/types/silence.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"main/pkg/constants"
 	"main/pkg/utils/generic"
+	"net/url"
+	"strings"
 	"time"
 )
 
@@ -45,6 +47,14 @@ type Silence struct {
 	Status    SilenceStatus   `json:"status,omitempty"`
 }
 
+func (s Silence) GetFilterQueryString() string {
+	filtersParts := generic.Map(s.Matchers, func(m SilenceMatcher) string {
+		return fmt.Sprintf("filter=%s", url.QueryEscape(m.SerializeQueryString()))
+	})
+
+	return strings.Join(filtersParts, "&")
+}
+
 type SilenceMatchers []SilenceMatcher
 
 type SilenceMatcher struct {
@@ -102,3 +112,9 @@ func (matchers SilenceMatchers) Equals(otherMatchers SilenceMatchers) bool {
 
 	return true
 }
+
+type SilenceWithAlerts struct {
+	Silence       Silence
+	AlertsPresent bool
+	Alerts        []AlertmanagerAlert
+}

pkg/types/silence_manager.go

@@ -0,0 +1,61 @@
+package types
+
+import (
+	"fmt"
+	"main/pkg/utils/generic"
+	"sync"
+	"time"
+)
+
+type SilenceManager interface {
+	GetSilences() (Silences, error)
+	GetSilenceMatchingAlerts(silence Silence) ([]AlertmanagerAlert, error)
+}
+
+func GetSilencesWithAlerts(manager SilenceManager) ([]SilenceWithAlerts, error) {
+	silences, err := manager.GetSilences()
+	if err != nil {
+		return []SilenceWithAlerts{}, err
+	}
+
+	silences = generic.Filter(silences, func(s Silence) bool {
+		return s.EndsAt.After(time.Now())
+	})
+
+	silencesWithAlerts := make([]SilenceWithAlerts, len(silences))
+
+	var wg sync.WaitGroup
+	var mutex sync.Mutex
+	var errs []error
+
+	for index, silence := range silences {
+		wg.Add(1)
+		go func(index int, silence Silence) {
+			defer wg.Done()
+
+			alerts, alertsErr := manager.GetSilenceMatchingAlerts(silence)
+			if alertsErr != nil {
+				mutex.Lock()
+				errs = append(errs, alertsErr)
+				mutex.Unlock()
+				return
+			}
+
+			mutex.Lock()
+			silencesWithAlerts[index] = SilenceWithAlerts{
+				Silence:       silence,
+				AlertsPresent: true,
+				Alerts:        alerts,
+			}
+			mutex.Unlock()
+		}(index, silence)
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		return []SilenceWithAlerts{}, fmt.Errorf("Error getting alerts for silence on %d silences!", len(errs))
+	}
+
+	return silencesWithAlerts, nil
+}

templates/alerts_firing.html

@@ -8,19 +8,21 @@
 {{- range $groupId, $group := .Data.GrafanaGroups }}
 {{- range $ruleId, $rule := $group.Rules }}
 - {{ GetEmojiByStatus $rule.State }} {{ $group.Name }} -> {{ $rule.Name }} ({{ len ($rule.Alerts) }}):
-{{- range $alertId, $alert := $rule.Alerts }}
+{{ range $alertId, $alert := $rule.Alerts }}
+<strong>Firing for:</strong> {{ FormatDuration $alert.ActiveSince }} (since {{ FormatDate $alert.ActiveAt }})
+{{- if $alert.Value }}
+<strong>Value: </strong>{{ StrToFloat64 $alert.Value }}
+{{- end }}
+<strong>Labels: </strong>
 {{- range $key, $label := $alert.Labels }}
 {{- if ne $key "alertname" }}
-{{ $key }} = {{ $label }}
-{{- end }}
+  {{ $key }} = {{ $label }}
 {{- end }}
-{{- if $alert.Value }}
-value = {{ StrToFloat64 $alert.Value }}
 {{- end }}
-firing for {{ FormatDuration $alert.ActiveSince }} (since {{ FormatDate $alert.ActiveAt }})
+{{ end }}
 {{- end }}
 {{- end }}
-{{ end }}
+
 
 
 {{- if .Data.PrometheusGroups }}