Skip to content

Commit

Permalink
Added slack alerts
Browse files Browse the repository at this point in the history
- Added pull image to tests
- Changed EmailSettings to Email
  • Loading branch information
jeffwillette committed Sep 19, 2017
1 parent 3a9f5c1 commit aa0513c
Show file tree
Hide file tree
Showing 8 changed files with 306 additions and 180 deletions.
23 changes: 22 additions & 1 deletion cmd/alertd_container_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
var cli *client.Client

var (
stress = "deltaskelta/alpine-stress"
stress = "deltaskelta/alpine-stress:latest"
)

func TestMain(m *testing.M) {
Expand All @@ -25,6 +25,27 @@ func TestMain(m *testing.M) {
log.Println(err)
}

images, err := cli.ImageList(context.TODO(), types.ImageListOptions{})
if err != nil {
log.Println(err)
}

hasImg := false
for _, v := range images {
for _, s := range v.RepoTags {
if s == stress {
hasImg = true
}
}
}

if !hasImg {
_, err := cli.ImagePull(context.TODO(), stress, types.ImagePullOptions{})
if err != nil {
log.Println(err)
}
}

code := m.Run()

_ = cli.ContainerRemove(context.TODO(), "test", types.ContainerRemoveOptions{
Expand Down
138 changes: 138 additions & 0 deletions cmd/alerters.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package cmd

import (
"bytes"
"fmt"
"log"
"net/http"
"net/smtp"
"reflect"
"strings"

"github.com/pkg/errors"
)

// Alerter is the interface which will handle alerting via different methods such as email
// and twitter/slack
type Alerter interface {
Valid() error
Alert(a *Alert) error
}

// Email implements the Alerter interface and sends emails
type Email struct {
SMTP string
Password string
Port string
From string
To []string
Subject string
}

// Alert sends an email alert
func (e Email) Alert(a *Alert) error {
// alerts in string form
alerts := a.Dump()

// The email message formatted properly
formattedMsg := []byte(fmt.Sprintf("To: %s\r\nSubject: %s\r\n\r\n%s\r\n",
e.To, e.Subject, alerts))

// Set up authentication/address information
auth := smtp.PlainAuth("", e.From, e.Password, e.SMTP)
addr := fmt.Sprintf("%s:%s", e.SMTP, e.Port)

err := smtp.SendMail(addr, auth, e.From, e.To, formattedMsg)
if err != nil {
return errors.Wrap(err, "error sending email")
}

log.Println("alert email sent")

return nil
}

// Valid returns true if the email settings are complete
func (e Email) Valid() error {
errString := []string{}

if reflect.DeepEqual(Email{}, e) {
return nil // assume that email alerts were omitted
}

if e.SMTP == "" {
errString = append(errString, ErrEmailNoSMTP.Error())
}

if len(e.To) < 1 {
errString = append(errString, ErrEmailNoTo.Error())
}

if e.From == "" {
errString = append(errString, ErrEmailNoFrom.Error())
}

if e.Password == "" {
errString = append(errString, ErrEmailNoPass.Error())
}

if e.Port == "" {
errString = append(errString, ErrEmailNoPort.Error())
}

if e.Subject == "" {
errString = append(errString, ErrEmailNoSubject.Error())
}

if len(errString) == 0 {
return nil
}

delimErr := strings.Join(errString, ", ")
err := errors.New(delimErr)

return errors.Wrap(err, "email settings validation fail")
}

// Slack contains all the info needed to connect to a slack channel
type Slack struct {
WebhookURL string
}

// Valid returns an error if slack settings are invalid
func (s Slack) Valid() error {
errString := []string{}

if reflect.DeepEqual(Slack{}, s) {
return nil // assume that slack was omitted
}

if s.WebhookURL == "" {
errString = append(errString, ErrSlackNoWebHookURL.Error())
}

if len(errString) == 0 {
return nil
}

delimErr := strings.Join(errString, ", ")
err := errors.New(delimErr)

return errors.Wrap(err, "slack settings validation fail")
}

// Alert sends the alert to a slack channel
func (s Slack) Alert(a *Alert) error {
alerts := a.Dump()

json := fmt.Sprintf("{\"text\": \"%s\"}", alerts)
body := bytes.NewReader([]byte(json))
resp, err := http.Post(s.WebhookURL, "application/json", body)
if err != nil {
return err
}
defer resp.Body.Close()

log.Println("sent alert to slack")
return nil
}
47 changes: 47 additions & 0 deletions cmd/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package cmd

import (
"strings"

"github.com/pkg/errors"
)

// these errors are for the purpose of being able to compare them later
var (
ErrEmptyConfig = errors.New("The configuration is completely empty (check config file)")
ErrEmailNoSMTP = errors.New("no email SMTP server")
ErrEmailNoTo = errors.New("no email to addresses")
ErrEmailNoFrom = errors.New("no email from addresses")
ErrEmailNoPass = errors.New("no email password")
ErrEmailNoPort = errors.New("no email port")
ErrEmailNoSubject = errors.New("no email subject")
ErrSlackNoWebHookURL = errors.New("no slack webhook url")
ErrNoContainers = errors.New("There were no containers found in the configuration file")
ErrExistCheckFail = errors.New("Existence check failure")
ErrExistCheckRecovered = errors.New("Existence check recovered")
ErrRunningCheckFail = errors.New("Running check failure")
ErrRunningCheckRecovered = errors.New("Running check recovered")
ErrCPUCheckFail = errors.New("CPU check failure")
ErrCPUCheckRecovered = errors.New("CPU check recovered")
ErrMemCheckFail = errors.New("Memory check failure")
ErrMemCheckRecovered = errors.New("Memory check recovered")
ErrMinPIDCheckFail = errors.New("Min PID check Failure")
ErrMinPIDCheckRecovered = errors.New("Min PID check recovered")
ErrMaxPIDCheckFail = errors.New("Max PID check Failure")
ErrMaxPIDCheckRecovered = errors.New("Max PID check recovered")
ErrUnknown = errors.New("Received an unknown error")
)

// ErrContainsErr returns true if the error string contains the message
func ErrContainsErr(e, b error) bool {
switch {
case e == nil && b == nil:
return true // they are both nil and essentially equal
case e == nil || b == nil:
return false // one of them is nil (previous case took care of both nils)
case strings.Contains(e.Error(), b.Error()):
return true // b is within a
default:
return false
}
}
14 changes: 11 additions & 3 deletions cmd/initconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,22 @@ containers:
maxMem: 20
minProcs: 4
# If email settings are present and active, then email alerts will be sent when an alert
# is triggered.
emailSettings:
## ALERTERS...
## If any of the below alerters are present, alerts will be sent through the proper
## channels. Completely delete the relevant section to disable them. To Test if an alerter
## authenticates properly, run the "testalert" command
email:
smtp: smtp.nonexistantserver.com
password: s00p3rS33cret
port: 587
from: [email protected]
subject: "DOCKER_ALERTD"
to:
- [email protected]
# You need to start a slack channel and activate an app to get a webhookURL for your channel
# see https://api.slack.com/apps for more information
slack:
webhookURL: https://some.url/provided/by/slack/
`)
86 changes: 86 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"log"
"os"
"reflect"
"strings"

"github.com/pkg/errors"

Expand Down Expand Up @@ -112,3 +114,87 @@ func initConfig() {
log.Println(errors.Wrap(err, "unmarshal config file"))
}
}

// Container gets data from the Unmarshaling of the configuration file JSON and stores
// the data throughout the course of the monitor.
type Container struct {
Name string
MaxCPU uint64
MaxMem uint64
MinProcs uint64
ExpectedRunning bool
}

// Conf struct that combines containers and email settings structs
type Conf struct {
Containers []Container
Email Email
Slack Slack
Iterations int64
Duration int64
Alerters []Alerter
}

// ValidateEmailSettings calls valid on the Email settings and adds them to the alerters
// if everything is ok
func (c *Conf) ValidateEmailSettings() error {
err := c.Email.Valid()
switch {
case reflect.DeepEqual(Email{}, c.Email):
return nil
case err != nil:
return err
default:
c.Alerters = append(c.Alerters, c.Email)
log.Println("email alerts active")
return nil
}
}

// ValidateSlackSettings validates slack settings and adds it to the alerters
func (c *Conf) ValidateSlackSettings() error {
err := c.Slack.Valid()
switch {
case reflect.DeepEqual(Slack{}, c.Slack):
return nil // assume that slack was omitted and not wanted
case err != nil:
return err
default:
c.Alerters = append(c.Alerters, c.Slack)
log.Println("slack alerts active")
return nil
}
}

// Validate validates the configuration that was passed in
func (c *Conf) Validate() error {
// the error to wrap and return at the end
errString := []string{}

if reflect.DeepEqual(&Conf{}, c) {
errString = append(errString, ErrEmptyConfig.Error())
}

if len(c.Containers) < 1 {
errString = append(errString, ErrNoContainers.Error())
}

if err := c.ValidateEmailSettings(); err != nil {
errString = append(errString, err.Error())
}

if err := c.ValidateSlackSettings(); err != nil {
errString = append(errString, err.Error())
}

// if the length of the string of errors is 0 then everything has completed
// successfully and everything is valid.
if len(errString) == 0 {
return nil
}

delimErr := strings.Join(errString, ", ")
err := errors.New(delimErr)

return errors.Wrap(err, "config validation fail")
}
2 changes: 0 additions & 2 deletions cmd/testalert.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ var testalertCmd = &cobra.Command{
},
}

Config.EmailSettings.Subject = "TEST: DOCKER-ALERTD"

err := Config.Validate()
if err != nil {
log.Println(err)
Expand Down
Loading

0 comments on commit aa0513c

Please sign in to comment.