Use Prometheus (#21)

* Use Prometheus

* Fix ci

* fix registration conflict

Makefile

@@ -20,11 +20,11 @@ run:
 	@go run main.go start --rpc --rpc-port=52345 -d
 
 run-containers:
-	@cd test_containers && docker-compose up -d && cd ..
+	@cd test_containers && docker compose up -d && cd ..
 	@/bin/bash -c "until curl -s localhost:8081 > /dev/null; do echo 'Waiting for EMQTT...' && sleep 1; done"
 
 kill-containers:
-	@cd test_containers && docker-compose stop && cd ..
+	@cd test_containers && docker compose stop && cd ..
 
 test: run-tests
 

api/app.go

@@ -9,6 +9,9 @@ package api
 
 import (
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -165,6 +168,7 @@ func (app *App) configureJaeger() {
 
 func (app *App) setConfigurationDefaults() {
 	app.Config.SetDefault("healthcheck.workingText", "WORKING")
+	app.Config.SetDefault("httpserver.metricsServer", 9090)
 }
 
 func (app *App) loadConfiguration() error {
@@ -184,7 +188,7 @@ func (app *App) loadConfiguration() error {
 	return nil
 }
 
-//OnErrorHandler handles panics
+// OnErrorHandler handles panics
 func (app *App) OnErrorHandler(err error, stack []byte) {
 	app.Logger.WithError(err).Error("Panic occurred.")
 
@@ -259,15 +263,31 @@ func (app *App) Start() error {
 		"operation": "Start",
 	})
 
+	// start metrics server
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
+	metricsServer := &http.Server{
+		Addr:    fmt.Sprintf(":%d", app.Config.GetInt("httpserver.metricsServer")),
+		Handler: mux,
+	}
+
+	go func() {
+		err := metricsServer.ListenAndServe()
+		if err != nil {
+			l.WithError(err).Error("Failed to start metrics server.")
+		}
+	}()
+
 	l.WithFields(log.Fields{
 		"host": app.Host,
 		"port": app.Port,
-	}).Info("App started.")
+	}).Infof("App starting on %s:%d", app.Host, app.Port)
 
 	err := app.App.Start(fmt.Sprintf("%s:%d", app.Host, app.Port))
 	if err != nil {
 		l.WithError(err).Error("App failed to start.")
 		return err
 	}
+
 	return nil
 }

api/middleware.go

@@ -10,6 +10,8 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"runtime/debug"
 	"time"
 
@@ -20,14 +22,14 @@ import (
 
 const responseTimeMillisecondsMetricName = "response_time_milliseconds"
 
-//NewVersionMiddleware with API version
+// NewVersionMiddleware with API version
 func NewVersionMiddleware() *VersionMiddleware {
 	return &VersionMiddleware{
 		Version: VERSION,
 	}
 }
 
-//VersionMiddleware inserts the current version in all requests
+// VersionMiddleware inserts the current version in all requests
 type VersionMiddleware struct {
 	Version string
 }
@@ -41,14 +43,14 @@ func (v *VersionMiddleware) Serve(next echo.HandlerFunc) echo.HandlerFunc {
 	}
 }
 
-//NewSentryMiddleware returns a new sentry middleware
+// NewSentryMiddleware returns a new sentry middleware
 func NewSentryMiddleware(app *App) *SentryMiddleware {
 	return &SentryMiddleware{
 		App: app,
 	}
 }
 
-//SentryMiddleware is responsible for sending all exceptions to sentry
+// SentryMiddleware is responsible for sending all exceptions to sentry
 type SentryMiddleware struct {
 	App *App
 }
@@ -78,18 +80,30 @@ func (s *SentryMiddleware) Serve(next echo.HandlerFunc) echo.HandlerFunc {
 
 // ResponseTimeMetricsMiddleware struct encapsulating DDStatsD
 type ResponseTimeMetricsMiddleware struct {
-	DDStatsD *DogStatsD
+	DDStatsD MetricsReporter
 }
 
-//ResponseTimeMetricsMiddleware returns a new ResponseTimeMetricsMiddleware
-func NewResponseTimeMetricsMiddleware(ddStatsD *DogStatsD) *ResponseTimeMetricsMiddleware {
+var latencyMetric *prometheus.HistogramVec
+
+func init() {
+	latencyMetric = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "arkadiko",
+		Name:      "response_time",
+		Help:      "",
+	},
+		[]string{"route", "method", "status"},
+	)
+}
+
+// ResponseTimeMetricsMiddleware returns a new ResponseTimeMetricsMiddleware
+func NewResponseTimeMetricsMiddleware(ddStatsD MetricsReporter) *ResponseTimeMetricsMiddleware {
 	return &ResponseTimeMetricsMiddleware{
 		DDStatsD: ddStatsD,
 	}
 }
 
-//ResponseTimeMetricsMiddleware is a middleware to measure the response time
-//of a route and send it do StatsD
+// ResponseTimeMetricsMiddleware is a middleware to measure the response time
+// of a route and send it do StatsD
 func (responseTimeMiddleware ResponseTimeMetricsMiddleware) Serve(next echo.HandlerFunc) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		startTime := time.Now()
@@ -106,7 +120,9 @@ func (responseTimeMiddleware ResponseTimeMetricsMiddleware) Serve(next echo.Hand
 			fmt.Sprintf("status:%d", status),
 		}
 
+		// Keeping both for retro compatibility in the short term
 		responseTimeMiddleware.DDStatsD.Timing(responseTimeMillisecondsMetricName, timeUsed, tags...)
+		latencyMetric.WithLabelValues(route, method, fmt.Sprintf("%d", status)).Observe(timeUsed.Seconds())
 
 		return result
 	}
@@ -142,19 +158,19 @@ func newHTTPFromCtx(ctx echo.Context) *raven.Http {
 	return h
 }
 
-//NewRecoveryMiddleware returns a configured middleware
+// NewRecoveryMiddleware returns a configured middleware
 func NewRecoveryMiddleware(onError func(error, []byte)) *RecoveryMiddleware {
 	return &RecoveryMiddleware{
 		OnError: onError,
 	}
 }
 
-//RecoveryMiddleware recovers from errors
+// RecoveryMiddleware recovers from errors
 type RecoveryMiddleware struct {
 	OnError func(error, []byte)
 }
 
-//Serve executes on error handler when errors happen
+// Serve executes on error handler when errors happen
 func (r *RecoveryMiddleware) Serve(next echo.HandlerFunc) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		defer func() {
@@ -179,7 +195,7 @@ func NewLoggerMiddleware(theLogger log.FieldLogger) *LoggerMiddleware {
 	return l
 }
 
-//LoggerMiddleware is responsible for logging to Zap all requests
+// LoggerMiddleware is responsible for logging to Zap all requests
 type LoggerMiddleware struct {
 	Logger log.FieldLogger
 }
@@ -258,13 +274,13 @@ func (l *LoggerMiddleware) Serve(next echo.HandlerFunc) echo.HandlerFunc {
 	}
 }
 
-//NewNewRelicMiddleware returns the logger middleware
+// NewNewRelicMiddleware returns the logger middleware
 func NewNewRelicMiddleware(app *App, theLogger log.FieldLogger) *NewRelicMiddleware {
 	l := &NewRelicMiddleware{App: app, Logger: theLogger}
 	return l
 }
 
-//NewRelicMiddleware is responsible for logging to Zap all requests
+// NewRelicMiddleware is responsible for logging to Zap all requests
 type NewRelicMiddleware struct {
 	App    *App
 	Logger log.FieldLogger

api/sendmqtt.go

@@ -10,6 +10,8 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"io/ioutil"
 	"net/http"
 	"time"
@@ -18,6 +20,18 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+var sendMqttLatencyMetric *prometheus.HistogramVec
+
+func init() {
+	sendMqttLatencyMetric = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "arkadiko",
+		Name:      "send_mqtt_latency",
+		Help:      "The latency of sending a message to mqtt",
+	},
+		[]string{"error", "retained"},
+	)
+}
+
 // SendMqttHandler is the handler responsible for sending messages to mqtt
 func SendMqttHandler(app *App) func(c echo.Context) error {
 	return func(c echo.Context) error {
@@ -94,6 +108,7 @@ func SendMqttHandler(app *App) func(c echo.Context) error {
 		}
 
 		app.DDStatsD.Timing("mqtt_latency", mqttLatency, tags...)
+		sendMqttLatencyMetric.WithLabelValues(fmt.Sprintf("%t", err != nil), fmt.Sprintf("%t", retained)).Observe(mqttLatency.Seconds())
 		lg = lg.WithField("mqttLatency", mqttLatency.Nanoseconds())
 		lg.Debug("sent mqtt message")
 		c.Set("mqttLatency", mqttLatency)

config/local.yml

@@ -23,3 +23,4 @@ httpserver:
   url: "http://localhost:8081"
   user: admin
   pass: public
+  metricsPort: 9090

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.4
 	github.com/opentracing/opentracing-go v1.2.0
+	github.com/prometheus/client_golang v0.9.3
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.7.0

go.sum

@@ -29,6 +29,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
@@ -97,7 +98,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -151,10 +151,8 @@ github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqy
 github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -179,6 +177,7 @@ github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRU
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/goveralls v0.0.7 h1:vzy0i4a2iDzEFMdXIxcanRadkr0FBvSBKUmj0P8SPlQ=
 github.com/mattn/goveralls v0.0.7/go.mod h1:h8b4ow6FxSPMQHF6o2ve3qsclnffZjYTNEKmLesRwqw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -220,13 +219,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
@@ -268,7 +271,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -417,7 +419,6 @@ golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 h1:SjQ2+AKWgZLc1xej6WSzL+D
 golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -461,7 +462,6 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=