From 06045779ac56c354b8dac517a2321ab9a29ecaab Mon Sep 17 00:00:00 2001 From: Doctor Vince Date: Tue, 20 Feb 2024 17:56:03 -0500 Subject: [PATCH] add a handler for the metrics endpoint (#35) * add a handler for the metrics endpoint * remove references to un-used metric --- cmd/server.go | 2 ++ go.mod | 8 +++++ go.sum | 10 +++++++ internal/metrics/metrics.go | 58 +++++++++++++++++++++++++++++++++++++ pkg/api/v1/routes/routes.go | 5 ++++ 5 files changed, 83 insertions(+) create mode 100644 internal/metrics/metrics.go diff --git a/cmd/server.go b/cmd/server.go index b987a6a..0bee2a6 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -8,6 +8,7 @@ import ( "time" "github.com/metal-toolbox/bomservice/internal/app" + "github.com/metal-toolbox/bomservice/internal/metrics" "github.com/metal-toolbox/bomservice/internal/model" "github.com/metal-toolbox/bomservice/internal/server" "github.com/metal-toolbox/bomservice/internal/store" @@ -48,6 +49,7 @@ var cmdServer = &cobra.Command{ app.Logger.Fatal(err) } }() + metrics.ListenAndServe() // sit around for term signal <-termCh diff --git a/go.mod b/go.mod index 3ee90c7..f279baa 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/golang/mock v1.6.0 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.18.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 @@ -21,7 +22,9 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.11.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cockroachdb/cockroach-go/v2 v2.3.6 // indirect @@ -65,6 +68,8 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -74,6 +79,9 @@ require ( github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/cachecontrol v0.2.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect diff --git a/go.sum b/go.sum index a62cf9b..1a2c581 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -616,18 +618,24 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -635,6 +643,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..5db2061 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,58 @@ +package metrics + +import ( + "log" + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + apiLatencySeconds *prometheus.HistogramVec +) + +func init() { + apiLatencySeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "bomservice", + Subsystem: "api", + Name: "latency_seconds", + Help: "api latency measurements in seconds", + // XXX: will need to tune these buckets once we understand common behaviors better + // buckets between 25ms to 10 s + Buckets: []float64{0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0}, + }, []string{ + "endpoint", + "response_code", + }, + ) +} + +// ListenAndServeMetrics exposes prometheus metrics as /metrics on port 9090 +func ListenAndServe() { + endpoint := "0.0.0.0:9090" + + go func() { + http.Handle("/metrics", promhttp.Handler()) + + server := &http.Server{ + Addr: endpoint, + ReadHeaderTimeout: 2 * time.Second, + } + + if err := server.ListenAndServe(); err != nil { + log.Println(err) + } + }() +} + +// APICallEpilog observes the results and latency of an API call +func APICallEpilog(start time.Time, endpoint string, responseCode int) { + code := strconv.Itoa(responseCode) + elapsed := time.Since(start).Seconds() + apiLatencySeconds.WithLabelValues(endpoint, code).Observe(elapsed) +} diff --git a/pkg/api/v1/routes/routes.go b/pkg/api/v1/routes/routes.go index 4fe3165..81dd2e7 100644 --- a/pkg/api/v1/routes/routes.go +++ b/pkg/api/v1/routes/routes.go @@ -3,8 +3,10 @@ package routes import ( "fmt" "strings" + "time" "github.com/gin-gonic/gin" + "github.com/metal-toolbox/bomservice/internal/metrics" "github.com/metal-toolbox/bomservice/internal/store" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -75,7 +77,10 @@ type apiHandler func(c *gin.Context) (int, *sservice.ServerResponse) // directly func wrapAPICall(fn apiHandler) gin.HandlerFunc { return func(ctx *gin.Context) { + endpoint := ctx.Request.URL.Path + start := time.Now() responseCode, obj := fn(ctx) + metrics.APICallEpilog(start, endpoint, responseCode) ctx.JSON(responseCode, obj) } }