diff --git a/lib/metrics.go b/lib/metrics.go index 36b0addc..d507c18d 100644 --- a/lib/metrics.go +++ b/lib/metrics.go @@ -15,11 +15,26 @@ var ( []string{"host"}, ) - routeReloadCountMetric = prometheus.NewCounter( + routeReloadCountMetric = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "router_route_reload_total", Help: "Total number of attempts to reload the routing table", }, + []string{}, + ) + + routeReloadDurationMetric = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "router_route_reload_duration_seconds", + Help: "Histogram of route reload durations in seconds", + Objectives: map[float64]float64{ + 0.5: 0.01, + 0.9: 0.01, + 0.95: 0.01, + 0.99: 0.005, + }, + }, + []string{}, ) routeReloadErrorCountMetric = prometheus.NewCounter( @@ -41,6 +56,7 @@ func registerMetrics(r prometheus.Registerer) { r.MustRegister( internalServerErrorCountMetric, routeReloadCountMetric, + routeReloadDurationMetric, routeReloadErrorCountMetric, routesCountMetric, ) diff --git a/lib/router.go b/lib/router.go index 03e56545..4f2f15f1 100644 --- a/lib/router.go +++ b/lib/router.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "os" + "strconv" "sync" "time" @@ -213,11 +214,11 @@ type mongoDatabase interface { // create a new proxy mux, load applications (backends) and routes into it, and // then flip the "mux" pointer in the Router. func (rt *Router) reloadRoutes(db *mgo.Database, currentOptime bson.MongoTimestamp) { + startTime := time.Now() defer func() { - // increment this metric regardless of whether the route reload succeeded - routeReloadCountMetric.Inc() - + success := true if r := recover(); r != nil { + success = false logWarn("router: recovered from panic in reloadRoutes:", r) logInfo("router: original routes have not been modified") errorMessage := fmt.Sprintf("panic: %v", r) @@ -228,6 +229,9 @@ func (rt *Router) reloadRoutes(db *mgo.Database, currentOptime bson.MongoTimesta } else { rt.mongoReadToOptime = currentOptime } + labels := prometheus.Labels{"success": strconv.FormatBool(success)} + routeReloadCountMetric.With(labels).Inc() + routeReloadDurationMetric.With(labels).Observe(time.Since(startTime).Seconds()) }() logInfo("router: reloading routes")