From fc94ed92fdef2e8dfedb00bf286c95f45208884e Mon Sep 17 00:00:00 2001 From: Nicolai Antiferov Date: Wed, 11 Sep 2024 14:24:07 +0300 Subject: [PATCH] Switch to universal data collection for modules --- Makefile | 2 +- README.md | 2 +- docker-compose.yml | 2 +- exporter/exporter.go | 9 +++--- exporter/http_test.go | 2 +- exporter/{search.go => modules.go} | 17 ++++------- exporter/{search_test.go => modules_test.go} | 30 ++++++++++---------- main.go | 4 +-- 8 files changed, 31 insertions(+), 37 deletions(-) rename exporter/{search.go => modules.go} (53%) rename exporter/{search_test.go => modules_test.go} (53%) diff --git a/Makefile b/Makefile index 74c9322a..aa4f3184 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ test: TEST_REDIS_CLUSTER_PASSWORD_URI="redis://localhost:17006" \ TEST_TILE38_URI="redis://localhost:19851" \ TEST_REDIS_SENTINEL_URI="redis://localhost:26379" \ - TEST_REDIS_SEARCH_URI="redis://localhost:36379" \ + TEST_REDIS_MODULES_URI="redis://localhost:36379" \ go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./... .PHONY: lint diff --git a/README.md b/README.md index 70c3d9aa..2cc1cdd2 100644 --- a/README.md +++ b/README.md @@ -161,12 +161,12 @@ Prometheus uses file watches and all changes to the json file are applied immedi | redis-only-metrics | REDIS_EXPORTER_REDIS_ONLY_METRICS | Whether to also export go runtime metrics, defaults to false. | | include-config-metrics | REDIS_EXPORTER_INCL_CONFIG_METRICS | Whether to include all config settings as metrics, defaults to false. | | include-system-metrics | REDIS_EXPORTER_INCL_SYSTEM_METRICS | Whether to include system metrics like `total_system_memory_bytes`, defaults to false. | +| include-modules-metrics | REDIS_EXPORTER_INCL_MODULES_METRICS | Whether to collect Redis Modules metrics, defaults to false. | | exclude-latency-histogram-metrics | REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS | Do not try to collect latency histogram metrics (to avoid `WARNING, LOGGED ONCE ONLY: cmd LATENCY HISTOGRAM` error on Redis < v7). | | redact-config-metrics | REDIS_EXPORTER_REDACT_CONFIG_METRICS | Whether to redact config settings that include potentially sensitive information like passwords. | | ping-on-connect | REDIS_EXPORTER_PING_ON_CONNECT | Whether to ping the redis instance after connecting and record the duration as a metric, defaults to false. | | is-tile38 | REDIS_EXPORTER_IS_TILE38 | Whether to scrape Tile38 specific metrics, defaults to false. | | is-cluster | REDIS_EXPORTER_IS_CLUSTER | Whether this is a redis cluster (Enable this if you need to fetch key level data on a Redis Cluster). | -| is-search | REDIS_EXPORTER_IS_SEARCH | Whether this is a redis has RediSearch module enabled and metrics should be collected. | | export-client-list | REDIS_EXPORTER_EXPORT_CLIENT_LIST | Whether to scrape Client List specific metrics, defaults to false. | | export-client-port | REDIS_EXPORTER_EXPORT_CLIENT_PORT | Whether to include the client's port when exporting the client list. Warning: including the port increases the number of metrics generated and will make your Prometheus server take up more memory | | skip-tls-verification | REDIS_EXPORTER_SKIP_TLS_VERIFICATION | Whether to to skip TLS verification when the exporter connects to a Redis instance | diff --git a/docker-compose.yml b/docker-compose.yml index a1e803bb..25da5938 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,7 @@ services: ports: - "19851:9851" - redis-search: + redis-stack: image: redis/redis-stack-server:7.4.0-v0 ports: - "36379:6379" diff --git a/exporter/exporter.go b/exporter/exporter.go index 0918ce05..4a2edcb6 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -65,7 +65,7 @@ type Options struct { ClientKeyFile string CaCertFile string InclConfigMetrics bool - IsSearch bool + InclModulesMetrics bool DisableExportingKeyValues bool ExcludeLatencyHistogramMetrics bool RedactConfigMetrics bool @@ -269,7 +269,8 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { "long_lock_waits": "long_lock_waits_total", "current_client_thread": "current_client_thread", - // RediSearch module metrics + // Redis Modules metrics + // RediSearch module "search_number_of_indexes": "search_number_of_indexes", "search_used_memory_indexes": "search_used_memory_indexes", "search_total_indexing_time": "search_total_indexing_time", @@ -706,8 +707,8 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { e.extractTile38Metrics(ch, c) } - if e.options.IsSearch { - e.extractSearchMetrics(ch, c) + if e.options.InclModulesMetrics { + e.extractModuleMetrics(ch, c) } if len(e.options.LuaScript) > 0 { diff --git a/exporter/http_test.go b/exporter/http_test.go index ef8565fc..203ab143 100644 --- a/exporter/http_test.go +++ b/exporter/http_test.go @@ -210,7 +210,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS5_URI"), os.Getenv("TEST_REDIS6_URI"), - os.Getenv("TEST_REDIS_SEARCH_URI"), + os.Getenv("TEST_REDIS_MODULES_URI"), // tile38 & Cluster need to be last in this list so we can identify them when selected, down in line 229 os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"), diff --git a/exporter/search.go b/exporter/modules.go similarity index 53% rename from exporter/search.go rename to exporter/modules.go index dcb69d87..965cd20a 100644 --- a/exporter/search.go +++ b/exporter/modules.go @@ -8,21 +8,14 @@ import ( log "github.com/sirupsen/logrus" ) -func (e *Exporter) extractSearchMetrics(ch chan<- prometheus.Metric, c redis.Conn) { - for _, section := range [5]string{"search_version", "search_index", "search_memory", "search_cursors", "search_gc"} { - info, err := redis.String(doRedisCmd(c, "INFO", section)) - if err != nil { - log.Errorf("extractSearchMetrics() err: %s", err) - return - } - e.registerSearchMetrics(ch, info) +func (e *Exporter) extractModuleMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + info, err := redis.String(doRedisCmd(c, "INFO", "MODULES")) + if err != nil { + log.Errorf("extractSearchMetrics() err: %s", err) + return } -} - -func (e *Exporter) registerSearchMetrics(ch chan<- prometheus.Metric, info string) { lines := strings.Split(info, "\r\n") - for _, line := range lines { log.Debugf("info: %s", line) diff --git a/exporter/search_test.go b/exporter/modules_test.go similarity index 53% rename from exporter/search_test.go rename to exporter/modules_test.go index 69bc0bde..19f67111 100644 --- a/exporter/search_test.go +++ b/exporter/modules_test.go @@ -8,24 +8,24 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func TestSearch(t *testing.T) { - if os.Getenv("TEST_REDIS_SEARCH_URI") == "" { - t.Skipf("TEST_REDIS_SEARCH_URI not set - skipping") +func TestModules(t *testing.T) { + if os.Getenv("TEST_REDIS_MODULES_URI") == "" { + t.Skipf("TEST_REDIS_MODULES_URI not set - skipping") } tsts := []struct { - addr string - isSearch bool - wantSearchMetrics bool + addr string + inclModulesMetrics bool + wantModulesMetrics bool }{ - {addr: os.Getenv("TEST_REDIS_SEARCH_URI"), isSearch: true, wantSearchMetrics: true}, - {addr: os.Getenv("TEST_REDIS_SEARCH_URI"), isSearch: false, wantSearchMetrics: false}, - {addr: os.Getenv("TEST_REDIS_URI"), isSearch: true, wantSearchMetrics: false}, - {addr: os.Getenv("TEST_REDIS_URI"), isSearch: false, wantSearchMetrics: false}, + {addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: true, wantModulesMetrics: true}, + {addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, + {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: true, wantModulesMetrics: false}, + {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, } for _, tst := range tsts { - e, _ := NewRedisExporter(tst.addr, Options{Namespace: "test", IsSearch: tst.isSearch}) + e, _ := NewRedisExporter(tst.addr, Options{Namespace: "test", InclModulesMetrics: tst.inclModulesMetrics}) chM := make(chan prometheus.Metric) go func() { @@ -52,16 +52,16 @@ func TestSearch(t *testing.T) { } } - if tst.wantSearchMetrics { + if tst.wantModulesMetrics { for want, found := range wantedMetrics { if !found { - t.Errorf("%s was *not* found in RediSearch metrics but expected", want) + t.Errorf("%s was *not* found in Redis Modules metrics but expected", want) } } - } else if !tst.wantSearchMetrics { + } else if !tst.wantModulesMetrics { for want, found := range wantedMetrics { if found { - t.Errorf("%s was *found* in RediSearch metrics but *not* expected", want) + t.Errorf("%s was *found* in Redis Modules metrics but *not* expected", want) } } } diff --git a/main.go b/main.go index fa996277..fdb1bc07 100644 --- a/main.go +++ b/main.go @@ -88,13 +88,13 @@ func main() { setClientName = flag.Bool("set-client-name", getEnvBool("REDIS_EXPORTER_SET_CLIENT_NAME", true), "Whether to set client name to redis_exporter") isTile38 = flag.Bool("is-tile38", getEnvBool("REDIS_EXPORTER_IS_TILE38", false), "Whether to scrape Tile38 specific metrics") isCluster = flag.Bool("is-cluster", getEnvBool("REDIS_EXPORTER_IS_CLUSTER", false), "Whether this is a redis cluster (Enable this if you need to fetch key level data on a Redis Cluster).") - isSearch = flag.Bool("is-search", getEnvBool("REDIS_EXPORTER_IS_SEARCH", false), "Whether to scrape RediSearch module specific metrics") exportClientList = flag.Bool("export-client-list", getEnvBool("REDIS_EXPORTER_EXPORT_CLIENT_LIST", false), "Whether to scrape Client List specific metrics") exportClientPort = flag.Bool("export-client-port", getEnvBool("REDIS_EXPORTER_EXPORT_CLIENT_PORT", false), "Whether to include the client's port when exporting the client list. Warning: including the port increases the number of metrics generated and will make your Prometheus server take up more memory") showVersion = flag.Bool("version", false, "Show version information and exit") redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS", false), "Whether to also export go runtime metrics") pingOnConnect = flag.Bool("ping-on-connect", getEnvBool("REDIS_EXPORTER_PING_ON_CONNECT", false), "Whether to ping the redis instance after connecting") inclConfigMetrics = flag.Bool("include-config-metrics", getEnvBool("REDIS_EXPORTER_INCL_CONFIG_METRICS", false), "Whether to include all config settings as metrics") + inclModulesMetrics = flag.Bool("include-modules-metrics", getEnvBool("REDIS_EXPORTER_INCL_MODULES_METRICS", false), "Whether to collect Redis Modules metrics") disableExportingKeyValues = flag.Bool("disable-exporting-key-values", getEnvBool("REDIS_EXPORTER_DISABLE_EXPORTING_KEY_VALUES", false), "Whether to disable values of keys stored in redis as labels or not when using check-keys/check-single-key") excludeLatencyHistogramMetrics = flag.Bool("exclude-latency-histogram-metrics", getEnvBool("REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS", false), "Do not try to collect latency histogram metrics") redactConfigMetrics = flag.Bool("redact-config-metrics", getEnvBool("REDIS_EXPORTER_REDACT_CONFIG_METRICS", true), "Whether to redact config settings that include potentially sensitive information like passwords") @@ -183,7 +183,7 @@ func main() { SetClientName: *setClientName, IsTile38: *isTile38, IsCluster: *isCluster, - IsSearch: *isSearch, + InclModulesMetrics: *inclModulesMetrics, ExportClientList: *exportClientList, ExportClientsInclPort: *exportClientPort, SkipTLSVerification: *skipTLSVerification,