Skip to content

Commit

Permalink
Switch to universal data collection for modules
Browse files Browse the repository at this point in the history
  • Loading branch information
@nantiferov
nantiferov committed Sep 11, 2024
1 parent 3f02f94 commit fc94ed9
Show file tree
Hide file tree
Showing 8 changed files with 31 additions and 37 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ test:
TEST_REDIS_CLUSTER_PASSWORD_URI="redis://localhost:17006" \
TEST_TILE38_URI="redis://localhost:19851" \
TEST_REDIS_SENTINEL_URI="redis://localhost:26379" \
TEST_REDIS_SEARCH_URI="redis://localhost:36379" \
TEST_REDIS_MODULES_URI="redis://localhost:36379" \
go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./...

.PHONY: lint
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,12 @@ Prometheus uses file watches and all changes to the json file are applied immedi
| redis-only-metrics | REDIS_EXPORTER_REDIS_ONLY_METRICS | Whether to also export go runtime metrics, defaults to false. |
| include-config-metrics | REDIS_EXPORTER_INCL_CONFIG_METRICS | Whether to include all config settings as metrics, defaults to false. |
| include-system-metrics | REDIS_EXPORTER_INCL_SYSTEM_METRICS | Whether to include system metrics like `total_system_memory_bytes`, defaults to false. |
| include-modules-metrics | REDIS_EXPORTER_INCL_MODULES_METRICS | Whether to collect Redis Modules metrics, defaults to false. |
| exclude-latency-histogram-metrics | REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS | Do not try to collect latency histogram metrics (to avoid `WARNING, LOGGED ONCE ONLY: cmd LATENCY HISTOGRAM` error on Redis < v7). |
| redact-config-metrics | REDIS_EXPORTER_REDACT_CONFIG_METRICS | Whether to redact config settings that include potentially sensitive information like passwords. |
| ping-on-connect | REDIS_EXPORTER_PING_ON_CONNECT | Whether to ping the redis instance after connecting and record the duration as a metric, defaults to false. |
| is-tile38 | REDIS_EXPORTER_IS_TILE38 | Whether to scrape Tile38 specific metrics, defaults to false. |
| is-cluster | REDIS_EXPORTER_IS_CLUSTER | Whether this is a redis cluster (Enable this if you need to fetch key level data on a Redis Cluster). |
| is-search | REDIS_EXPORTER_IS_SEARCH | Whether this is a redis has RediSearch module enabled and metrics should be collected. |
| export-client-list | REDIS_EXPORTER_EXPORT_CLIENT_LIST | Whether to scrape Client List specific metrics, defaults to false. |
| export-client-port | REDIS_EXPORTER_EXPORT_CLIENT_PORT | Whether to include the client's port when exporting the client list. Warning: including the port increases the number of metrics generated and will make your Prometheus server take up more memory |
| skip-tls-verification | REDIS_EXPORTER_SKIP_TLS_VERIFICATION | Whether to to skip TLS verification when the exporter connects to a Redis instance |
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ services:
ports:
- "19851:9851"

redis-search:
redis-stack:
image: redis/redis-stack-server:7.4.0-v0
ports:
- "36379:6379"
9 changes: 5 additions & 4 deletions exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ type Options struct {
ClientKeyFile string
CaCertFile string
InclConfigMetrics bool
IsSearch bool
InclModulesMetrics bool
DisableExportingKeyValues bool
ExcludeLatencyHistogramMetrics bool
RedactConfigMetrics bool
Expand Down Expand Up @@ -269,7 +269,8 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
"long_lock_waits": "long_lock_waits_total",
"current_client_thread": "current_client_thread",

// RediSearch module metrics
// Redis Modules metrics
// RediSearch module
"search_number_of_indexes": "search_number_of_indexes",
"search_used_memory_indexes": "search_used_memory_indexes",
"search_total_indexing_time": "search_total_indexing_time",
Expand Down Expand Up @@ -706,8 +707,8 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
e.extractTile38Metrics(ch, c)
}

if e.options.IsSearch {
e.extractSearchMetrics(ch, c)
if e.options.InclModulesMetrics {
e.extractModuleMetrics(ch, c)
}

if len(e.options.LuaScript) > 0 {
Expand Down
2 changes: 1 addition & 1 deletion exporter/http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) {

os.Getenv("TEST_REDIS5_URI"),
os.Getenv("TEST_REDIS6_URI"),
os.Getenv("TEST_REDIS_SEARCH_URI"),
os.Getenv("TEST_REDIS_MODULES_URI"),

// tile38 & Cluster need to be last in this list so we can identify them when selected, down in line 229
os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"),
Expand Down
17 changes: 5 additions & 12 deletions exporter/search.go → exporter/modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,14 @@ import (
log "github.com/sirupsen/logrus"
)

func (e *Exporter) extractSearchMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
for _, section := range [5]string{"search_version", "search_index", "search_memory", "search_cursors", "search_gc"} {
info, err := redis.String(doRedisCmd(c, "INFO", section))
if err != nil {
log.Errorf("extractSearchMetrics() err: %s", err)
return
}
e.registerSearchMetrics(ch, info)
func (e *Exporter) extractModuleMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
info, err := redis.String(doRedisCmd(c, "INFO", "MODULES"))
if err != nil {
log.Errorf("extractSearchMetrics() err: %s", err)
return

Check warning on line 15 in exporter/modules.go

View check run for this annotation

Codecov / codecov/patch

exporter/modules.go#L14-L15 

Added lines #L14 - L15 were not covered by tests
}

}

func (e *Exporter) registerSearchMetrics(ch chan<- prometheus.Metric, info string) {
lines := strings.Split(info, "\r\n")

for _, line := range lines {
log.Debugf("info: %s", line)

Expand Down
30 changes: 15 additions & 15 deletions exporter/search_test.go → exporter/modules_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,24 @@ import (
"github.com/prometheus/client_golang/prometheus"
)

func TestSearch(t *testing.T) {
if os.Getenv("TEST_REDIS_SEARCH_URI") == "" {
t.Skipf("TEST_REDIS_SEARCH_URI not set - skipping")
func TestModules(t *testing.T) {
if os.Getenv("TEST_REDIS_MODULES_URI") == "" {
t.Skipf("TEST_REDIS_MODULES_URI not set - skipping")
}

tsts := []struct {
addr string
isSearch bool
wantSearchMetrics bool
addr string
inclModulesMetrics bool
wantModulesMetrics bool
}{
{addr: os.Getenv("TEST_REDIS_SEARCH_URI"), isSearch: true, wantSearchMetrics: true},
{addr: os.Getenv("TEST_REDIS_SEARCH_URI"), isSearch: false, wantSearchMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), isSearch: true, wantSearchMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), isSearch: false, wantSearchMetrics: false},
{addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: true, wantModulesMetrics: true},
{addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: false, wantModulesMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: true, wantModulesMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: false, wantModulesMetrics: false},
}

for _, tst := range tsts {
e, _ := NewRedisExporter(tst.addr, Options{Namespace: "test", IsSearch: tst.isSearch})
e, _ := NewRedisExporter(tst.addr, Options{Namespace: "test", InclModulesMetrics: tst.inclModulesMetrics})

chM := make(chan prometheus.Metric)
go func() {
Expand All @@ -52,16 +52,16 @@ func TestSearch(t *testing.T) {
}
}

if tst.wantSearchMetrics {
if tst.wantModulesMetrics {
for want, found := range wantedMetrics {
if !found {
t.Errorf("%s was *not* found in RediSearch metrics but expected", want)
t.Errorf("%s was *not* found in Redis Modules metrics but expected", want)
}
}
} else if !tst.wantSearchMetrics {
} else if !tst.wantModulesMetrics {
for want, found := range wantedMetrics {
if found {
t.Errorf("%s was *found* in RediSearch metrics but *not* expected", want)
t.Errorf("%s was *found* in Redis Modules metrics but *not* expected", want)
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,13 @@ func main() {
setClientName = flag.Bool("set-client-name", getEnvBool("REDIS_EXPORTER_SET_CLIENT_NAME", true), "Whether to set client name to redis_exporter")
isTile38 = flag.Bool("is-tile38", getEnvBool("REDIS_EXPORTER_IS_TILE38", false), "Whether to scrape Tile38 specific metrics")
isCluster = flag.Bool("is-cluster", getEnvBool("REDIS_EXPORTER_IS_CLUSTER", false), "Whether this is a redis cluster (Enable this if you need to fetch key level data on a Redis Cluster).")
isSearch = flag.Bool("is-search", getEnvBool("REDIS_EXPORTER_IS_SEARCH", false), "Whether to scrape RediSearch module specific metrics")
exportClientList = flag.Bool("export-client-list", getEnvBool("REDIS_EXPORTER_EXPORT_CLIENT_LIST", false), "Whether to scrape Client List specific metrics")
exportClientPort = flag.Bool("export-client-port", getEnvBool("REDIS_EXPORTER_EXPORT_CLIENT_PORT", false), "Whether to include the client's port when exporting the client list. Warning: including the port increases the number of metrics generated and will make your Prometheus server take up more memory")
showVersion = flag.Bool("version", false, "Show version information and exit")
redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS", false), "Whether to also export go runtime metrics")
pingOnConnect = flag.Bool("ping-on-connect", getEnvBool("REDIS_EXPORTER_PING_ON_CONNECT", false), "Whether to ping the redis instance after connecting")
inclConfigMetrics = flag.Bool("include-config-metrics", getEnvBool("REDIS_EXPORTER_INCL_CONFIG_METRICS", false), "Whether to include all config settings as metrics")
inclModulesMetrics = flag.Bool("include-modules-metrics", getEnvBool("REDIS_EXPORTER_INCL_MODULES_METRICS", false), "Whether to collect Redis Modules metrics")

Check warning on line 97 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L97 

Added line #L97 was not covered by tests
disableExportingKeyValues = flag.Bool("disable-exporting-key-values", getEnvBool("REDIS_EXPORTER_DISABLE_EXPORTING_KEY_VALUES", false), "Whether to disable values of keys stored in redis as labels or not when using check-keys/check-single-key")
excludeLatencyHistogramMetrics = flag.Bool("exclude-latency-histogram-metrics", getEnvBool("REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS", false), "Do not try to collect latency histogram metrics")
redactConfigMetrics = flag.Bool("redact-config-metrics", getEnvBool("REDIS_EXPORTER_REDACT_CONFIG_METRICS", true), "Whether to redact config settings that include potentially sensitive information like passwords")
Expand Down Expand Up @@ -183,7 +183,7 @@ func main() {
SetClientName: *setClientName,
IsTile38: *isTile38,
IsCluster: *isCluster,
IsSearch: *isSearch,
InclModulesMetrics: *inclModulesMetrics,

Check warning on line 186 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L186 

Added line #L186 was not covered by tests
ExportClientList: *exportClientList,
ExportClientsInclPort: *exportClientPort,
SkipTLSVerification: *skipTLSVerification,
Expand Down

0 comments on commit fc94ed9

Please sign in to comment.