From a99ec7aa6c3544227f90cbb16bb953ccce16ada2 Mon Sep 17 00:00:00 2001 From: Shahar Tal Date: Thu, 17 Aug 2023 20:23:10 +0300 Subject: [PATCH 1/4] Added indices filter flag Signed-off-by: Shahar Tal --- collector/indices.go | 10 ++++++++-- collector/indices_mappings.go | 19 +++++++++++-------- collector/indices_mappings_test.go | 4 ++-- collector/indices_settings.go | 19 +++++++++++-------- collector/indices_settings_test.go | 2 +- collector/indices_test.go | 4 ++-- main.go | 9 ++++++--- 7 files changed, 41 insertions(+), 26 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index a0c4d96e..b76f6087 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -59,6 +59,7 @@ type Indices struct { logger log.Logger client *http.Client url *url.URL + indicesFilter string shards bool aliases bool clusterInfoCh chan *clusterinfo.Response @@ -74,7 +75,7 @@ type Indices struct { } // NewIndices defines Indices Prometheus metrics -func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool, indicesFilter string) *Indices { indexLabels := labels{ keys: func(...string) []string { @@ -122,6 +123,7 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo logger: logger, client: client, url: url, + indicesFilter: indicesFilter, shards: shards, aliases: includeAliases, clusterInfoCh: make(chan *clusterinfo.Response), @@ -1102,7 +1104,11 @@ func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse u := *i.url - u.Path = path.Join(u.Path, "/_all/_stats") + indicesStatsQueryPath := fmt.Sprintf("/%s/_stats", i.indicesFilter) + _ = level.Debug(i.logger).Log( + "msg", fmt.Sprintf("indices fetch query path: %s", indicesStatsQueryPath), + ) + u.Path = path.Join(u.Path, indicesStatsQueryPath) if i.shards { u.RawQuery = "ignore_unavailable=true&level=shards" } else { diff --git a/collector/indices_mappings.go b/collector/indices_mappings.go index bfe072ab..577f5298 100644 --- a/collector/indices_mappings.go +++ b/collector/indices_mappings.go @@ -38,9 +38,10 @@ type indicesMappingsMetric struct { // IndicesMappings information struct type IndicesMappings struct { - logger log.Logger - client *http.Client - url *url.URL + logger log.Logger + client *http.Client + url *url.URL + indicesFilter string up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter @@ -49,13 +50,14 @@ type IndicesMappings struct { } // NewIndicesMappings defines Indices IndexMappings Prometheus metrics -func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL) *IndicesMappings { +func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL, indicesFilter string) *IndicesMappings { subsystem := "indices_mappings_stats" return &IndicesMappings{ - logger: logger, - client: client, - url: url, + logger: logger, + client: client, + url: url, + indicesFilter: indicesFilter, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "up"), @@ -157,7 +159,8 @@ func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) { u := *im.url - u.Path = path.Join(u.Path, "/_all/_mappings") + indicesMappingsQueryPath := fmt.Sprintf("/%s/_mappings", im.indicesFilter) + u.Path = path.Join(u.Path, indicesMappingsQueryPath) return im.getAndParseURL(&u) } diff --git a/collector/indices_mappings_test.go b/collector/indices_mappings_test.go index 7031ada2..64e8d682 100644 --- a/collector/indices_mappings_test.go +++ b/collector/indices_mappings_test.go @@ -124,7 +124,7 @@ func TestMapping(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u) + c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u, "_all") imr, err := c.fetchAndDecodeIndicesMappings() if err != nil { t.Fatalf("Failed to fetch or decode indices mappings: %s", err) @@ -362,7 +362,7 @@ func TestIndexMappingFieldCount(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u) + c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u, "_all") indicesMappingsResponse, err := c.fetchAndDecodeIndicesMappings() if err != nil { t.Fatalf("Failed to fetch or decode indices mappings: %s", err) diff --git a/collector/indices_settings.go b/collector/indices_settings.go index f1926798..af9e9b42 100644 --- a/collector/indices_settings.go +++ b/collector/indices_settings.go @@ -29,9 +29,10 @@ import ( // IndicesSettings information struct type IndicesSettings struct { - logger log.Logger - client *http.Client - url *url.URL + logger log.Logger + client *http.Client + url *url.URL + indicesFilter string up prometheus.Gauge readOnlyIndices prometheus.Gauge @@ -52,11 +53,12 @@ type indicesSettingsMetric struct { } // NewIndicesSettings defines Indices Settings Prometheus metrics -func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL) *IndicesSettings { +func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL, indicesFilter string) *IndicesSettings { return &IndicesSettings{ - logger: logger, - client: client, - url: url, + logger: logger, + client: client, + url: url, + indicesFilter: indicesFilter, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "up"), @@ -154,7 +156,8 @@ func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error { func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) { u := *cs.url - u.Path = path.Join(u.Path, "/_all/_settings") + indicesSettingsQueryPath := fmt.Sprintf("/%s/_settings", cs.indicesFilter) + u.Path = path.Join(u.Path, indicesSettingsQueryPath) var asr IndicesSettingsResponse err := cs.getAndParseURL(&u, &asr) if err != nil { diff --git a/collector/indices_settings_test.go b/collector/indices_settings_test.go index 7a3d0fcd..a16f360f 100644 --- a/collector/indices_settings_test.go +++ b/collector/indices_settings_test.go @@ -70,7 +70,7 @@ func TestIndicesSettings(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u) + c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u, "_all") nsr, err := c.fetchAndDecodeIndicesSettings() if err != nil { t.Fatalf("Failed to fetch or decode indices settings: %s", err) diff --git a/collector/indices_test.go b/collector/indices_test.go index 1d164b65..d325d645 100644 --- a/collector/indices_test.go +++ b/collector/indices_test.go @@ -49,7 +49,7 @@ func TestIndices(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false, "_all") stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) @@ -123,7 +123,7 @@ func TestAliases(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true, "_all") stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) diff --git a/main.go b/main.go index e50dd09e..429cbcb0 100644 --- a/main.go +++ b/main.go @@ -89,6 +89,9 @@ func main() { esExportSLM = kingpin.Flag("es.slm", "Export stats for SLM snapshots."). Default("false").Bool() + esIndicesFilter = kingpin.Flag("es.indices_filter", + "Indices filter by name for which metrics should be exposed, using prefix with wildcards and/or commas as multi-selection delimiter."). + Default("_all").String() esExportDataStream = kingpin.Flag("es.data_stream", "Export stas for Data Streams."). Default("false").Bool() @@ -203,7 +206,7 @@ func main() { if *esExportIndices || *esExportShards { prometheus.MustRegister(collector.NewShards(logger, httpClient, esURL)) - iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases) + iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases, *esIndicesFilter) prometheus.MustRegister(iC) if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil { level.Error(logger).Log("msg", "failed to register indices collector in cluster info") @@ -224,11 +227,11 @@ func main() { } if *esExportIndicesSettings { - prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL, *esIndicesFilter)) } if *esExportIndicesMappings { - prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL, *esIndicesFilter)) } if *esExportILM { From 9cfa3a3912b2688441af157298ad23a8f264ad39 Mon Sep 17 00:00:00 2001 From: Shahar Tal Date: Thu, 17 Aug 2023 20:35:07 +0300 Subject: [PATCH 2/4] added es.indices_filter option to readme Signed-off-by: Shahar Tal --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4d22d104..152cbbb2 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ elasticsearch_exporter --help | es.shards | 1.0.3rc1 | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false | | es.snapshots | 1.0.4rc1 | If true, query stats for the cluster snapshots. | false | | es.slm | | If true, query stats for SLM. | false | +| es.indices_filter | | If set, query [settings, stats, mappings] will be filtered by the provided pattern. `Example: prefix_1_*,prefix_2_*` | _all | | es.data_stream | | If true, query state for Data Steams. | false | | es.timeout | 1.0.2 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s | | es.ca | 1.0.2 | Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection. | | From bd71ed2ba57bbff0fe3453c886f4d5564079dfb6 Mon Sep 17 00:00:00 2001 From: Shahar Tal Date: Fri, 25 Aug 2023 14:52:44 +0300 Subject: [PATCH 3/4] rename es.indices_filter flag into es.indices_selector Signed-off-by: Shahar Tal --- README.md | 2 +- collector/indices.go | 20 ++++++++++---------- collector/indices_mappings.go | 20 ++++++++++---------- collector/indices_settings.go | 20 ++++++++++---------- main.go | 10 +++++----- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 152cbbb2..88f07bc5 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ elasticsearch_exporter --help | es.shards | 1.0.3rc1 | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false | | es.snapshots | 1.0.4rc1 | If true, query stats for the cluster snapshots. | false | | es.slm | | If true, query stats for SLM. | false | -| es.indices_filter | | If set, query [settings, stats, mappings] will be filtered by the provided pattern. `Example: prefix_1_*,prefix_2_*` | _all | +| es.indices_selector | | If set, Only the selcted indices will be queried [settings, stats, mappings] based on the provided pattern. `Example: prefix_1_*,prefix_2_*` | _all | | es.data_stream | | If true, query state for Data Steams. | false | | es.timeout | 1.0.2 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s | | es.ca | 1.0.2 | Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection. | | diff --git a/collector/indices.go b/collector/indices.go index b76f6087..be7facce 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -59,7 +59,7 @@ type Indices struct { logger log.Logger client *http.Client url *url.URL - indicesFilter string + indicesSelector string shards bool aliases bool clusterInfoCh chan *clusterinfo.Response @@ -75,7 +75,7 @@ type Indices struct { } // NewIndices defines Indices Prometheus metrics -func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool, indicesFilter string) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool, indicesSelector string) *Indices { indexLabels := labels{ keys: func(...string) []string { @@ -120,13 +120,13 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo } indices := &Indices{ - logger: logger, - client: client, - url: url, - indicesFilter: indicesFilter, - shards: shards, - aliases: includeAliases, - clusterInfoCh: make(chan *clusterinfo.Response), + logger: logger, + client: client, + url: url, + indicesSelector: indicesSelector, + shards: shards, + aliases: includeAliases, + clusterInfoCh: make(chan *clusterinfo.Response), lastClusterInfo: &clusterinfo.Response{ ClusterName: "unknown_cluster", }, @@ -1104,7 +1104,7 @@ func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse u := *i.url - indicesStatsQueryPath := fmt.Sprintf("/%s/_stats", i.indicesFilter) + indicesStatsQueryPath := fmt.Sprintf("/%s/_stats", i.indicesSelector) _ = level.Debug(i.logger).Log( "msg", fmt.Sprintf("indices fetch query path: %s", indicesStatsQueryPath), ) diff --git a/collector/indices_mappings.go b/collector/indices_mappings.go index 577f5298..733cf974 100644 --- a/collector/indices_mappings.go +++ b/collector/indices_mappings.go @@ -38,10 +38,10 @@ type indicesMappingsMetric struct { // IndicesMappings information struct type IndicesMappings struct { - logger log.Logger - client *http.Client - url *url.URL - indicesFilter string + logger log.Logger + client *http.Client + url *url.URL + indicesSelector string up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter @@ -50,14 +50,14 @@ type IndicesMappings struct { } // NewIndicesMappings defines Indices IndexMappings Prometheus metrics -func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL, indicesFilter string) *IndicesMappings { +func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL, indicesSelector string) *IndicesMappings { subsystem := "indices_mappings_stats" return &IndicesMappings{ - logger: logger, - client: client, - url: url, - indicesFilter: indicesFilter, + logger: logger, + client: client, + url: url, + indicesSelector: indicesSelector, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "up"), @@ -159,7 +159,7 @@ func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) { u := *im.url - indicesMappingsQueryPath := fmt.Sprintf("/%s/_mappings", im.indicesFilter) + indicesMappingsQueryPath := fmt.Sprintf("/%s/_mappings", im.indicesSelector) u.Path = path.Join(u.Path, indicesMappingsQueryPath) return im.getAndParseURL(&u) } diff --git a/collector/indices_settings.go b/collector/indices_settings.go index af9e9b42..50da9b05 100644 --- a/collector/indices_settings.go +++ b/collector/indices_settings.go @@ -29,10 +29,10 @@ import ( // IndicesSettings information struct type IndicesSettings struct { - logger log.Logger - client *http.Client - url *url.URL - indicesFilter string + logger log.Logger + client *http.Client + url *url.URL + indicesSelector string up prometheus.Gauge readOnlyIndices prometheus.Gauge @@ -53,12 +53,12 @@ type indicesSettingsMetric struct { } // NewIndicesSettings defines Indices Settings Prometheus metrics -func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL, indicesFilter string) *IndicesSettings { +func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL, indicesSelector string) *IndicesSettings { return &IndicesSettings{ - logger: logger, - client: client, - url: url, - indicesFilter: indicesFilter, + logger: logger, + client: client, + url: url, + indicesSelector: indicesSelector, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "up"), @@ -156,7 +156,7 @@ func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error { func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) { u := *cs.url - indicesSettingsQueryPath := fmt.Sprintf("/%s/_settings", cs.indicesFilter) + indicesSettingsQueryPath := fmt.Sprintf("/%s/_settings", cs.indicesSelector) u.Path = path.Join(u.Path, indicesSettingsQueryPath) var asr IndicesSettingsResponse err := cs.getAndParseURL(&u, &asr) diff --git a/main.go b/main.go index 429cbcb0..0833bc80 100644 --- a/main.go +++ b/main.go @@ -89,8 +89,8 @@ func main() { esExportSLM = kingpin.Flag("es.slm", "Export stats for SLM snapshots."). Default("false").Bool() - esIndicesFilter = kingpin.Flag("es.indices_filter", - "Indices filter by name for which metrics should be exposed, using prefix with wildcards and/or commas as multi-selection delimiter."). + esIndicesSelector = kingpin.Flag("es.indices_selector", + "Selcted indices by name for which metrics should be exposed, using prefix with wildcards and/or commas as multi-selection delimiter."). Default("_all").String() esExportDataStream = kingpin.Flag("es.data_stream", "Export stas for Data Streams."). @@ -206,7 +206,7 @@ func main() { if *esExportIndices || *esExportShards { prometheus.MustRegister(collector.NewShards(logger, httpClient, esURL)) - iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases, *esIndicesFilter) + iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases, *esIndicesSelector) prometheus.MustRegister(iC) if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil { level.Error(logger).Log("msg", "failed to register indices collector in cluster info") @@ -227,11 +227,11 @@ func main() { } if *esExportIndicesSettings { - prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL, *esIndicesFilter)) + prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL, *esIndicesSelector)) } if *esExportIndicesMappings { - prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL, *esIndicesFilter)) + prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL, *esIndicesSelector)) } if *esExportILM { From eab5fd1c3a8d276a45b6b3551f815f6cd62d66e1 Mon Sep 17 00:00:00 2001 From: Shahar Tal Date: Wed, 15 Nov 2023 12:36:08 +0200 Subject: [PATCH 4/4] Update main.go Modified. es.indices_selector description Co-authored-by: Joe Adams Signed-off-by: Shahar Tal --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 9e96de4f..6ba1bbec 100644 --- a/main.go +++ b/main.go @@ -87,7 +87,7 @@ func main() { "Export stats for SLM snapshots."). Default("false").Bool() esIndicesSelector = kingpin.Flag("es.indices_selector", - "Selcted indices by name for which metrics should be exposed, using prefix with wildcards and/or commas as multi-selection delimiter."). + "Select indices by name for which metrics should be exposed, using prefix with wildcards and/or commas as multi-selection delimiter."). Default("_all").String() esExportDataStream = kingpin.Flag("es.data_stream", "Export stas for Data Streams.").