diff --git a/CHANGELOG.md b/CHANGELOG.md index cd096f168317..71a4edcd8ad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,8 @@ Main (unreleased) Previously, only `remote.*` and `local.*` components could be referenced without a circular dependency. (@rfratto) +- Add support for Basic Auth-secured connection with Elasticsearch cluster using `prometheus.exporter.elasticsearch`. (@hainenber) + - Add a `resource_to_telemetry_conversion` argument to `otelcol.exporter.prometheus` for converting resource attributes to Prometheus labels. (@hainenber) diff --git a/component/prometheus/exporter/elasticsearch/elasticsearch.go b/component/prometheus/exporter/elasticsearch/elasticsearch.go index 89123359cb9f..e09ed24c98d7 100644 --- a/component/prometheus/exporter/elasticsearch/elasticsearch.go +++ b/component/prometheus/exporter/elasticsearch/elasticsearch.go @@ -4,6 +4,7 @@ import ( "time" "github.com/grafana/agent/component" + commonCfg "github.com/grafana/agent/component/common/config" "github.com/grafana/agent/component/prometheus/exporter" "github.com/grafana/agent/pkg/integrations" "github.com/grafana/agent/pkg/integrations/elasticsearch_exporter" @@ -35,23 +36,24 @@ var DefaultArguments = Arguments{ } type Arguments struct { - Address string `river:"address,attr,optional"` - Timeout time.Duration `river:"timeout,attr,optional"` - AllNodes bool `river:"all,attr,optional"` - Node string `river:"node,attr,optional"` - ExportIndices bool `river:"indices,attr,optional"` - ExportIndicesSettings bool `river:"indices_settings,attr,optional"` - ExportClusterSettings bool `river:"cluster_settings,attr,optional"` - ExportShards bool `river:"shards,attr,optional"` - IncludeAliases bool `river:"aliases,attr,optional"` - ExportSnapshots bool `river:"snapshots,attr,optional"` - ExportClusterInfoInterval time.Duration `river:"clusterinfo_interval,attr,optional"` - CA string `river:"ca,attr,optional"` - ClientPrivateKey string `river:"client_private_key,attr,optional"` - ClientCert string `river:"client_cert,attr,optional"` - InsecureSkipVerify bool `river:"ssl_skip_verify,attr,optional"` - ExportDataStreams bool `river:"data_stream,attr,optional"` - ExportSLM bool `river:"slm,attr,optional"` + Address string `river:"address,attr,optional"` + Timeout time.Duration `river:"timeout,attr,optional"` + AllNodes bool `river:"all,attr,optional"` + Node string `river:"node,attr,optional"` + ExportIndices bool `river:"indices,attr,optional"` + ExportIndicesSettings bool `river:"indices_settings,attr,optional"` + ExportClusterSettings bool `river:"cluster_settings,attr,optional"` + ExportShards bool `river:"shards,attr,optional"` + IncludeAliases bool `river:"aliases,attr,optional"` + ExportSnapshots bool `river:"snapshots,attr,optional"` + ExportClusterInfoInterval time.Duration `river:"clusterinfo_interval,attr,optional"` + CA string `river:"ca,attr,optional"` + ClientPrivateKey string `river:"client_private_key,attr,optional"` + ClientCert string `river:"client_cert,attr,optional"` + InsecureSkipVerify bool `river:"ssl_skip_verify,attr,optional"` + ExportDataStreams bool `river:"data_stream,attr,optional"` + ExportSLM bool `river:"slm,attr,optional"` + BasicAuth *commonCfg.BasicAuth `river:"basic_auth,block,optional"` } // SetToDefault implements river.Defaulter. @@ -78,5 +80,6 @@ func (a *Arguments) Convert() *elasticsearch_exporter.Config { InsecureSkipVerify: a.InsecureSkipVerify, ExportDataStreams: a.ExportDataStreams, ExportSLM: a.ExportSLM, + BasicAuth: a.BasicAuth.Convert(), } } diff --git a/component/prometheus/exporter/elasticsearch/elasticsearch_test.go b/component/prometheus/exporter/elasticsearch/elasticsearch_test.go index 3e87a5a98dc6..5c71a8ac712f 100644 --- a/component/prometheus/exporter/elasticsearch/elasticsearch_test.go +++ b/component/prometheus/exporter/elasticsearch/elasticsearch_test.go @@ -4,8 +4,11 @@ import ( "testing" "time" + commonCfg "github.com/grafana/agent/component/common/config" "github.com/grafana/agent/pkg/integrations/elasticsearch_exporter" "github.com/grafana/river" + "github.com/grafana/river/rivertypes" + promCfg "github.com/prometheus/common/config" "github.com/stretchr/testify/require" ) @@ -27,6 +30,10 @@ func TestRiverUnmarshal(t *testing.T) { ssl_skip_verify = true data_stream = true slm = true + basic_auth { + username = "username" + password = "pass" + } ` var args Arguments @@ -50,6 +57,10 @@ func TestRiverUnmarshal(t *testing.T) { InsecureSkipVerify: true, ExportDataStreams: true, ExportSLM: true, + BasicAuth: &commonCfg.BasicAuth{ + Username: "username", + Password: rivertypes.Secret("pass"), + }, } require.Equal(t, expected, args) @@ -73,6 +84,10 @@ func TestConvert(t *testing.T) { ssl_skip_verify = true data_stream = true slm = true + basic_auth { + username = "username" + password = "pass" + } ` var args Arguments err := river.Unmarshal([]byte(riverConfig), &args) @@ -97,6 +112,10 @@ func TestConvert(t *testing.T) { InsecureSkipVerify: true, ExportDataStreams: true, ExportSLM: true, + BasicAuth: &promCfg.BasicAuth{ + Username: "username", + Password: promCfg.Secret("pass"), + }, } require.Equal(t, expected, *res) } diff --git a/converter/internal/staticconvert/internal/build/elasticsearch_exporter.go b/converter/internal/staticconvert/internal/build/elasticsearch_exporter.go index 97022a1f182f..67dda9e0285c 100644 --- a/converter/internal/staticconvert/internal/build/elasticsearch_exporter.go +++ b/converter/internal/staticconvert/internal/build/elasticsearch_exporter.go @@ -1,9 +1,11 @@ package build import ( + commonCfg "github.com/grafana/agent/component/common/config" "github.com/grafana/agent/component/discovery" "github.com/grafana/agent/component/prometheus/exporter/elasticsearch" "github.com/grafana/agent/pkg/integrations/elasticsearch_exporter" + "github.com/grafana/river/rivertypes" ) func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elasticsearch_exporter.Config, instanceKey *string) discovery.Exports { @@ -12,7 +14,7 @@ func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elastics } func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsearch.Arguments { - return &elasticsearch.Arguments{ + arg := &elasticsearch.Arguments{ Address: config.Address, Timeout: config.Timeout, AllNodes: config.AllNodes, @@ -31,4 +33,14 @@ func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsear ExportDataStreams: config.ExportDataStreams, ExportSLM: config.ExportSLM, } + + if config.BasicAuth != nil { + arg.BasicAuth = &commonCfg.BasicAuth{ + Username: config.BasicAuth.Username, + Password: rivertypes.Secret(config.BasicAuth.Password), + PasswordFile: config.BasicAuth.PasswordFile, + } + } + + return arg } diff --git a/docs/sources/flow/reference/components/prometheus.exporter.elasticsearch.md b/docs/sources/flow/reference/components/prometheus.exporter.elasticsearch.md index fec1953dbef1..6feb9c683eeb 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.elasticsearch.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.elasticsearch.md @@ -56,6 +56,21 @@ Omitted fields take their default values. | `data_streams` | `bool` | Export stats for Data Streams. | | no | | `slm` | `bool` | Export stats for SLM (Snapshot Lifecycle Management). | | no | +## Blocks + +The following blocks are supported inside the definition of +`prometheus.exporter.elasticsearch`: + +| Hierarchy | Block | Description | Required | +| ------------------- | ----------------- | -------------------------------------------------------- | -------- | +| basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no | + +[basic_auth]: #basic_auth-block + +### basic_auth block + +{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" version="" >}} + ## Exported fields {{< docs/shared lookup="flow/reference/components/exporter-component-exports.md" source="agent" version="" >}} @@ -84,6 +99,10 @@ from `prometheus.exporter.elasticsearch`: ```river prometheus.exporter.elasticsearch "example" { address = "http://localhost:9200" + basic_auth { + username = USERNAME + password = PASSWORD + } } // Configure a prometheus.scrape component to collect Elasticsearch metrics. diff --git a/docs/sources/static/configuration/integrations/elasticsearch-exporter-config.md b/docs/sources/static/configuration/integrations/elasticsearch-exporter-config.md index 22f26d4f15ad..9e0f3ee0f88b 100644 --- a/docs/sources/static/configuration/integrations/elasticsearch-exporter-config.md +++ b/docs/sources/static/configuration/integrations/elasticsearch-exporter-config.md @@ -116,4 +116,12 @@ Full reference of options: # Export stats for SLM (Snapshot Lifecycle Management). [ slm: ] + + # Sets the `Authorization` header on every ES probe with the + # configured username and password. + # password and password_file are mutually exclusive. + basic_auth: + [ username: ] + [ password: ] + [ password_file: ] ``` diff --git a/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go b/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go index d22fd2c618d8..d221a3521f72 100644 --- a/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go +++ b/pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go @@ -4,9 +4,12 @@ package elasticsearch_exporter //nolint:golint import ( "context" + "encoding/base64" "fmt" "net/http" "net/url" + "os" + "strings" "time" "github.com/go-kit/log" @@ -15,6 +18,7 @@ import ( integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" "github.com/grafana/agent/pkg/integrations/v2/metricsutils" "github.com/prometheus/client_golang/prometheus" + promCfg "github.com/prometheus/common/config" "github.com/prometheus-community/elasticsearch_exporter/collector" "github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo" @@ -66,6 +70,21 @@ type Config struct { ExportDataStreams bool `yaml:"data_stream,omitempty"` // Export stats for Snapshot Lifecycle Management ExportSLM bool `yaml:"slm,omitempty"` + // BasicAuth block allows secure connection with Elasticsearch cluster via Basic-Auth + BasicAuth *promCfg.BasicAuth `yaml:"basic_auth,omitempty"` +} + +// Custom http.Transport struct for Basic Auth-secured communication with ES cluster +type BasicAuthHTTPTransport struct { + http.Transport + authHeader string +} + +func (b *BasicAuthHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if b.authHeader != "" { + req.Header.Add("authorization", b.authHeader) + } + return b.Transport.RoundTrip(req) } // UnmarshalYAML implements yaml.Unmarshaler for Config @@ -115,14 +134,39 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) { // returns nil if not provided and falls back to simple TCP. tlsConfig := createTLSConfig(c.CA, c.ClientCert, c.ClientPrivateKey, c.InsecureSkipVerify) - httpClient := &http.Client{ - Timeout: c.Timeout, - Transport: &http.Transport{ + esHttpTransport := &BasicAuthHTTPTransport{ + Transport: http.Transport{ TLSClientConfig: tlsConfig, Proxy: http.ProxyFromEnvironment, }, } + if c.BasicAuth != nil { + password := string(c.BasicAuth.Password) + if len(c.BasicAuth.PasswordFile) > 0 { + buff, err := os.ReadFile(c.BasicAuth.PasswordFile) + if err != nil { + return nil, fmt.Errorf("unable to load password file %s: %w", c.BasicAuth.PasswordFile, err) + } + password = strings.TrimSpace(string(buff)) + } + username := c.BasicAuth.Username + if len(c.BasicAuth.UsernameFile) > 0 { + buff, err := os.ReadFile(c.BasicAuth.UsernameFile) + if err != nil { + return nil, fmt.Errorf("unable to load username file %s: %w", c.BasicAuth.UsernameFile, err) + } + username = strings.TrimSpace(string(buff)) + } + encodedAuth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + esHttpTransport.authHeader = "Basic " + encodedAuth + } + + httpClient := &http.Client{ + Timeout: c.Timeout, + Transport: esHttpTransport, + } + clusterInfoRetriever := clusterinfo.New(logger, httpClient, esURL, c.ExportClusterInfoInterval) collectors := []prometheus.Collector{