Skip to content

Commit

Permalink
feat(exporter/elasticsearch): add Basic Auth support (grafana#5814)
Browse files Browse the repository at this point in the history
* feat(exporter/elasticsearch): add Basic Auth support

Signed-off-by: hainenber <[email protected]>

* misc: fix changelog entry (grafana#5812)

PR grafana#5802 accidentally added the changelog entry to a published release.

* fix(exporter/elasticsearch): only add Auth header when needed

Signed-off-by: hainenber <[email protected]>

* doc(exporter/elasticsearch): add Basic Auth block desc

Signed-off-by: hainenber <[email protected]>

* feat(exporter/elasticsearch): add Basic Auth support

Signed-off-by: hainenber <[email protected]>

* feat(exporter/elasticsearch): support Basic Auth for static config

Signed-off-by: hainenber <[email protected]>

* chore(CHANGELOG): remove merge conflict artifact

Signed-off-by: hainenber <[email protected]>

* misc: fix changelog entry

Signed-off-by: hainenber <[email protected]>

* feat(converter): map ES exporter's BasicAuth flow from static to flow

Signed-off-by: hainenber <[email protected]>

* fix(converter): nil check for ES exporter's Basic Auth

Signed-off-by: hainenber <[email protected]>

* lint

Signed-off-by: erikbaranowski <[email protected]>

---------

Signed-off-by: hainenber <[email protected]>
Signed-off-by: erikbaranowski <[email protected]>
Co-authored-by: Robert Fratto <[email protected]>
Co-authored-by: erikbaranowski <[email protected]>
  • Loading branch information
3 people authored and BarunKGP committed Feb 20, 2024
1 parent 04244c3 commit 33b34a0
Show file tree
Hide file tree
Showing 7 changed files with 128 additions and 21 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ Main (unreleased)
Previously, only `remote.*` and `local.*` components could be referenced
without a circular dependency. (@rfratto)

- Add support for Basic Auth-secured connection with Elasticsearch cluster using `prometheus.exporter.elasticsearch`. (@hainenber)

- Add a `resource_to_telemetry_conversion` argument to `otelcol.exporter.prometheus`
for converting resource attributes to Prometheus labels. (@hainenber)

Expand Down
37 changes: 20 additions & 17 deletions component/prometheus/exporter/elasticsearch/elasticsearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"time"

"github.com/grafana/agent/component"
commonCfg "github.com/grafana/agent/component/common/config"
"github.com/grafana/agent/component/prometheus/exporter"
"github.com/grafana/agent/pkg/integrations"
"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
Expand Down Expand Up @@ -35,23 +36,24 @@ var DefaultArguments = Arguments{
}

type Arguments struct {
Address string `river:"address,attr,optional"`
Timeout time.Duration `river:"timeout,attr,optional"`
AllNodes bool `river:"all,attr,optional"`
Node string `river:"node,attr,optional"`
ExportIndices bool `river:"indices,attr,optional"`
ExportIndicesSettings bool `river:"indices_settings,attr,optional"`
ExportClusterSettings bool `river:"cluster_settings,attr,optional"`
ExportShards bool `river:"shards,attr,optional"`
IncludeAliases bool `river:"aliases,attr,optional"`
ExportSnapshots bool `river:"snapshots,attr,optional"`
ExportClusterInfoInterval time.Duration `river:"clusterinfo_interval,attr,optional"`
CA string `river:"ca,attr,optional"`
ClientPrivateKey string `river:"client_private_key,attr,optional"`
ClientCert string `river:"client_cert,attr,optional"`
InsecureSkipVerify bool `river:"ssl_skip_verify,attr,optional"`
ExportDataStreams bool `river:"data_stream,attr,optional"`
ExportSLM bool `river:"slm,attr,optional"`
Address string `river:"address,attr,optional"`
Timeout time.Duration `river:"timeout,attr,optional"`
AllNodes bool `river:"all,attr,optional"`
Node string `river:"node,attr,optional"`
ExportIndices bool `river:"indices,attr,optional"`
ExportIndicesSettings bool `river:"indices_settings,attr,optional"`
ExportClusterSettings bool `river:"cluster_settings,attr,optional"`
ExportShards bool `river:"shards,attr,optional"`
IncludeAliases bool `river:"aliases,attr,optional"`
ExportSnapshots bool `river:"snapshots,attr,optional"`
ExportClusterInfoInterval time.Duration `river:"clusterinfo_interval,attr,optional"`
CA string `river:"ca,attr,optional"`
ClientPrivateKey string `river:"client_private_key,attr,optional"`
ClientCert string `river:"client_cert,attr,optional"`
InsecureSkipVerify bool `river:"ssl_skip_verify,attr,optional"`
ExportDataStreams bool `river:"data_stream,attr,optional"`
ExportSLM bool `river:"slm,attr,optional"`
BasicAuth *commonCfg.BasicAuth `river:"basic_auth,block,optional"`
}

// SetToDefault implements river.Defaulter.
Expand All @@ -78,5 +80,6 @@ func (a *Arguments) Convert() *elasticsearch_exporter.Config {
InsecureSkipVerify: a.InsecureSkipVerify,
ExportDataStreams: a.ExportDataStreams,
ExportSLM: a.ExportSLM,
BasicAuth: a.BasicAuth.Convert(),
}
}
19 changes: 19 additions & 0 deletions component/prometheus/exporter/elasticsearch/elasticsearch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@ import (
"testing"
"time"

commonCfg "github.com/grafana/agent/component/common/config"
"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
"github.com/grafana/river"
"github.com/grafana/river/rivertypes"
promCfg "github.com/prometheus/common/config"
"github.com/stretchr/testify/require"
)

Expand All @@ -27,6 +30,10 @@ func TestRiverUnmarshal(t *testing.T) {
ssl_skip_verify = true
data_stream = true
slm = true
basic_auth {
username = "username"
password = "pass"
}
`

var args Arguments
Expand All @@ -50,6 +57,10 @@ func TestRiverUnmarshal(t *testing.T) {
InsecureSkipVerify: true,
ExportDataStreams: true,
ExportSLM: true,
BasicAuth: &commonCfg.BasicAuth{
Username: "username",
Password: rivertypes.Secret("pass"),
},
}

require.Equal(t, expected, args)
Expand All @@ -73,6 +84,10 @@ func TestConvert(t *testing.T) {
ssl_skip_verify = true
data_stream = true
slm = true
basic_auth {
username = "username"
password = "pass"
}
`
var args Arguments
err := river.Unmarshal([]byte(riverConfig), &args)
Expand All @@ -97,6 +112,10 @@ func TestConvert(t *testing.T) {
InsecureSkipVerify: true,
ExportDataStreams: true,
ExportSLM: true,
BasicAuth: &promCfg.BasicAuth{
Username: "username",
Password: promCfg.Secret("pass"),
},
}
require.Equal(t, expected, *res)
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package build

import (
commonCfg "github.com/grafana/agent/component/common/config"
"github.com/grafana/agent/component/discovery"
"github.com/grafana/agent/component/prometheus/exporter/elasticsearch"
"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
"github.com/grafana/river/rivertypes"
)

func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elasticsearch_exporter.Config, instanceKey *string) discovery.Exports {
Expand All @@ -12,7 +14,7 @@ func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elastics
}

func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsearch.Arguments {
return &elasticsearch.Arguments{
arg := &elasticsearch.Arguments{
Address: config.Address,
Timeout: config.Timeout,
AllNodes: config.AllNodes,
Expand All @@ -31,4 +33,14 @@ func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsear
ExportDataStreams: config.ExportDataStreams,
ExportSLM: config.ExportSLM,
}

if config.BasicAuth != nil {
arg.BasicAuth = &commonCfg.BasicAuth{
Username: config.BasicAuth.Username,
Password: rivertypes.Secret(config.BasicAuth.Password),
PasswordFile: config.BasicAuth.PasswordFile,
}
}

return arg
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,21 @@ Omitted fields take their default values.
| `data_streams` | `bool` | Export stats for Data Streams. | | no |
| `slm` | `bool` | Export stats for SLM (Snapshot Lifecycle Management). | | no |

## Blocks

The following blocks are supported inside the definition of
`prometheus.exporter.elasticsearch`:

| Hierarchy | Block | Description | Required |
| ------------------- | ----------------- | -------------------------------------------------------- | -------- |
| basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no |

[basic_auth]: #basic_auth-block

### basic_auth block

{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" version="<AGENT VERSION>" >}}

## Exported fields

{{< docs/shared lookup="flow/reference/components/exporter-component-exports.md" source="agent" version="<AGENT_VERSION>" >}}
Expand Down Expand Up @@ -84,6 +99,10 @@ from `prometheus.exporter.elasticsearch`:
```river
prometheus.exporter.elasticsearch "example" {
address = "http://localhost:9200"
basic_auth {
username = USERNAME
password = PASSWORD
}
}
// Configure a prometheus.scrape component to collect Elasticsearch metrics.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,4 +116,12 @@ Full reference of options:

# Export stats for SLM (Snapshot Lifecycle Management).
[ slm: <boolean> ]

# Sets the `Authorization` header on every ES probe with the
# configured username and password.
# password and password_file are mutually exclusive.
basic_auth:
[ username: <string> ]
[ password: <secret> ]
[ password_file: <string> ]
```
50 changes: 47 additions & 3 deletions pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@ package elasticsearch_exporter //nolint:golint

import (
"context"
"encoding/base64"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"time"

"github.com/go-kit/log"
Expand All @@ -15,6 +18,7 @@ import (
integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
"github.com/prometheus/client_golang/prometheus"
promCfg "github.com/prometheus/common/config"

"github.com/prometheus-community/elasticsearch_exporter/collector"
"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
Expand Down Expand Up @@ -66,6 +70,21 @@ type Config struct {
ExportDataStreams bool `yaml:"data_stream,omitempty"`
// Export stats for Snapshot Lifecycle Management
ExportSLM bool `yaml:"slm,omitempty"`
// BasicAuth block allows secure connection with Elasticsearch cluster via Basic-Auth
BasicAuth *promCfg.BasicAuth `yaml:"basic_auth,omitempty"`
}

// Custom http.Transport struct for Basic Auth-secured communication with ES cluster
type BasicAuthHTTPTransport struct {
http.Transport
authHeader string
}

func (b *BasicAuthHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if b.authHeader != "" {
req.Header.Add("authorization", b.authHeader)
}
return b.Transport.RoundTrip(req)
}

// UnmarshalYAML implements yaml.Unmarshaler for Config
Expand Down Expand Up @@ -115,14 +134,39 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) {
// returns nil if not provided and falls back to simple TCP.
tlsConfig := createTLSConfig(c.CA, c.ClientCert, c.ClientPrivateKey, c.InsecureSkipVerify)

httpClient := &http.Client{
Timeout: c.Timeout,
Transport: &http.Transport{
esHttpTransport := &BasicAuthHTTPTransport{
Transport: http.Transport{
TLSClientConfig: tlsConfig,
Proxy: http.ProxyFromEnvironment,
},
}

if c.BasicAuth != nil {
password := string(c.BasicAuth.Password)
if len(c.BasicAuth.PasswordFile) > 0 {
buff, err := os.ReadFile(c.BasicAuth.PasswordFile)
if err != nil {
return nil, fmt.Errorf("unable to load password file %s: %w", c.BasicAuth.PasswordFile, err)
}
password = strings.TrimSpace(string(buff))
}
username := c.BasicAuth.Username
if len(c.BasicAuth.UsernameFile) > 0 {
buff, err := os.ReadFile(c.BasicAuth.UsernameFile)
if err != nil {
return nil, fmt.Errorf("unable to load username file %s: %w", c.BasicAuth.UsernameFile, err)
}
username = strings.TrimSpace(string(buff))
}
encodedAuth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
esHttpTransport.authHeader = "Basic " + encodedAuth
}

httpClient := &http.Client{
Timeout: c.Timeout,
Transport: esHttpTransport,
}

clusterInfoRetriever := clusterinfo.New(logger, httpClient, esURL, c.ExportClusterInfoInterval)

collectors := []prometheus.Collector{
Expand Down

0 comments on commit 33b34a0

Please sign in to comment.