Skip to content

Commit

Permalink
feat: implement git backend environments storage
Browse files Browse the repository at this point in the history
  • Loading branch information
GeorgeMac committed Jan 29, 2025
1 parent afd9531 commit 90b12c2
Show file tree
Hide file tree
Showing 90 changed files with 5,392 additions and 4,952 deletions.
5 changes: 5 additions & 0 deletions errors/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ import (
"fmt"
)

// Is delegates to the stdlib errors function
func Is(err, target error) bool {
return errors.Is(err, target)
}

// As is a utility for one-lining errors.As statements.
// e.g. cerr, match := errors.As[MyCustomError](err).
func As[E error](err error) (e E, _ bool) {
Expand Down
1 change: 1 addition & 0 deletions go.work.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1100,6 +1100,7 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI=
github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
Expand Down
31 changes: 15 additions & 16 deletions internal/cmd/grpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import (
"sync"
"time"

"go.flipt.io/flipt/internal/server/ofrep"

otlpRuntime "go.opentelemetry.io/contrib/instrumentation/runtime"

"go.opentelemetry.io/contrib/propagators/autoprop"
Expand All @@ -17,7 +15,6 @@ import (
"go.flipt.io/flipt/internal/containers"
"go.flipt.io/flipt/internal/info"
"go.flipt.io/flipt/internal/metrics"
fliptserver "go.flipt.io/flipt/internal/server"
analytics "go.flipt.io/flipt/internal/server/analytics"
"go.flipt.io/flipt/internal/server/analytics/clickhouse"
"go.flipt.io/flipt/internal/server/analytics/prometheus"
Expand All @@ -26,12 +23,13 @@ import (
authzbundle "go.flipt.io/flipt/internal/server/authz/engine/bundle"
authzrego "go.flipt.io/flipt/internal/server/authz/engine/rego"
authzmiddlewaregrpc "go.flipt.io/flipt/internal/server/authz/middleware/grpc"
serverenvironments "go.flipt.io/flipt/internal/server/environments"
"go.flipt.io/flipt/internal/server/evaluation"
evaluationdata "go.flipt.io/flipt/internal/server/evaluation/data"
"go.flipt.io/flipt/internal/server/metadata"
middlewaregrpc "go.flipt.io/flipt/internal/server/middleware/grpc"
"go.flipt.io/flipt/internal/storage"
fsstore "go.flipt.io/flipt/internal/storage/fs/store"
"go.flipt.io/flipt/internal/server/ofrep"
"go.flipt.io/flipt/internal/storage/environments"
"go.flipt.io/flipt/internal/tracing"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/otel"
Expand Down Expand Up @@ -109,14 +107,13 @@ func NewGRPCServer(
return server.ln.Close()
})

var store storage.Store

store, err = fsstore.NewStore(ctx, logger, cfg)
// configure a declarative backend store
environmentStore, err := environments.NewStore(ctx, logger, cfg)
if err != nil {
return nil, err
}

logger.Debug("store enabled", zap.Stringer("store", store))
logger.Debug("store enabled")

// Initialize metrics exporter if enabled
if cfg.Metrics.Enabled {
Expand Down Expand Up @@ -193,14 +190,19 @@ func NewGRPCServer(
}

var (
fliptsrv = fliptserver.New(logger, store)
metasrv = metadata.New(cfg, info)
evalsrv = evaluation.New(logger, store)
evaldatasrv = evaluationdata.New(logger, store)
evalsrv = evaluation.New(logger, environmentStore)
evaldatasrv = evaluationdata.New(logger, environmentStore)
healthsrv = health.NewServer()
ofrepsrv = ofrep.New(logger, evalsrv, store)
// TODO(georgemac): wire back in legacy flipt compat services
ofrepsrv = ofrep.New(logger, evalsrv, nil)
)

envsrv, err := serverenvironments.NewServer(logger, environmentStore)
if err != nil {
return nil, fmt.Errorf("building configuration server: %w", err)
}

var (
// authnOpts is a slice of options that will be passed to the authentication service.
// it's initialized with the default option of skipping authentication for the health service which should never require authentication.
Expand All @@ -214,10 +216,8 @@ func NewGRPCServer(
}
)

skipAuthnIfExcluded(fliptsrv, cfg.Authentication.Exclude.Management)
skipAuthnIfExcluded(evalsrv, cfg.Authentication.Exclude.Evaluation)
skipAuthnIfExcluded(evaldatasrv, cfg.Authentication.Exclude.Evaluation)
skipAuthnIfExcluded(ofrepsrv, cfg.Authentication.Exclude.OFREP)

register, authInterceptors, authShutdown, err := authenticationGRPC(
ctx,
Expand Down Expand Up @@ -262,7 +262,6 @@ func NewGRPCServer(
}

// initialize servers
register.Add(fliptsrv)
register.Add(metasrv)
register.Add(evalsrv)
register.Add(evaldatasrv)
Expand Down
1 change: 1 addition & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type Config struct {
Server ServerConfig `json:"server,omitempty" mapstructure:"server" yaml:"server,omitempty"`
Environments EnvironmentsConfig `json:"environments,omitempty" mapstructure:"environments" yaml:"environments,omitempty"`
Storage StoragesConfig `json:"storage,omitempty" mapstructure:"storage" yaml:"storage,omitempty"`
Credentials CredentialsConfig `json:"credentials,omitempty" mapstructure:"credentials" yaml:"credentials,omitempty"`
Metrics MetricsConfig `json:"metrics,omitempty" mapstructure:"metrics" yaml:"metrics,omitempty"`
Tracing TracingConfig `json:"tracing,omitempty" mapstructure:"tracing" yaml:"tracing,omitempty"`
UI UIConfig `json:"ui,omitempty" mapstructure:"ui" yaml:"ui,omitempty"`
Expand Down
131 changes: 131 additions & 0 deletions internal/config/credentials.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package config

import (
"errors"
"fmt"

"github.com/spf13/viper"
)

type CredentialsConfig map[string]*CredentialConfig

func (c *CredentialsConfig) validate() error {
for name, v := range *c {
if err := v.validate(); err != nil {
return fmt.Errorf("credential %q: %w", name, err)
}
}

return nil
}

func (c *CredentialsConfig) setDefaults(v *viper.Viper) error {
credentials, ok := v.AllSettings()["credentials"].(map[string]any)
if !ok {
return nil
}

for name := range credentials {
getString := func(s string) string {
return v.GetString("credentials." + name + "." + s)
}

setDefault := func(k string, s any) {
v.SetDefault("credentials."+name+"."+k, s)
}

switch getString("type") {
case string(CredentialTypeSSH):
if getString("ssh.password") != "" ||
getString("ssh.private_key_path") != "" ||
getString("ssh.private_key_bytes") != "" {
setDefault("ssh.user", "git")
}
}
}

return nil
}

type CredentialType string

const (
CredentialTypeBasic = CredentialType("basic")
CredentialTypeSSH = CredentialType("ssh")
CredentialTypeAccessToken = CredentialType("access_token")
)

type CredentialConfig struct {
Type CredentialType `json:"type,omitempty" mapstructure:"type" yaml:"type,omitempty"`
Basic *BasicAuthConfig `json:"basic,omitempty" mapstructure:"basic" yaml:"basic,omitempty"`
SSH *SSHAuthConfig `json:"-" mapstructure:"ssh,omitempty" yaml:"-"`
AccessToken *string `json:"-" mapstructure:"access_token" yaml:"-"`
}

func (c *CredentialConfig) validate() error {
switch c.Type {
case CredentialTypeBasic:
if err := c.Basic.validate(); err != nil {
return err
}
case CredentialTypeSSH:
if err := c.SSH.validate(); err != nil {
return err
}
case CredentialTypeAccessToken:
if c.AccessToken == nil || *c.AccessToken == "" {
return errFieldRequired("", "access_token")
}
default:
return fmt.Errorf("unexpected credential type %q", c.Type)
}

return nil
}

// BasicAuthConfig has configuration for authenticating with private git repositories
// with basic auth.
type BasicAuthConfig struct {
Username string `json:"-" mapstructure:"username" yaml:"-"`
Password string `json:"-" mapstructure:"password" yaml:"-"`
}

func (b BasicAuthConfig) validate() error {
if (b.Username != "" && b.Password == "") || (b.Username == "" && b.Password != "") {
return errors.New("both username and password need to be provided for basic auth")
}

return nil
}

// SSHAuthConfig provides configuration support for SSH private key credentials when
// authenticating with private git repositories
type SSHAuthConfig struct {
User string `json:"-" mapstructure:"user" yaml:"-" `
Password string `json:"-" mapstructure:"password" yaml:"-" `
PrivateKeyBytes string `json:"-" mapstructure:"private_key_bytes" yaml:"-" `
PrivateKeyPath string `json:"-" mapstructure:"private_key_path" yaml:"-" `
InsecureIgnoreHostKey bool `json:"-" mapstructure:"insecure_ignore_host_key" yaml:"-"`
}

func (a *SSHAuthConfig) validate() (err error) {
defer func() {
if err != nil {
err = fmt.Errorf("ssh authentication: %w", err)
}
}()

if a == nil {
return errors.New("configuration is mising")
}

if a.Password == "" {
return errors.New("password required")
}

if (a.PrivateKeyBytes == "" && a.PrivateKeyPath == "") || (a.PrivateKeyBytes != "" && a.PrivateKeyPath != "") {
return errors.New("please provide exclusively one of private_key_bytes or private_key_path")
}

return nil
}
102 changes: 13 additions & 89 deletions internal/config/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,15 @@ type StorageBackendConfig struct {
// StorageConfig contains fields which will configure the type of backend in which Flipt will serve
// flag state.
type StorageConfig struct {
Remote string `json:"remote,omitempty" mapstructure:"remote" yaml:"remote,omitempty"`
Backend StorageBackendConfig `json:"backend,omitempty" mapstructure:"backend" yaml:"backend,omitempty"`
Branch string `json:"branch,omitempty" mapstructure:"branch" yaml:"branch,omitempty"`
CaCertBytes string `json:"-" mapstructure:"ca_cert_bytes" yaml:"-"`
CaCertPath string `json:"-" mapstructure:"ca_cert_path" yaml:"-"`
PollInterval time.Duration `json:"pollInterval,omitempty" mapstructure:"poll_interval" yaml:"poll_interval,omitempty"`
InsecureSkipTLS bool `json:"-" mapstructure:"insecure_skip_tls" yaml:"-"`
Authentication StorageGitAuthenticationConfig `json:"-" mapstructure:"authentication,omitempty" yaml:"-"`
Remote string `json:"remote,omitempty" mapstructure:"remote" yaml:"remote,omitempty"`
Backend StorageBackendConfig `json:"backend,omitempty" mapstructure:"backend" yaml:"backend,omitempty"`
Branch string `json:"branch,omitempty" mapstructure:"branch" yaml:"branch,omitempty"`
CaCertBytes string `json:"-" mapstructure:"ca_cert_bytes" yaml:"-"`
CaCertPath string `json:"-" mapstructure:"ca_cert_path" yaml:"-"`
PollInterval time.Duration `json:"pollInterval,omitempty" mapstructure:"poll_interval" yaml:"poll_interval,omitempty"`
InsecureSkipTLS bool `json:"-" mapstructure:"insecure_skip_tls" yaml:"-"`
Credentials string `json:"-" mapstructure:"credentials" yaml:"-"`
Signature SignatureConfig `json:"signature" mapstructure:"signature,omitempty" yaml:"signature"`
}

func (c *StorageConfig) validate() error {
Expand All @@ -104,88 +105,11 @@ func (c *StorageConfig) validate() error {
return errors.New("local path must be specified")
}

return c.Authentication.validate()
}

// StorageGitAuthenticationConfig holds structures for various types of auth we support.
// Token auth will take priority over Basic auth if both are provided.
//
// To make things easier, if there are multiple inputs that a particular auth method needs, and
// not all inputs are given but only partially, we will return a validation error.
// (e.g. if username for basic auth is given, and token is also given a validation error will be returned)
type StorageGitAuthenticationConfig struct {
BasicAuth *GitBasicAuth `json:"-" mapstructure:"basic,omitempty" yaml:"-"`
TokenAuth *GitTokenAuth `json:"-" mapstructure:"token,omitempty" yaml:"-"`
SSHAuth *GitSSHAuth `json:"-" mapstructure:"ssh,omitempty" yaml:"-"`
}

func (a *StorageGitAuthenticationConfig) validate() error {
if a.BasicAuth != nil {
if err := a.BasicAuth.validate(); err != nil {
return err
}
}
if a.TokenAuth != nil {
if err := a.TokenAuth.validate(); err != nil {
return err
}
}
if a.SSHAuth != nil {
if err := a.SSHAuth.validate(); err != nil {
return err
}
}

return nil
}

// GitBasicAuth has configuration for authenticating with private git repositories
// with basic auth.
type GitBasicAuth struct {
Username string `json:"-" mapstructure:"username" yaml:"-"`
Password string `json:"-" mapstructure:"password" yaml:"-"`
}

func (b *GitBasicAuth) validate() error {
if (b.Username != "" && b.Password == "") || (b.Username == "" && b.Password != "") {
return errors.New("both username and password need to be provided for basic auth")
}

return nil
}

// GitTokenAuth has configuration for authenticating with private git repositories
// with token auth.
type GitTokenAuth struct {
AccessToken string `json:"-" mapstructure:"access_token" yaml:"-"`
}

func (t GitTokenAuth) validate() error { return nil }

// GitSSHAuth provides configuration support for SSH private key credentials when
// authenticating with private git repositories
type GitSSHAuth struct {
User string `json:"-" mapstructure:"user" yaml:"-" `
Password string `json:"-" mapstructure:"password" yaml:"-" `
PrivateKeyBytes string `json:"-" mapstructure:"private_key_bytes" yaml:"-" `
PrivateKeyPath string `json:"-" mapstructure:"private_key_path" yaml:"-" `
InsecureIgnoreHostKey bool `json:"-" mapstructure:"insecure_ignore_host_key" yaml:"-"`
}

func (a *GitSSHAuth) validate() (err error) {
defer func() {
if err != nil {
err = fmt.Errorf("ssh authentication %w", err)
}
}()

if a.Password == "" {
return errors.New("password required")
}

if (a.PrivateKeyBytes == "" && a.PrivateKeyPath == "") || (a.PrivateKeyBytes != "" && a.PrivateKeyPath != "") {
return errors.New("please provide exclusively one of private_key_bytes or private_key_path")
}

return nil
// SignatureConfig contains details for producing git author and committer metadata.
type SignatureConfig struct {
Name string `json:"name" mapstructure:"name" yaml:"name"`
Email string `json:"email" mapstructure:"email" yaml:"email"`
}
Loading

0 comments on commit 90b12c2

Please sign in to comment.