diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml index 73434107be..b99f28c56e 100644 --- a/.github/workflows/lint_and_test.yaml +++ b/.github/workflows/lint_and_test.yaml @@ -38,7 +38,7 @@ jobs: - name: Unit test run: | PATH=$PATH:$(go env GOPATH)/bin make build - PATH=$PATH:$(go env GOPATH)/bin make test + PATH=$PATH:$(go env GOPATH)/bin make test-all lint: name: Lint diff --git a/.gitignore b/.gitignore index 2a9bec21b3..2bbcc71a08 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ bin/ go.mod.bak dist/ .cache_ggshield +.DS_Store diff --git a/Makefile b/Makefile index 78a0609f3b..da70c94a05 100644 --- a/Makefile +++ b/Makefile @@ -60,9 +60,14 @@ force-test: .PHONY: test test: - @echo "> Test" + @echo "> Run Unit Tests" @go test ./examples/lib/... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/demoplugin/... $(REPO_ROOT)/pkg/... +.PHONY: test-all +test-all: install-requirements + @echo "> Run All Tests" + @go test --tags=integration ./examples/lib/... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/demoplugin/... $(REPO_ROOT)/pkg/... + .PHONY: generate generate: @$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminstaller/... $(REPO_ROOT)/examples/... diff --git a/docs/reference/ocm_controller.md b/docs/reference/ocm_controller.md index 38c7c859a5..e74255bbaa 100644 --- a/docs/reference/ocm_controller.md +++ b/docs/reference/ocm_controller.md @@ -22,4 +22,5 @@ ocm controller [] ... ##### Sub Commands * [ocm controller install](ocm_controller_install.md) — Install either a specific or latest version of the ocm-controller. Optionally install prerequisites required by the controller. +* [ocm controller uninstall](ocm_controller_uninstall.md) — Uninstalls the ocm-controller and all of its dependencies diff --git a/docs/reference/ocm_controller_uninstall.md b/docs/reference/ocm_controller_uninstall.md new file mode 100644 index 0000000000..2d4c0f1734 --- /dev/null +++ b/docs/reference/ocm_controller_uninstall.md @@ -0,0 +1,32 @@ +## ocm controller uninstall — Uninstalls The Ocm-Controller And All Of Its Dependencies + +### Synopsis + +``` +ocm controller uninstall controller +``` + +### Options + +``` + -u, --base-url string the base url to the ocm-controller's release page (default "https://github.com/open-component-model/ocm-controller/releases") + --cert-manager-base-url string the base url to the cert-manager's release page (default "https://github.com/cert-manager/cert-manager/releases") + --cert-manager-release-api-url string the base url to the cert-manager's API release page (default "https://api.github.com/repos/cert-manager/cert-manager/releases") + --cert-manager-version string version for cert-manager (default "v1.13.2") + -c, --controller-name string name of the controller that's used for status check (default "ocm-controller") + -d, --dry-run if enabled, prints the downloaded manifest file + -h, --help help for uninstall + -n, --namespace string the namespace into which the controller is installed (default "ocm-system") + -a, --release-api-url string the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases") + -t, --timeout duration maximum time to wait for deployment to be ready (default 1m0s) + -p, --uninstall-prerequisites uninstall prerequisites required by ocm-controller + -v, --version string the version of the controller to install (default "latest") +``` + +### SEE ALSO + +##### Parents + +* [ocm controller](ocm_controller.md) — Commands acting on the ocm-controller +* [ocm](ocm.md) — Open Component Model command line client + diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index 998d942fda..fdf51d3363 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -128,7 +128,7 @@ The following credential consumer types are used/supported: - scheme: (optional) URL scheme - port: (optional) server port - namespace: vault namespace - - secretEngine: secret engine + - mountPath: mount path - pathprefix: path prefix for secret @@ -138,7 +138,6 @@ The following credential consumer types are used/supported: - token: vault token - roleid: applrole role id - secretid: applrole secret id - - secretid: applrole secret id The only supported auth methods, so far, are token and approle. @@ -315,7 +314,7 @@ The following types are currently available: - scheme: (optional) URL scheme - port: (optional) server port - namespace: vault namespace - - secretEngine: secret engine + - mountPath: mount path - pathprefix: path prefix for secret @@ -325,7 +324,6 @@ The following types are currently available: - token: vault token - roleid: applrole role id - secretid: applrole secret id - - secretid: applrole secret id The only supported auth methods, so far, are token and approle. @@ -335,7 +333,7 @@ The following types are currently available: The repository specification supports the following fields: - serverURL: *string* (required): the URL of the vault instance - namespace: *string* (optional): the namespace used to evaluate secrets - - secretsEngine: *string* (optional): the secrets engine to use (default: secrets) + - mountPath: *string* (optional): the mount path to use (default: secrets) - path: *string* (optional): the path prefix used to lookup secrets - secrets: *[]string* (optional): list of secrets - propagateConsumerIdentity: *bool*(optional): evaluate metadata for consumer id propagation diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index b33dcb9eb2..cf3d1cbc4d 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -54,7 +54,7 @@ Matchers exist for the following usage contexts or consumer types: - scheme: (optional) URL scheme - port: (optional) server port - namespace: vault namespace - - secretEngine: secret engine + - mountPath: mount path - pathprefix: path prefix for secret @@ -64,7 +64,6 @@ Matchers exist for the following usage contexts or consumer types: - token: vault token - roleid: applrole role id - secretid: applrole secret id - - secretid: applrole secret id The only supported auth methods, so far, are token and approle. diff --git a/examples/lib/tour/doc.go b/examples/lib/tour/doc.go index d20d8d6dd7..1ec6e0514f 100644 --- a/examples/lib/tour/doc.go +++ b/examples/lib/tour/doc.go @@ -1,3 +1,3 @@ -//go:generate mdref --headings --list docsrc . +//go:generate mdref --headings docsrc . package tour diff --git a/go.mod b/go.mod index 7f7515bba9..d0c13944d7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/open-component-model/ocm -go 1.22.1 +go 1.22.2 replace github.com/spf13/cobra => github.com/open-component-model/cobra v0.0.0-20230329075350-b1fd876abfb9 @@ -222,6 +222,7 @@ require ( github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hashicorp/vault/api v1.13.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/in-toto/in-toto-golang v0.9.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index decd37aade..de4bf148a7 100644 --- a/go.sum +++ b/go.sum @@ -591,8 +591,8 @@ github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31 github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc= github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY= -github.com/hashicorp/vault/api v1.12.2 h1:7YkCTE5Ni90TcmYHDBExdt4WGJxhpzaHqR6uGbQb/rE= -github.com/hashicorp/vault/api v1.12.2/go.mod h1:LSGf1NGT1BnvFFnKVtnvcaLBM2Lz+gJdpL6HUYed8KE= +github.com/hashicorp/vault/api v1.13.0 h1:RTCGpE2Rgkn9jyPcFlc7YmNocomda44k5ck8FKMH41Y= +github.com/hashicorp/vault/api v1.13.0/go.mod h1:0cb/uZUv1w2cVu9DIvuW1SMlXXC6qtATJt+LXJRx+kg= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= diff --git a/hack/Makefile b/hack/Makefile index 451cf5cc81..aeab22f85b 100644 --- a/hack/Makefile +++ b/hack/Makefile @@ -1,4 +1,8 @@ GOPATH := $(shell go env GOPATH) +LOCALBIN := $(shell pwd)/../bin +OS := $(shell go env GOOS 2>/dev/null || sh -c 'uname -o' | sed 's/.*/\L&/' ) +ARCH := $(shell go env GOARCH 2>/dev/null || sh -c 'uname -m' | sed 's/.*/\L&/' ) +OS_ARCH := $(OS)_$(ARCH) ifeq ($(OS),Windows_NT) detected_OS := Windows @@ -39,6 +43,16 @@ GO_BINDATA := $(shell (go-bindata -version 2>/dev/null || echo 0.0.0) | head -n ifneq ("v$(GO_BINDATA)",$(GO_BINDATA_VERSION)) deps += go-bindata endif +VAULT_VERSION := 1.16.2 +VAULT := $(shell ($(LOCALBIN)/vault --version 2>/dev/null || echo 0.0) | sed 's/.*Vault v\([0-9\.]*\).*/\1/') +ifeq ($(VAULT), $(VAULT_VERSION)) + deps += vault +endif +OCI_REGISTRY_VERSION := 3.0.0-alpha.1 +OCI_REGISTRY := $(shell (registry --version 2>/dev/null || echo 0.0) | sed 's/.* v\([0-9a-z\.\-]*\).*/\1/') +ifeq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION)) + deps += oci-registry +endif .PHONY: install-requirements install-requirements: $(deps) $(GOPATH)/bin/goimports mdref @@ -58,6 +72,21 @@ golangci-lint-version: go-bindata: go install -v github.com/go-bindata/go-bindata/v3/...@$(GO_BINDATA_VERSION) +.PHONY: vault +vault: + @if [ "$(VAULT)" != "$(VAULT_VERSION)" ]; then \ + curl -o $(LOCALBIN)/vault.zip https://releases.hashicorp.com/vault/$(VAULT_VERSION)/vault_$(VAULT_VERSION)_$(OS_ARCH).zip; \ + unzip -o $(LOCALBIN)/vault.zip -d $(LOCALBIN); \ + rm $(LOCALBIN)/vault.zip; \ + chmod a+x $(LOCALBIN)/vault;\ + fi + +.PHONY: oci-registry +oci-registry: + @if [ "$(OCI_REGISTRY)" != "$(OCI_REGISTRY_VERSION)" ]; then \ + go install -v github.com/distribution/distribution/v3/cmd/registry@v3.0.0-alpha.1; \ + fi + $(GOPATH)/bin/goimports: go install -v golang.org/x/tools/cmd/goimports@latest @@ -65,7 +94,6 @@ $(GOPATH)/bin/goimports: mdref: go install -v github.com/mandelsoft/mdref@master - Linux_jq: $(info -> jq is missing) $(info - sudo apt-get install jq / sudo dnf install jq / sudo zypper install jq / sudo pacman -S jq) diff --git a/pkg/contexts/credentials/cpi/interface.go b/pkg/contexts/credentials/cpi/interface.go index 982084eea9..2499903c1c 100644 --- a/pkg/contexts/credentials/cpi/interface.go +++ b/pkg/contexts/credentials/cpi/interface.go @@ -30,6 +30,7 @@ type ( GenericRepositorySpec = internal.GenericRepositorySpec GenericCredentialsSpec = internal.GenericCredentialsSpec DirectCredentials = internal.DirectCredentials + EvaluationContext = internal.EvaluationContext ) type ( @@ -102,6 +103,18 @@ func RequiredCredentialsForConsumer(ctx ContextProvider, id ConsumerIdentity, ma return internal.CredentialsForConsumer(ctx, id, true, matchers...) } +func GetCredentialsForConsumer(ctx Context, ectx EvaluationContext, identity ConsumerIdentity, matchers ...IdentityMatcher) (CredentialsSource, error) { + return internal.GetCredentialsForConsumer(ctx, ectx, identity, matchers...) +} + +func GetEvaluationContextFor[T any](ectx EvaluationContext) T { + return internal.GetEvaluationContextFor[T](ectx) +} + +func SetEvaluationContextFor(ectx EvaluationContext, e any) { + internal.SetEvaluationContextFor(ectx, e) +} + var ( CompleteMatch = internal.CompleteMatch NoMatch = internal.NoMatch diff --git a/pkg/contexts/credentials/internal/consumers.go b/pkg/contexts/credentials/internal/consumers.go index 145c43270a..f7ebfbc3bc 100644 --- a/pkg/contexts/credentials/internal/consumers.go +++ b/pkg/contexts/credentials/internal/consumers.go @@ -1,13 +1,33 @@ package internal import ( + "fmt" + "slices" "sort" "sync" + "github.com/mandelsoft/goutils/exception" + "github.com/mandelsoft/goutils/general" "github.com/mandelsoft/goutils/maputils" + "github.com/mandelsoft/goutils/sliceutils" + "github.com/mandelsoft/goutils/stringutils" ) -// UsageContext descibes a dediacetd type specific +type CredentialRecursion []ConsumerIdentity + +func (c CredentialRecursion) String() string { + return stringutils.Join(c) +} + +func (c CredentialRecursion) Contains(identity ConsumerIdentity) bool { + return slices.ContainsFunc(c, general.ContainsFuncFor(identity)) +} + +func (c CredentialRecursion) Append(identity ConsumerIdentity) CredentialRecursion { + return sliceutils.CopyAppendUniqueFunc(c, general.EqualsFuncFor[ConsumerIdentity](), identity) +} + +// UsageContext describes a dedicated type specific // sub usage kinds for an object requiring credentials. // For example, for an object providing a hierarchical // namespace this might be a namespace prefix for @@ -73,7 +93,7 @@ func (c *_consumers) Get(id ConsumerIdentity) (CredentialsSource, bool) { // Match matches a given request (pattern) against configured // identities. -func (c *_consumers) Match(pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) { +func (c *_consumers) Match(ectx EvaluationContext, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) { var found *_consumer for _, s := range c.data { if m(pattern, cur, s.identity) { @@ -195,18 +215,72 @@ func (p *consumerProviderRegistry) Get(id ConsumerIdentity) (CredentialsSource, return nil, false } -func (p *consumerProviderRegistry) Match(pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) { - p.lock.Lock() - defer p.lock.Unlock() +func (p *consumerProviderRegistry) checkHandleProvider(ectx EvaluationContext, prov ConsumerProvider, pattern ConsumerIdentity) (rctx EvaluationContext, useprov bool, usestack bool) { + if pr, ok := prov.(ConsumerIdentityProvider); ok { + r := GetEvaluationContextFor[CredentialRecursion](ectx) + if r == nil { + r = CredentialRecursion{} + } + if r.Contains(pr.GetConsumerId()) { + return ectx, false, true + } + r = r.Append(pr.GetConsumerId()) + ectx = SetEvaluationContextFor(ectx, r) + } + return ectx, true, true +} - credsrc, cur := p.explicit.Match(pattern, cur, m) +type UnwindStack struct { + error +} + +func (u *UnwindStack) Unwrap() error { + return u.error +} + +func (p *consumerProviderRegistry) catchedMatch(ectx EvaluationContext, sub ConsumerProvider, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (cs CredentialsSource, ci ConsumerIdentity) { + defer exception.CatchError(func(err error) { + log.Trace("caught unwind stack error: {{error}}", "error", err) + cs = nil + ci = cur + }, exception.ByPrototypes(&UnwindStack{})) + log.Trace("pattern: {{pattern}}\ncontext: {{context}}\nprovider: {{provider}}", + "pattern", pattern, "context", ectx, "provider", sub) + ectx, useprov, _ := p.checkHandleProvider(ectx, sub, pattern) + if !useprov { + return nil, cur + } + log.Trace("attempt match with provider: {{provider}}", "provider", sub) + return sub.Match(ectx, pattern, cur, m) +} + +func (p *consumerProviderRegistry) Match(ectx EvaluationContext, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) { + p.lock.RLock() + defer p.lock.RUnlock() + + credsrc, cur := p.explicit.Match(ectx, pattern, cur, m) for _, sub := range p.providers { var f CredentialsSource - f, cur = sub.Match(pattern, cur, m) + f, cur = p.catchedMatch(ectx, sub, pattern, cur, m) if f != nil { credsrc = f } } + // If this is the case, we are in a situation where we have excluded all providers (since they are all in the stack). + // If we would simply return with no credentials, the follow-up coding would assume, that it should query the + // credential repository without any credentials, since none have been found. + // INSTEAD, we have to step down to the previous recursion level and check the other potentially available providers + // for credentials. + // BUT in case we have explicit credentials, then we should use those. + if credsrc == nil { + r := GetEvaluationContextFor[CredentialRecursion](ectx) + // unwind the stack only makes sense when we are in a recursive call, thus we have at least one provider on the + // credential recursion stack + if len(r) > 0 && len(r) == len(p.providers) { + exception.Throw(&UnwindStack{fmt.Errorf("impossible credential recursion detected - unwind stack")}) + } + } + log.Trace("return credential source") return credsrc, cur } diff --git a/pkg/contexts/credentials/internal/context.go b/pkg/contexts/credentials/internal/context.go index 9976029323..09c6e297e4 100644 --- a/pkg/contexts/credentials/internal/context.go +++ b/pkg/contexts/credentials/internal/context.go @@ -2,9 +2,13 @@ package internal import ( "context" + "fmt" "reflect" "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/generics" + "github.com/mandelsoft/goutils/maputils" + "golang.org/x/exp/maps" "github.com/open-component-model/ocm/pkg/contexts/config" cfgcpi "github.com/open-component-model/ocm/pkg/contexts/config/cpi" @@ -30,7 +34,36 @@ type ContextProvider interface { type ConsumerProvider interface { Unregister(id ProviderIdentity) Get(id ConsumerIdentity) (CredentialsSource, bool) - Match(id ConsumerIdentity, cur ConsumerIdentity, matcher IdentityMatcher) (CredentialsSource, ConsumerIdentity) + Match(ectx EvaluationContext, id ConsumerIdentity, cur ConsumerIdentity, matcher IdentityMatcher) (CredentialsSource, ConsumerIdentity) +} + +type EvaluationContext *evaluationContext + +type evaluationContext struct { + data map[reflect.Type]interface{} +} + +func (e evaluationContext) String() string { + return fmt.Sprintf("%v", maputils.Transform(e.data, func(k reflect.Type, v interface{}) (string, string) { + return k.Name(), fmt.Sprintf("%v", v) + })) +} + +func GetEvaluationContextFor[T any](ectx EvaluationContext) T { + var _nil T + if ectx.data == nil { + return _nil + } + return generics.Cast[T](ectx.data[generics.TypeOf[T]()]) +} + +func SetEvaluationContextFor(ectx EvaluationContext, e any) EvaluationContext { + if ectx.data == nil { + ectx.data = map[reflect.Type]interface{}{} + } + n := &evaluationContext{maps.Clone(ectx.data)} + n.data[reflect.TypeOf(e)] = e + return n } type Context interface { @@ -53,6 +86,7 @@ type Context interface { UnregisterConsumerProvider(id ProviderIdentity) GetCredentialsForConsumer(ConsumerIdentity, ...IdentityMatcher) (CredentialsSource, error) + getCredentialsForConsumer(EvaluationContext, ConsumerIdentity, ...IdentityMatcher) (CredentialsSource, error) SetCredentialsForConsumer(identity ConsumerIdentity, creds CredentialsSource) SetCredentialsForConsumerWithProvider(pid ProviderIdentity, identity ConsumerIdentity, creds CredentialsSource) @@ -208,17 +242,24 @@ func (c *_context) CredentialsForConfig(data []byte, unmarshaler runtime.Unmarsh var emptyIdentity = ConsumerIdentity{} func (c *_context) GetCredentialsForConsumer(identity ConsumerIdentity, matchers ...IdentityMatcher) (CredentialsSource, error) { + return c.getCredentialsForConsumer(nil, identity, matchers...) +} + +func (c *_context) getCredentialsForConsumer(ectx EvaluationContext, identity ConsumerIdentity, matchers ...IdentityMatcher) (CredentialsSource, error) { err := c.Update() if err != nil { return nil, err } + if ectx == nil { + ectx = &evaluationContext{} + } m := c.defaultMatcher(identity, matchers...) var credsrc CredentialsSource if m == nil { credsrc, _ = c.consumerProviders.Get(identity) } else { - credsrc, _ = c.consumerProviders.Match(identity, nil, m) + credsrc, _ = c.consumerProviders.Match(ectx, identity, nil, m) } if credsrc == nil { credsrc, _ = c.consumerProviders.Get(emptyIdentity) @@ -270,3 +311,12 @@ func (c *_context) RegisterConsumerProvider(id ProviderIdentity, provider Consum func (c *_context) UnregisterConsumerProvider(id ProviderIdentity) { c.consumerProviders.Unregister(id) } + +/////////////////////////////////////// + +func GetCredentialsForConsumer(ctx Context, ectx EvaluationContext, identity ConsumerIdentity, matchers ...IdentityMatcher) (CredentialsSource, error) { + if ectx == nil { + ectx = &evaluationContext{} + } + return ctx.getCredentialsForConsumer(ectx, identity, matchers...) +} diff --git a/pkg/contexts/credentials/internal/identity.go b/pkg/contexts/credentials/internal/identity.go index 57b8c5f90f..36cf4f36cb 100644 --- a/pkg/contexts/credentials/internal/identity.go +++ b/pkg/contexts/credentials/internal/identity.go @@ -103,6 +103,10 @@ func NewConsumerIdentity(typ string, attrs ...string) ConsumerIdentity { return r } +func ConsumerIdentityEqual(a, b ConsumerIdentity) bool { + return a.Equals(b) +} + // IsSet checks whether an identity is given. func (i ConsumerIdentity) IsSet() bool { return len(i) != 0 diff --git a/pkg/contexts/credentials/internal/logging.go b/pkg/contexts/credentials/internal/logging.go new file mode 100644 index 0000000000..c80cfbbdc8 --- /dev/null +++ b/pkg/contexts/credentials/internal/logging.go @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + ocmlog "github.com/open-component-model/ocm/pkg/logging" +) + +var ( + REALM = ocmlog.DefineSubRealm("Credentials", "credentials") + log = ocmlog.DynamicLogger(REALM) +) diff --git a/pkg/contexts/credentials/repositories/dockerconfig/provider.go b/pkg/contexts/credentials/repositories/dockerconfig/provider.go index 88335fc3e9..1ac017391e 100644 --- a/pkg/contexts/credentials/repositories/dockerconfig/provider.go +++ b/pkg/contexts/credentials/repositories/dockerconfig/provider.go @@ -20,7 +20,7 @@ var _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) func (p *ConsumerProvider) Unregister(id cpi.ProviderIdentity) { } -func (p *ConsumerProvider) Match(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { +func (p *ConsumerProvider) Match(ectx cpi.EvaluationContext, req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { return p.get(req, cur, m) } diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index 010dd12ac5..2268ff5faf 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -15,7 +15,7 @@ var _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) func (p *ConsumerProvider) Unregister(_ cpi.ProviderIdentity) { } -func (p *ConsumerProvider) Match(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { +func (p *ConsumerProvider) Match(ectx cpi.EvaluationContext, req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { return p.get(req, cur, m) } diff --git a/pkg/contexts/credentials/repositories/vault/a_usage.go b/pkg/contexts/credentials/repositories/vault/a_usage.go index 2407385a5c..5c165c063a 100644 --- a/pkg/contexts/credentials/repositories/vault/a_usage.go +++ b/pkg/contexts/credentials/repositories/vault/a_usage.go @@ -46,7 +46,7 @@ The repository specification supports the following fields: ` + listformat.FormatListElements("", listformat.StringElementDescriptionList{ "serverURL", "*string* (required): the URL of the vault instance", "namespace", "*string* (optional): the namespace used to evaluate secrets", - "secretsEngine", "*string* (optional): the secrets engine to use (default: secrets)", + "mountPath", "*string* (optional): the mount path to use (default: secrets)", "path", "*string* (optional): the path prefix used to lookup secrets", "secrets", "*[]string* (optional): list of secrets", "propagateConsumerIdentity", "*bool*(optional): evaluate metadata for consumer id propagation", diff --git a/pkg/contexts/credentials/repositories/vault/identity/identity.go b/pkg/contexts/credentials/repositories/vault/identity/identity.go index fc01e52d7d..4cae7c00f0 100644 --- a/pkg/contexts/credentials/repositories/vault/identity/identity.go +++ b/pkg/contexts/credentials/repositories/vault/identity/identity.go @@ -16,12 +16,12 @@ const CONSUMER_TYPE = "HashiCorpVault" // identity properties. const ( - ID_HOSTNAME = hostpath.ID_HOSTNAME - ID_SCHEMA = hostpath.ID_SCHEME - ID_PORT = hostpath.ID_PORT - ID_PATHPREFIX = hostpath.ID_PATHPREFIX - ID_SECRETENGINE = "secretEngine" - ID_NAMESPACE = "namespace" + ID_HOSTNAME = hostpath.ID_HOSTNAME + ID_SCHEMA = hostpath.ID_SCHEME + ID_PORT = hostpath.ID_PORT + ID_PATHPREFIX = hostpath.ID_PATHPREFIX + ID_MOUNTPATH = "mountPath" + ID_NAMESPACE = "namespace" ) // credential properties. @@ -43,7 +43,7 @@ func IdentityMatcher(request, cur, id cpi.ConsumerIdentity) bool { if id[ID_NAMESPACE] != request[ID_NAMESPACE] { return false } - if id[ID_SECRETENGINE] != "" && id[ID_SECRETENGINE] != request[ID_SECRETENGINE] { + if id[ID_MOUNTPATH] != "" && id[ID_MOUNTPATH] != request[ID_MOUNTPATH] { return false } return identityMatcher(request, cur, id) @@ -55,14 +55,13 @@ func init() { ATTR_TOKEN, "vault token", ATTR_ROLEID, "applrole role id", ATTR_SECRETID, "applrole secret id", - ATTR_SECRETID, "applrole secret id", }) ids := listformat.FormatListElements("", listformat.StringElementDescriptionList{ ID_HOSTNAME, "vault server host", ID_SCHEMA, "(optional) URL scheme", ID_PORT, "(optional) server port", ID_NAMESPACE, "vault namespace", - ID_SECRETENGINE, "secret engine", + ID_MOUNTPATH, "mount path", ID_PATHPREFIX, "path prefix for secret", }) cpi.RegisterStandardIdentity(CONSUMER_TYPE, identityMatcher, @@ -76,7 +75,7 @@ The only supported auth methods, so far, are token and approl `) } -func GetConsumerId(serverurl string, namespace string, secretengine string, secretpath string) (cpi.ConsumerIdentity, error) { +func GetConsumerId(serverurl string, namespace string, mountpath string, secretpath string) (cpi.ConsumerIdentity, error) { if serverurl == "" { return nil, errors.Newf("server address must be given") } @@ -106,8 +105,8 @@ func GetConsumerId(serverurl string, namespace string, secretengine string, secr if namespace != "" { id[ID_NAMESPACE] = namespace } - if secretengine != "" { - id[ID_SECRETENGINE] = secretengine + if mountpath != "" { + id[ID_MOUNTPATH] = mountpath } if secretpath != "" { @@ -116,8 +115,8 @@ func GetConsumerId(serverurl string, namespace string, secretengine string, secr return id, nil } -func GetCredentials(ctx cpi.ContextProvider, serverurl, namespace string, secretengine, secretpath string) (cpi.Credentials, error) { - id, err := GetConsumerId(serverurl, namespace, secretengine, secretpath) +func GetCredentials(ctx cpi.ContextProvider, serverurl, namespace string, mountpath, secretpath string) (cpi.Credentials, error) { + id, err := GetConsumerId(serverurl, namespace, mountpath, secretpath) if err != nil { return nil, err } diff --git a/pkg/contexts/credentials/repositories/vault/options.go b/pkg/contexts/credentials/repositories/vault/options.go index aeba1f6b3b..e78cf0dfac 100644 --- a/pkg/contexts/credentials/repositories/vault/options.go +++ b/pkg/contexts/credentials/repositories/vault/options.go @@ -11,7 +11,7 @@ type Option = optionutils.Option[*Options] type Options struct { Namespace string `json:"namespace,omitempty"` - SecretsEngine string `json:"secretsEngine,omitempty"` + MountPath string `json:"mountPath,omitempty"` Path string `json:"path,omitempty"` Secrets []string `json:"secrets,omitempty"` PropgateConsumerIdentity bool `json:"propagateConsumerIdentity,omitempty"` @@ -23,8 +23,8 @@ func (o *Options) ApplyTo(opts *Options) { if o.Namespace != "" { opts.Namespace = o.Namespace } - if o.SecretsEngine != "" { - opts.SecretsEngine = o.SecretsEngine + if o.MountPath != "" { + opts.MountPath = o.MountPath } if o.Path != "" { opts.Path = o.Path @@ -49,14 +49,14 @@ func WithNamespace(s string) Option { //////////////////////////////////////////////////////////////////////////////// -type se string +type m string -func (o se) ApplyTo(opts *Options) { - opts.SecretsEngine = string(o) +func (o m) ApplyTo(opts *Options) { + opts.MountPath = string(o) } -func WithSecretsEngine(s string) Option { - return se(s) +func WithMountPath(s string) Option { + return m(s) } //////////////////////////////////////////////////////////////////////////////// diff --git a/pkg/contexts/credentials/repositories/vault/provider.go b/pkg/contexts/credentials/repositories/vault/provider.go index fefc3c34f2..12d1a13d1e 100644 --- a/pkg/contexts/credentials/repositories/vault/provider.go +++ b/pkg/contexts/credentials/repositories/vault/provider.go @@ -3,6 +3,7 @@ package vault import ( "context" "encoding/json" + "net/http" "path" "strings" "sync" @@ -14,6 +15,7 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/credentials/internal" "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault/identity" ) @@ -29,17 +31,31 @@ type mapping struct { Name string } -type ConsumerProvider struct { - lock sync.Mutex - credentials map[string]cpi.DirectCredentials - repository *Repository +type credentialCache struct { creds cpi.CredentialsSource + credentials map[string]cpi.DirectCredentials consumer []*mapping +} + +func newCredentialCache(creds cpi.CredentialsSource) *credentialCache { + return &credentialCache{ + creds: creds, + credentials: map[string]cpi.DirectCredentials{}, + } +} + +type ConsumerProvider struct { + lock sync.Mutex + repository *Repository + cache *credentialCache updated bool } -var _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) +var ( + _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) + _ cpi.ConsumerIdentityProvider = (*ConsumerProvider)(nil) +) func NewConsumerProvider(repo *Repository) (*ConsumerProvider, error) { src, err := repo.ctx.GetCredentialsForConsumer(repo.id) @@ -47,25 +63,32 @@ func NewConsumerProvider(repo *Repository) (*ConsumerProvider, error) { return nil, err } return &ConsumerProvider{ - creds: src, - repository: repo, - credentials: map[string]cpi.DirectCredentials{}, + cache: newCredentialCache(src), + repository: repo, }, nil } -func (p *ConsumerProvider) update() error { - var err error +func (p *ConsumerProvider) String() string { + return p.repository.id.String() +} + +func (p *ConsumerProvider) GetConsumerId(uctx ...internal.UsageContext) internal.ConsumerIdentity { + return p.repository.GetConsumerId() +} + +func (p *ConsumerProvider) GetIdentityMatcher() string { + return p.repository.GetIdentityMatcher() +} +func (p *ConsumerProvider) update(ectx cpi.EvaluationContext) error { if p.updated { return nil } - p.updated = true - - p.creds, err = p.repository.ctx.GetCredentialsForConsumer(p.repository.id, identity.IdentityMatcher) + credsrc, err := cpi.GetCredentialsForConsumer(p.repository.ctx, ectx, p.repository.id, identity.IdentityMatcher) if err != nil { return err } - creds, err := p.creds.Credentials(p.repository.ctx) + creds, err := credsrc.Credentials(p.repository.ctx) if err != nil { return err } @@ -74,9 +97,6 @@ func (p *ConsumerProvider) update() error { return err } - p.credentials = map[string]cpi.DirectCredentials{} - p.consumer = nil - ctx := context.Background() client, err := vault.New( @@ -100,12 +120,15 @@ func (p *ConsumerProvider) update() error { return err } + cache := newCredentialCache(credsrc) + // TODO: support for pure path based access for other secret engine types secrets := slices.Clone(p.repository.spec.Secrets) if len(secrets) == 0 { s, err := client.Secrets.KvV2List(ctx, p.repository.spec.Path, - vault.WithMountPath(p.repository.spec.SecretsEngine)) + vault.WithMountPath(p.repository.spec.MountPath)) if err != nil { + p.error(err, "error listing secrets", "") return err } for _, k := range s.Data.Keys { @@ -125,16 +148,18 @@ func (p *ConsumerProvider) update() error { } } if len(id) > 0 { - p.consumer = append(p.consumer, &mapping{ + cache.consumer = append(cache.consumer, &mapping{ Id: cpi.ConsumerIdentity(id), Name: n, }) } if len(creds) > 0 { - p.credentials[n] = cpi.DirectCredentials(creds) + cache.credentials[n] = cpi.DirectCredentials(creds) } } } + p.cache = cache + p.updated = true return nil } @@ -159,10 +184,15 @@ func (p *ConsumerProvider) error(err error, msg string, secret string, keypairs if err == nil { return } - log.Error(msg, append(keypairs, + f := log.Info + var v *vault.ResponseError + if errors.As(err, &v) && v.StatusCode != http.StatusNotFound { + f = log.Error + } + f(msg, append(keypairs, "server", p.repository.spec.ServerURL, "namespace", p.repository.spec.Namespace, - "engine", p.repository.spec.SecretsEngine, + "engine", p.repository.spec.MountPath, "path", path.Join(p.repository.spec.Path, secret), "error", err.Error(), )..., @@ -174,7 +204,7 @@ func (p *ConsumerProvider) read(ctx context.Context, client *vault.Client, secre secret = path.Join(p.repository.spec.Path, secret) s, err := client.Secrets.KvV2Read(ctx, secret, - vault.WithMountPath(p.repository.spec.SecretsEngine)) + vault.WithMountPath(p.repository.spec.MountPath)) if err != nil { return nil, nil, nil, err } @@ -226,16 +256,16 @@ func getProps(data map[string]interface{}) common.Properties { func (p *ConsumerProvider) Unregister(id cpi.ProviderIdentity) { } -func (p *ConsumerProvider) Match(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { - return p.get(req, cur, m) +func (p *ConsumerProvider) Match(ectx cpi.EvaluationContext, req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { + return p.get(ectx, req, cur, m) } func (p *ConsumerProvider) Get(req cpi.ConsumerIdentity) (cpi.CredentialsSource, bool) { - creds, _ := p.get(req, nil, cpi.CompleteMatch) + creds, _ := p.get(nil, req, nil, cpi.CompleteMatch) return creds, creds != nil } -func (p *ConsumerProvider) get(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { +func (p *ConsumerProvider) get(ectx cpi.EvaluationContext, req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { if req.Equals(p.repository.id) { return nil, cur } @@ -243,13 +273,17 @@ func (p *ConsumerProvider) get(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentit p.lock.Lock() defer p.lock.Unlock() - p.update() + err := p.update(ectx) + if err != nil { + log.Info("error accessing credentials provider", "error", err) + } + var creds cpi.CredentialsSource - for _, a := range p.consumer { + for _, a := range p.cache.consumer { if m(req, cur, a.Id) { cur = a.Id - creds = p.credentials[a.Name] + creds = p.cache.credentials[a.Name] } } return creds, cur @@ -262,11 +296,11 @@ func (c *ConsumerProvider) ExistsCredentials(name string) (bool, error) { c.lock.Lock() defer c.lock.Unlock() - err := c.update() + err := c.update(nil) if err != nil { return false, err } - _, ok := c.credentials[name] + _, ok := c.cache.credentials[name] return ok, nil } @@ -274,11 +308,11 @@ func (c *ConsumerProvider) LookupCredentials(name string) (cpi.Credentials, erro c.lock.Lock() defer c.lock.Unlock() - err := c.update() + err := c.update(nil) if err != nil { return nil, err } - src, ok := c.credentials[name] + src, ok := c.cache.credentials[name] if ok { return src, nil } diff --git a/pkg/contexts/credentials/repositories/vault/repo_int_test.go b/pkg/contexts/credentials/repositories/vault/repo_int_test.go new file mode 100644 index 0000000000..49105706dd --- /dev/null +++ b/pkg/contexts/credentials/repositories/vault/repo_int_test.go @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package vault_test + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/hashicorp/vault-client-go" + "github.com/hashicorp/vault-client-go/schema" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials" + "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + me "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault" + "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault/identity" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/runtime" +) + +type vaultMode string + +const ( + HTTP vaultMode = "dev" + HTTPS vaultMode = "dev-tls" +) + +const ( + VAULT_APP_ROLE = "ocmrole" + VAULT_APP_ROLE1 = "ocmrole1" + VAULT_SECRET = "mysecret" + VAULT_CUSTOM_SECRETS = "secret-list" + VAULT_SECRET_2 = "mysecret2" + + VAULT_POLICY_NAME = "ocm" + VAULT_POLICY_NAME1 = "ocm1" + + VAULT_ROOT_TOKEN = "toorl" + VAULT_TLS_DIR = "./vault-tls" +) + +const ( + VAULT_POLICY_RULE = ` +path "secret/*" +{ + capabilities = ["read","list"] +} +` + VAULT_INSUFFICIENT_POLICY_RULE = ` +path "secret/notmysecret" +{ + capabilities = ["read", "list"] +} +` +) + +var _ = Describe("vault config", func() { + var DefaultContext credentials.Context + var cancelFunc context.CancelFunc + var vaultClient *vault.Client + var cmd *exec.Cmd + + ctx := context.Background() + + BeforeEach(func() { + cmd, vaultClient, cancelFunc = Must3(StartVaultServer(HTTP, VAULT_ROOT_TOKEN, VAULT_ADDRESS)) + DefaultContext = credentials.New() + }) + + AfterEach(func() { + cancelFunc() + _ = cmd.Wait() + Expect(os.RemoveAll(VAULT_TLS_DIR)).To(Succeed()) + }) + + Context("authentication to vault and reading secrets", func() { + + spec := me.NewRepositorySpec("http://"+VAULT_ADDRESS, me.WithPath(VAULT_PATH_REPO1), me.WithMountPath("secret")) + spec1 := me.NewRepositorySpec("http://"+VAULT_ADDRESS, me.WithPath(VAULT_PATH_REPO2), me.WithMountPath("secret")) + + It("authenticate with token and retrieve credentials", func() { + data := map[string]any{ + "password1": "ocm-password-1", + "password2": "ocm-password-2", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, + VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + + consumerId := Must(identity.GetConsumerId(vaultClient.Configuration().Address, + "", "secret", VAULT_PATH_REPO1)) + creds := credentials.NewCredentials(common.Properties{ + identity.ATTR_AUTHMETH: identity.AUTH_TOKEN, + identity.ATTR_TOKEN: VAULT_ROOT_TOKEN, + }) + DefaultContext.SetCredentialsForConsumer(consumerId, creds) + + repo := Must(DefaultContext.RepositoryForSpec(spec, nil)) + Expect(repo).ToNot(BeNil()) + + c, err := repo.LookupCredentials(VAULT_SECRET) + Expect(c.Properties()).To(YAMLEqual(data)) + Expect(err).To(BeNil()) + }) + + It("authenticate with approle and retrieve credentials", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_POLICY_RULE) + + data := map[string]any{ + "password1": "ocm-password-1", + "password2": "ocm-password-2", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + + repo := Must(DefaultContext.RepositoryForSpec(spec, nil)) + Expect(repo).ToNot(BeNil()) + + c, err := repo.LookupCredentials(VAULT_SECRET) + Expect(c.Properties()).To(YAMLEqual(data)) + Expect(err).To(BeNil()) + }) + + It("authenticate with approle with unsufficient authorizations and fail to retrieve credentials", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_INSUFFICIENT_POLICY_RULE) + + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, schema.KvV2WriteRequest{ + Data: map[string]any{ + "password1": "ocm-password-1", + "password2": "ocm-password-2", + }}, + vault.WithMountPath("secret"), + )) + + repo := Must(DefaultContext.RepositoryForSpec(spec, nil)) + Expect(repo).ToNot(BeNil()) + + c, err := repo.LookupCredentials(VAULT_SECRET) + Expect(err).To(HaveOccurred()) + Expect(c).To(BeNil()) + }) + + It("authenticate with approle and specify a subset of secrets at the specified path in the repository spec", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_POLICY_RULE) + + data := map[string]any{ + "password1": "ocm-password-1", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET_2, schema.KvV2WriteRequest{ + Data: map[string]any{ + "password2": "ocm-password-2", + }}, + vault.WithMountPath("secret"), + )) + + // This is how we restrict the secrets accessible through the respository + spec.Secrets = append(spec.Secrets, VAULT_SECRET) + repo := Must(DefaultContext.RepositoryForSpec(spec, nil)) + Expect(repo).ToNot(BeNil()) + + c, err := repo.LookupCredentials(VAULT_SECRET) + Expect(c).To(YAMLEqual(data)) + Expect(err).ToNot(HaveOccurred()) + + c, err = repo.LookupCredentials(VAULT_SECRET_2) + Expect(err).To(BeNil()) + Expect(c).To(BeNil()) + }) + + It("authenticate with approle and specify a subset of secrets at the specified path in a dedicated secret", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_POLICY_RULE) + + data := map[string]any{ + "password1": "ocm-password-1", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET_2, schema.KvV2WriteRequest{ + Data: map[string]any{ + "password2": "ocm-password-2", + }}, + vault.WithMountPath("secret"), + )) + + // You have to specify a value, but it is essentially a placeholder here + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_CUSTOM_SECRETS, schema.KvV2WriteRequest{ + Data: map[string]any{ + "description": "specify a list in the metadata", + }, + }, + vault.WithMountPath("secret"), + )) + metadata := map[string]any{ + me.CUSTOM_SECRETS: VAULT_SECRET, + } + _ = Must(vaultClient.Secrets.KvV2WriteMetadata(ctx, VAULT_PATH_REPO1+"/"+VAULT_CUSTOM_SECRETS, + schema.KvV2WriteMetadataRequest{CustomMetadata: metadata}, + vault.WithMountPath("secret"), + )) + + // This is how we restrict the secrets accessible through the respository + spec.Secrets = append(spec.Secrets, VAULT_CUSTOM_SECRETS) + repo := Must(DefaultContext.RepositoryForSpec(spec, nil)) + Expect(repo).ToNot(BeNil()) + + c, err := repo.LookupCredentials(VAULT_SECRET) + Expect(c).To(YAMLEqual(data)) + Expect(err).ToNot(HaveOccurred()) + + c, err = repo.LookupCredentials(VAULT_SECRET_2) + Expect(err).To(BeNil()) + Expect(c).To(BeNil()) + }) + + It("authenticate with approle and consume secrets with a consumer id from the provider", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_POLICY_RULE) + + data := map[string]any{ + "password1": "ocm-password-1", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + cid := hostpath.GetConsumerIdentity(hostpath.IDENTITY_TYPE, "https://test-url.com") + cidData := Must(runtime.DefaultJSONEncoding.Marshal(cid)) + metadata := map[string]any{ + me.CUSTOM_CONSUMERID: string(cidData), + } + _ = Must(vaultClient.Secrets.KvV2WriteMetadata(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteMetadataRequest{CustomMetadata: metadata}, + vault.WithMountPath("secret"), + )) + + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET_2, schema.KvV2WriteRequest{ + Data: map[string]any{ + "password2": "ocm-password-2", + }}, + vault.WithMountPath("secret"), + )) + + repo := Must(me.NewRepository(DefaultContext, spec)) + Expect(repo).ToNot(BeNil()) + provider := Must(me.NewConsumerProvider(repo)) + c, ok := provider.Get(cid) + Expect(ok).To(BeTrue()) + Expect(c).ToNot(BeNil()) + }) + + It("authenticate with approle and consume secrets with a consumer id from the credential context", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, VAULT_POLICY_RULE) + + data := map[string]any{ + "password1": "ocm-password-1", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + cid := hostpath.GetConsumerIdentity(hostpath.IDENTITY_TYPE, "https://test-url.com") + cidData := Must(runtime.DefaultJSONEncoding.Marshal(cid)) + metadata := map[string]any{ + me.CUSTOM_CONSUMERID: string(cidData), + } + _ = Must(vaultClient.Secrets.KvV2WriteMetadata(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteMetadataRequest{CustomMetadata: metadata}, + vault.WithMountPath("secret"), + )) + + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET_2, schema.KvV2WriteRequest{ + Data: map[string]any{ + "password2": "ocm-password-2", + }}, + vault.WithMountPath("secret"), + )) + + spec.PropgateConsumerIdentity = true + repo := Must(DefaultContext.RepositoryForSpec(spec)) + Expect(repo).ToNot(BeNil()) + + c := Must(DefaultContext.GetCredentialsForConsumer(cid)) + Expect(c).To(YAMLEqual(data)) + }) + + It("recursive authentication", func() { + SetUpVaultAccess(ctx, DefaultContext, vaultClient, fmt.Sprintf(` +path "secret/data/%s/*" +{ + capabilities = ["read"] +} +path "secret/metadata/%s/*" +{ + capabilities = ["list"] +} +`, VAULT_PATH_REPO1, VAULT_PATH_REPO1)) + + _ = Must(vaultClient.System.PoliciesWriteAclPolicy(ctx, VAULT_POLICY_NAME1, schema.PoliciesWriteAclPolicyRequest{Policy: fmt.Sprintf(` +path "secret/data/%s/*" +{ + capabilities = ["read"] +} +path "secret/metadata/%s/*" +{ + capabilities = ["list"] +} +`, VAULT_PATH_REPO2, VAULT_PATH_REPO2)})) + _ = Must(vaultClient.Auth.AppRoleWriteRole(ctx, VAULT_APP_ROLE1, schema.AppRoleWriteRoleRequest{TokenType: "batch", SecretIdTtl: "10m", TokenTtl: "20m", TokenMaxTtl: "30m", SecretIdNumUses: 40, TokenPolicies: []string{VAULT_POLICY_NAME1}})) + + role := Must(vaultClient.Auth.AppRoleReadRoleId(ctx, VAULT_APP_ROLE1)) + roleid := role.Data.RoleId + // Unfortunately, this function is currently bugged, therefore we fall back to the generic function + //secretid := Must(client.Auth.AppRoleWriteSecretId(ctx, VAULT_APP_ROLE, schema.AppRoleWriteSecretIdRequest{})) + secret := Must(vaultClient.Write(ctx, fmt.Sprintf("/v1/auth/approle/role/%s/secret-id", VAULT_APP_ROLE1), map[string]interface{}{})) + secretid := secret.Data["secret_id"].(string) + + // Write a secret with the credentials for vault repo 2 (VAULT_PATH_REPO2) into vault repo 1 and write the + // consumer id of vault repo 2 into the secrets metadata + consumerId := Must(identity.GetConsumerId(VAULT_HTTP_URL, "", "secret", VAULT_PATH_REPO2)) + data := map[string]any{ + identity.ATTR_AUTHMETH: identity.AUTH_APPROLE, + identity.ATTR_ROLEID: roleid, + identity.ATTR_SECRETID: secretid, + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + consumerIdData := Must(runtime.DefaultJSONEncoding.Marshal(consumerId)) + metadata := map[string]any{ + me.CUSTOM_CONSUMERID: string(consumerIdData), + } + _ = Must(vaultClient.Secrets.KvV2WriteMetadata(ctx, VAULT_PATH_REPO1+"/"+VAULT_SECRET, + schema.KvV2WriteMetadataRequest{CustomMetadata: metadata}, + vault.WithMountPath("secret"), + )) + + // Write a secret with arbitrary data into vault repo 2 + data = map[string]any{ + "password1": "ocm-password-1", + } + _ = Must(vaultClient.Secrets.KvV2Write(ctx, VAULT_PATH_REPO2+"/"+VAULT_SECRET, + schema.KvV2WriteRequest{Data: data}, + vault.WithMountPath("secret"), + )) + consumerId = hostpath.GetConsumerIdentity(hostpath.IDENTITY_TYPE, "https://test-url.com") + consumerIdData = Must(runtime.DefaultJSONEncoding.Marshal(consumerId)) + metadata = map[string]any{ + me.CUSTOM_CONSUMERID: string(consumerIdData), + } + _ = Must(vaultClient.Secrets.KvV2WriteMetadata(ctx, VAULT_PATH_REPO2+"/"+VAULT_SECRET, + schema.KvV2WriteMetadataRequest{CustomMetadata: metadata}, + vault.WithMountPath("secret"), + )) + + spec.PropgateConsumerIdentity = true + repo := Must(DefaultContext.RepositoryForSpec(spec)) + Expect(repo).ToNot(BeNil()) + + fmt.Println("***add second provider:") + spec1.PropgateConsumerIdentity = true + repo = Must(DefaultContext.RepositoryForSpec(spec1)) + Expect(repo).ToNot(BeNil()) + + fmt.Println("***query credential:") + c := Must(DefaultContext.GetCredentialsForConsumer(consumerId)) + Expect(c).To(YAMLEqual(data)) + }) + + //D(irect): + // - has general credentials matching parent path of P2 + // - has credentials for P1 + //P1: + // - has specialized credentials for P2 + //P2: + // - has credentials for C + // + // + //query C: + //- D: -> nothing + //- P1: query P1 + // - D: -> found + // - P1: omit (recursion) + // - P2: query P2 + // - D: -> found + // - P1: omit (recursion) + // - P2: omit (recursion) + // explore, whether an additional attempt with P1 BUT only with credentialless providers / direct creds + // would work as a general solution. + // -> select D(P2) WRONG (a1) + //- P2: query P2 + // - D: -> found + // - P1: query P1 + // - D: found + // - P1: omit (recursion) + // - P2: omit (recursion) + // -> select D(P1) CORRECT (b) + // -> found + // - P2: omit (recursion) + // -> select P1(P2) CORRECT (a2) + // -> found + //-> select P2(C) + // + // The Problem here is, that the case a1 and case b are formally indistinguishable. While a2 and b lead to the + // correct result, we would fail in a1. + It("recursive authentication with overlapping credentials", func() { + //TODO + }) + }) +}) + +func StartVaultServer(mode vaultMode, rootToken, address string) (*exec.Cmd, *vault.Client, context.CancelFunc, error) { + cmdctx, cancelFunc := context.WithCancel(context.Background()) + if mode == "" { + mode = HTTP + } + url := address + switch mode { + case HTTP: + url = "http://" + url + case HTTPS: + url = "https://" + url + } + + cmd := exec.CommandContext(cmdctx, "../../../../../bin/vault", "server", "-"+string(mode), fmt.Sprintf("-dev-root-token-id=%s", rootToken), fmt.Sprintf("-dev-listen-address=%s", address)) + //cmd.Stdout = os.Stdout + //cmd.Stderr = os.Stderr + + vaultClient, err := vault.New( + vault.WithAddress(url), + vault.WithRequestTimeout(30*time.Second), + ) + if err != nil { + return nil, nil, cancelFunc, err + } + + // authenticate with root token + err = vaultClient.SetToken(rootToken) + if err != nil { + return nil, nil, cancelFunc, err + } + + err = cmd.Start() + if err == nil { + err = WaitForTCPServer(address, time.Minute) + } + return cmd, vaultClient, cancelFunc, err +} + +func WaitForTCPServer(address string, dur time.Duration) error { + var conn net.Conn + var d net.Dialer + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + end := time.Now().Add(dur) + err := errors.New("timed out waiting for server to start") + for time.Now().Before(end) { + conn, err = d.DialContext(ctx, "tcp", address) + if err != nil { + time.Sleep(time.Second) + continue + } + conn.Close() + break + } + return err +} + +func SetUpVaultAccess(ctx context.Context, credctx credentials.Context, client *vault.Client, policy string) { + _ = Must(client.System.AuthEnableMethod(ctx, "approle", schema.AuthEnableMethodRequest{Type: "approle"})) + _ = Must(client.System.PoliciesWriteAclPolicy(ctx, VAULT_POLICY_NAME, schema.PoliciesWriteAclPolicyRequest{Policy: policy})) + _ = Must(client.Auth.AppRoleWriteRole(ctx, VAULT_APP_ROLE, schema.AppRoleWriteRoleRequest{TokenType: "batch", SecretIdTtl: "10m", TokenTtl: "20m", TokenMaxTtl: "30m", SecretIdNumUses: 40, TokenPolicies: []string{VAULT_POLICY_NAME}})) + + role := Must(client.Auth.AppRoleReadRoleId(ctx, VAULT_APP_ROLE)) + roleid := role.Data.RoleId + // Unfortunately, this function is currently bugged, therefore we fall back to the generic function + //secretid := Must(client.Auth.AppRoleWriteSecretId(ctx, VAULT_APP_ROLE, schema.AppRoleWriteSecretIdRequest{})) + secret := Must(client.Write(ctx, fmt.Sprintf("/v1/auth/approle/role/%s/secret-id", VAULT_APP_ROLE), map[string]interface{}{})) + secretid := secret.Data["secret_id"].(string) + + consumerId := Must(identity.GetConsumerId(client.Configuration().Address, "", "secret", VAULT_PATH_REPO1)) + creds := credentials.NewCredentials(common.Properties{ + identity.ATTR_AUTHMETH: identity.AUTH_APPROLE, + identity.ATTR_ROLEID: roleid, + identity.ATTR_SECRETID: secretid, + }) + credctx.SetCredentialsForConsumer(consumerId, creds) +} diff --git a/pkg/contexts/credentials/repositories/vault/repo_test.go b/pkg/contexts/credentials/repositories/vault/repo_test.go new file mode 100644 index 0000000000..7c9e5044d4 --- /dev/null +++ b/pkg/contexts/credentials/repositories/vault/repo_test.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package vault_test + +import ( + "encoding/json" + "fmt" + "reflect" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials" + me "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault" + "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault/identity" +) + +const ( + VAULT_ADDRESS = "127.0.0.1:8200" + VAULT_HTTP_URL = "http://" + VAULT_ADDRESS + VAULT_NAMESPACE = "test-namespace" + VAULT_MOUNT_PATH = "secret" + VAULT_PATH_REPO1 = "mysecrets/repo1" + VAULT_PATH_REPO2 = "mysecrets/repo2" +) + +var _ = Describe("", func() { + Context("serialization and deserialization", func() { + DefaultContext := credentials.New() + + specdata := fmt.Sprintf("{\"type\": %q, \"serverURL\": %q, \"namespace\": %q, \"mountPath\": %q, \"path\": %q, \"secrets\": [\"secret1\", \"secret2\", \"secret3\"], \"propagateConsumerIdentity\": true }", me.Type, "http://"+VAULT_ADDRESS, VAULT_NAMESPACE, VAULT_MOUNT_PATH, VAULT_PATH_REPO1) + spec := me.NewRepositorySpec("http://"+VAULT_ADDRESS, me.WithNamespace(VAULT_NAMESPACE), me.WithMountPath(VAULT_MOUNT_PATH), me.WithPath(VAULT_PATH_REPO1), me.WithSecrets("secret1", "secret2", "secret3"), me.WithPropagation()) + + specdata2 := fmt.Sprintf("{\"type\": %q, \"serverURL\": %q }", me.Type, "http://"+VAULT_ADDRESS) + spec2 := me.NewRepositorySpec("http://" + VAULT_ADDRESS) + + It("serializes repo spec", func() { + data := Must(json.Marshal(spec)) + Expect(data).To(YAMLEqual([]byte(specdata))) + + data = Must(json.Marshal(spec2)) + Expect(data).To(YAMLEqual([]byte(specdata2))) + }) + + It("deserializes repo spec", func() { + localspec := Must(DefaultContext.RepositorySpecForConfig([]byte(specdata), nil)) + Expect(reflect.TypeOf(localspec).String()).To(Equal("*vault.RepositorySpec")) + Expect(localspec).To(Equal(spec)) + + localspec = Must(DefaultContext.RepositorySpecForConfig([]byte(specdata2), nil)) + Expect(reflect.TypeOf(localspec).String()).To(Equal("*vault.RepositorySpec")) + Expect(localspec).To(Equal(spec2)) + }) + + It("resolves repository", func() { + // Since vault always requires credentials to be accessed, RepositoryForConfig checks whether credentials + // for a corresponding consumer exist. Thus, creating such credentials is required to test the method even + // though they are not used + consumerId := Must(identity.GetConsumerId(VAULT_HTTP_URL, VAULT_NAMESPACE, VAULT_MOUNT_PATH, VAULT_PATH_REPO1)) + creds := credentials.NewCredentials(common.Properties{ + identity.ATTR_AUTHMETH: identity.AUTH_TOKEN, + identity.ATTR_TOKEN: "token", + }) + DefaultContext.SetCredentialsForConsumer(consumerId, creds) + + repo := Must(DefaultContext.RepositoryForConfig([]byte(specdata), nil)) + Expect(repo).ToNot(BeNil()) + }) + }) +}) diff --git a/pkg/contexts/credentials/repositories/vault/repository.go b/pkg/contexts/credentials/repositories/vault/repository.go index cd736718ba..449db5667a 100644 --- a/pkg/contexts/credentials/repositories/vault/repository.go +++ b/pkg/contexts/credentials/repositories/vault/repository.go @@ -21,7 +21,7 @@ var ( ) func NewRepository(ctx cpi.Context, spec *RepositorySpec) (*Repository, error) { - id, err := identity.GetConsumerId(spec.ServerURL, spec.Namespace, spec.SecretsEngine, spec.Path) + id, err := identity.GetConsumerId(spec.ServerURL, spec.Namespace, spec.MountPath, spec.Path) if err != nil { return nil, err } diff --git a/pkg/contexts/credentials/repositories/vault/type.go b/pkg/contexts/credentials/repositories/vault/type.go index 9352544297..2e00632427 100644 --- a/pkg/contexts/credentials/repositories/vault/type.go +++ b/pkg/contexts/credentials/repositories/vault/type.go @@ -53,8 +53,8 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds cpi.Credentials) (cpi } spec := *a spec.Secrets = slices.Clone(a.Secrets) - if spec.SecretsEngine == "" { - spec.SecretsEngine = "secrets" + if spec.MountPath == "" { + spec.MountPath = "secret" } return repos.GetRepository(ctx, &spec) } @@ -70,7 +70,7 @@ func (a *RepositorySpec) GetKey() cpi.ProviderIdentity { } func (a *RepositorySpec) GetConsumerId(uctx ...internal.UsageContext) internal.ConsumerIdentity { - id, err := identity.GetConsumerId(a.ServerURL, a.Namespace, a.SecretsEngine, a.Path) + id, err := identity.GetConsumerId(a.ServerURL, a.Namespace, a.MountPath, a.Path) if err != nil { return nil }