Vault tests (#749)

## Description
Implement integration tests with vault.

It also fixes a bug where getting the credentials for a consumer
provider lead to a deadlock or potentially an infinite recursion since
we attempt to retrieve credentials to access the credential provider
from the credential provider itself.

Introduces an additional make target called `make test-all`. This runs 
all test including the ones tagged with `go:build integration`.

.github/workflows/lint_and_test.yaml

@@ -38,7 +38,7 @@ jobs:
       - name: Unit test
         run: |
           PATH=$PATH:$(go env GOPATH)/bin make build
-          PATH=$PATH:$(go env GOPATH)/bin make test
+          PATH=$PATH:$(go env GOPATH)/bin make test-all
 
   lint:
     name: Lint

.gitignore

@@ -19,3 +19,4 @@ bin/
 go.mod.bak
 dist/
 .cache_ggshield
+.DS_Store

Makefile

@@ -60,9 +60,14 @@ force-test:
 
 .PHONY: test
 test:
-	@echo "> Test"
+	@echo "> Run Unit Tests"
 	@go test  ./examples/lib/... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/demoplugin/... $(REPO_ROOT)/pkg/...
 
+.PHONY: test-all
+test-all: install-requirements
+	@echo "> Run All Tests"
+	@go test --tags=integration ./examples/lib/... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/demoplugin/... $(REPO_ROOT)/pkg/...
+
 .PHONY: generate
 generate:
 	@$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminstaller/... $(REPO_ROOT)/examples/...

docs/reference/ocm_controller.md

@@ -22,4 +22,5 @@ ocm controller [<options>] <sub command> ...
 ##### Sub Commands
 
 * [ocm controller <b>install</b>](ocm_controller_install.md)	 &mdash; Install either a specific or latest version of the ocm-controller. Optionally install prerequisites required by the controller.
+* [ocm controller <b>uninstall</b>](ocm_controller_uninstall.md)	 &mdash; Uninstalls the ocm-controller and all of its dependencies
 

docs/reference/ocm_controller_uninstall.md

@@ -0,0 +1,32 @@
+## ocm controller uninstall — Uninstalls The Ocm-Controller And All Of Its Dependencies
+
+### Synopsis
+
+```
+ocm controller uninstall controller
+```
+
+### Options
+
+```
+  -u, --base-url string                       the base url to the ocm-controller's release page (default "https://github.com/open-component-model/ocm-controller/releases")
+      --cert-manager-base-url string          the base url to the cert-manager's release page (default "https://github.com/cert-manager/cert-manager/releases")
+      --cert-manager-release-api-url string   the base url to the cert-manager's API release page (default "https://api.github.com/repos/cert-manager/cert-manager/releases")
+      --cert-manager-version string           version for cert-manager (default "v1.13.2")
+  -c, --controller-name string                name of the controller that's used for status check (default "ocm-controller")
+  -d, --dry-run                               if enabled, prints the downloaded manifest file
+  -h, --help                                  help for uninstall
+  -n, --namespace string                      the namespace into which the controller is installed (default "ocm-system")
+  -a, --release-api-url string                the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases")
+  -t, --timeout duration                      maximum time to wait for deployment to be ready (default 1m0s)
+  -p, --uninstall-prerequisites               uninstall prerequisites required by ocm-controller
+  -v, --version string                        the version of the controller to install (default "latest")
+```
+
+### SEE ALSO
+
+##### Parents
+
+* [ocm controller](ocm_controller.md)	 — Commands acting on the ocm-controller
+* [ocm](ocm.md)	 — Open Component Model command line client
+

docs/reference/ocm_credential-handling.md

@@ -128,7 +128,7 @@ The following credential consumer types are used/supported:
       - <code>scheme</code>: (optional) URL scheme
       - <code>port</code>: (optional) server port
       - <code>namespace</code>: vault namespace
-      - <code>secretEngine</code>: secret engine
+      - <code>mountPath</code>: mount path
       - <code>pathprefix</code>: path prefix for secret
 
 
@@ -138,7 +138,6 @@ The following credential consumer types are used/supported:
       - <code>token</code>: vault token
       - <code>roleid</code>: applrole role id
       - <code>secretid</code>: applrole secret id
-      - <code>secretid</code>: applrole secret id
 
     The only supported auth methods, so far, are <code>token</code> and <code>approle</code>.
 
@@ -315,7 +314,7 @@ The following types are currently available:
     - <code>scheme</code>: (optional) URL scheme
     - <code>port</code>: (optional) server port
     - <code>namespace</code>: vault namespace
-    - <code>secretEngine</code>: secret engine
+    - <code>mountPath</code>: mount path
     - <code>pathprefix</code>: path prefix for secret
 
 
@@ -325,7 +324,6 @@ The following types are currently available:
     - <code>token</code>: vault token
     - <code>roleid</code>: applrole role id
     - <code>secretid</code>: applrole secret id
-    - <code>secretid</code>: applrole secret id
 
   The only supported auth methods, so far, are <code>token</code> and <code>approle</code>.
 
@@ -335,7 +333,7 @@ The following types are currently available:
     The repository specification supports the following fields:
       - <code>serverURL</code>: *string* (required): the URL of the vault instance
       - <code>namespace</code>: *string* (optional): the namespace used to evaluate secrets
-      - <code>secretsEngine</code>: *string* (optional): the secrets engine to use (default: secrets)
+      - <code>mountPath</code>: *string* (optional): the mount path to use (default: secrets)
       - <code>path</code>: *string* (optional): the path prefix used to lookup secrets
       - <code>secrets</code>: *[]string* (optional): list of secrets
       - <code>propagateConsumerIdentity</code>: *bool*(optional): evaluate metadata for consumer id propagation

docs/reference/ocm_get_credentials.md

@@ -54,7 +54,7 @@ Matchers exist for the following usage contexts or consumer types:
       - <code>scheme</code>: (optional) URL scheme
       - <code>port</code>: (optional) server port
       - <code>namespace</code>: vault namespace
-      - <code>secretEngine</code>: secret engine
+      - <code>mountPath</code>: mount path
       - <code>pathprefix</code>: path prefix for secret
 
 
@@ -64,7 +64,6 @@ Matchers exist for the following usage contexts or consumer types:
       - <code>token</code>: vault token
       - <code>roleid</code>: applrole role id
       - <code>secretid</code>: applrole secret id
-      - <code>secretid</code>: applrole secret id
 
     The only supported auth methods, so far, are <code>token</code> and <code>approle</code>.
 

examples/lib/tour/doc.go

@@ -1,3 +1,3 @@
-//go:generate mdref --headings --list docsrc .
+//go:generate mdref --headings docsrc .
 
 package tour

go.mod

@@ -1,6 +1,6 @@
 module github.com/open-component-model/ocm
 
-go 1.22.1
+go 1.22.2
 
 replace github.com/spf13/cobra => github.com/open-component-model/cobra v0.0.0-20230329075350-b1fd876abfb9
 
@@ -222,6 +222,7 @@ require (
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
+	github.com/hashicorp/vault/api v1.13.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/in-toto/in-toto-golang v0.9.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

go.sum

@@ -591,8 +591,8 @@ github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31
 github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
 github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
 github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
-github.com/hashicorp/vault/api v1.12.2 h1:7YkCTE5Ni90TcmYHDBExdt4WGJxhpzaHqR6uGbQb/rE=
-github.com/hashicorp/vault/api v1.12.2/go.mod h1:LSGf1NGT1BnvFFnKVtnvcaLBM2Lz+gJdpL6HUYed8KE=
+github.com/hashicorp/vault/api v1.13.0 h1:RTCGpE2Rgkn9jyPcFlc7YmNocomda44k5ck8FKMH41Y=
+github.com/hashicorp/vault/api v1.13.0/go.mod h1:0cb/uZUv1w2cVu9DIvuW1SMlXXC6qtATJt+LXJRx+kg=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM=

hack/Makefile

@@ -1,4 +1,8 @@
 GOPATH                                         := $(shell go env GOPATH)
+LOCALBIN := $(shell pwd)/../bin
+OS := $(shell go env GOOS 2>/dev/null || sh -c 'uname -o' | sed 's/.*/\L&/' )
+ARCH := $(shell go env GOARCH 2>/dev/null || sh -c 'uname -m' | sed 's/.*/\L&/' )
+OS_ARCH := $(OS)_$(ARCH)
 
 ifeq ($(OS),Windows_NT)
 	detected_OS := Windows
@@ -39,6 +43,16 @@ GO_BINDATA := $(shell (go-bindata -version 2>/dev/null || echo 0.0.0) | head -n
 ifneq ("v$(GO_BINDATA)",$(GO_BINDATA_VERSION))
 	deps += go-bindata
 endif
+VAULT_VERSION := 1.16.2
+VAULT := $(shell ($(LOCALBIN)/vault --version 2>/dev/null || echo 0.0) | sed 's/.*Vault v\([0-9\.]*\).*/\1/')
+ifeq ($(VAULT), $(VAULT_VERSION))
+	deps += vault
+endif
+OCI_REGISTRY_VERSION := 3.0.0-alpha.1
+OCI_REGISTRY := $(shell (registry --version 2>/dev/null || echo 0.0) | sed 's/.* v\([0-9a-z\.\-]*\).*/\1/')
+ifeq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION))
+	deps += oci-registry
+endif
 
 .PHONY: install-requirements
 install-requirements: $(deps) $(GOPATH)/bin/goimports mdref
@@ -58,14 +72,28 @@ golangci-lint-version:
 go-bindata:
 	go install -v  github.com/go-bindata/go-bindata/v3/...@$(GO_BINDATA_VERSION)
 
+.PHONY: vault
+vault:
+	@if [ "$(VAULT)" != "$(VAULT_VERSION)" ]; then \
+		curl -o $(LOCALBIN)/vault.zip https://releases.hashicorp.com/vault/$(VAULT_VERSION)/vault_$(VAULT_VERSION)_$(OS_ARCH).zip; \
+		unzip -o $(LOCALBIN)/vault.zip -d $(LOCALBIN); \
+		rm $(LOCALBIN)/vault.zip; \
+		chmod a+x $(LOCALBIN)/vault;\
+	fi
+
+.PHONY: oci-registry
+oci-registry:
+	@if [ "$(OCI_REGISTRY)" != "$(OCI_REGISTRY_VERSION)" ]; then \
+		go install -v github.com/distribution/distribution/v3/cmd/[email protected]; \
+	fi
+
 $(GOPATH)/bin/goimports:
 	go install -v golang.org/x/tools/cmd/goimports@latest
 
 .PHONY: mdref
 mdref:
 	go install -v github.com/mandelsoft/mdref@master
 
-
 Linux_jq:
 	$(info -> jq is missing)
 	$(info -  sudo apt-get install jq / sudo dnf install jq / sudo zypper install jq / sudo pacman -S jq)

pkg/contexts/credentials/cpi/interface.go

@@ -30,6 +30,7 @@ type (
 	GenericRepositorySpec  = internal.GenericRepositorySpec
 	GenericCredentialsSpec = internal.GenericCredentialsSpec
 	DirectCredentials      = internal.DirectCredentials
+	EvaluationContext      = internal.EvaluationContext
 )
 
 type (
@@ -102,6 +103,18 @@ func RequiredCredentialsForConsumer(ctx ContextProvider, id ConsumerIdentity, ma
 	return internal.CredentialsForConsumer(ctx, id, true, matchers...)
 }
 
+func GetCredentialsForConsumer(ctx Context, ectx EvaluationContext, identity ConsumerIdentity, matchers ...IdentityMatcher) (CredentialsSource, error) {
+	return internal.GetCredentialsForConsumer(ctx, ectx, identity, matchers...)
+}
+
+func GetEvaluationContextFor[T any](ectx EvaluationContext) T {
+	return internal.GetEvaluationContextFor[T](ectx)
+}
+
+func SetEvaluationContextFor(ectx EvaluationContext, e any) {
+	internal.SetEvaluationContextFor(ectx, e)
+}
+
 var (
 	CompleteMatch = internal.CompleteMatch
 	NoMatch       = internal.NoMatch

pkg/contexts/credentials/internal/consumers.go

@@ -1,13 +1,33 @@
 package internal
 
 import (
+	"fmt"
+	"slices"
 	"sort"
 	"sync"
 
+	"github.com/mandelsoft/goutils/exception"
+	"github.com/mandelsoft/goutils/general"
 	"github.com/mandelsoft/goutils/maputils"
+	"github.com/mandelsoft/goutils/sliceutils"
+	"github.com/mandelsoft/goutils/stringutils"
 )
 
-// UsageContext descibes a dediacetd type specific
+type CredentialRecursion []ConsumerIdentity
+
+func (c CredentialRecursion) String() string {
+	return stringutils.Join(c)
+}
+
+func (c CredentialRecursion) Contains(identity ConsumerIdentity) bool {
+	return slices.ContainsFunc(c, general.ContainsFuncFor(identity))
+}
+
+func (c CredentialRecursion) Append(identity ConsumerIdentity) CredentialRecursion {
+	return sliceutils.CopyAppendUniqueFunc(c, general.EqualsFuncFor[ConsumerIdentity](), identity)
+}
+
+// UsageContext describes a dedicated type specific
 // sub usage kinds for an object requiring credentials.
 // For example, for an object providing a hierarchical
 // namespace this might be a namespace prefix for
@@ -73,7 +93,7 @@ func (c *_consumers) Get(id ConsumerIdentity) (CredentialsSource, bool) {
 
 // Match matches a given request (pattern) against configured
 // identities.
-func (c *_consumers) Match(pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) {
+func (c *_consumers) Match(ectx EvaluationContext, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) {
 	var found *_consumer
 	for _, s := range c.data {
 		if m(pattern, cur, s.identity) {
@@ -195,18 +215,72 @@ func (p *consumerProviderRegistry) Get(id ConsumerIdentity) (CredentialsSource,
 	return nil, false
 }
 
-func (p *consumerProviderRegistry) Match(pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) {
-	p.lock.Lock()
-	defer p.lock.Unlock()
+func (p *consumerProviderRegistry) checkHandleProvider(ectx EvaluationContext, prov ConsumerProvider, pattern ConsumerIdentity) (rctx EvaluationContext, useprov bool, usestack bool) {
+	if pr, ok := prov.(ConsumerIdentityProvider); ok {
+		r := GetEvaluationContextFor[CredentialRecursion](ectx)
+		if r == nil {
+			r = CredentialRecursion{}
+		}
+		if r.Contains(pr.GetConsumerId()) {
+			return ectx, false, true
+		}
+		r = r.Append(pr.GetConsumerId())
+		ectx = SetEvaluationContextFor(ectx, r)
+	}
+	return ectx, true, true
+}
 
-	credsrc, cur := p.explicit.Match(pattern, cur, m)
+type UnwindStack struct {
+	error
+}
+
+func (u *UnwindStack) Unwrap() error {
+	return u.error
+}
+
+func (p *consumerProviderRegistry) catchedMatch(ectx EvaluationContext, sub ConsumerProvider, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (cs CredentialsSource, ci ConsumerIdentity) {
+	defer exception.CatchError(func(err error) {
+		log.Trace("caught unwind stack error: {{error}}", "error", err)
+		cs = nil
+		ci = cur
+	}, exception.ByPrototypes(&UnwindStack{}))
+	log.Trace("pattern: {{pattern}}\ncontext: {{context}}\nprovider: {{provider}}",
+		"pattern", pattern, "context", ectx, "provider", sub)
+	ectx, useprov, _ := p.checkHandleProvider(ectx, sub, pattern)
+	if !useprov {
+		return nil, cur
+	}
+	log.Trace("attempt match with provider: {{provider}}", "provider", sub)
+	return sub.Match(ectx, pattern, cur, m)
+}
+
+func (p *consumerProviderRegistry) Match(ectx EvaluationContext, pattern ConsumerIdentity, cur ConsumerIdentity, m IdentityMatcher) (CredentialsSource, ConsumerIdentity) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	credsrc, cur := p.explicit.Match(ectx, pattern, cur, m)
 	for _, sub := range p.providers {
 		var f CredentialsSource
-		f, cur = sub.Match(pattern, cur, m)
+		f, cur = p.catchedMatch(ectx, sub, pattern, cur, m)
 		if f != nil {
 			credsrc = f
 		}
 	}
+	// If this is the case, we are in a situation where we have excluded all providers (since they are all in the stack).
+	// If we would simply return with no credentials, the follow-up coding would assume, that it should query the
+	// credential repository without any credentials, since none have been found.
+	// INSTEAD, we have to step down to the previous recursion level and check the other potentially available providers
+	// for credentials.
+	// BUT in case we have explicit credentials, then we should use those.
+	if credsrc == nil {
+		r := GetEvaluationContextFor[CredentialRecursion](ectx)
+		// unwind the stack only makes sense when we are in a recursive call, thus we have at least one provider on the
+		// credential recursion stack
+		if len(r) > 0 && len(r) == len(p.providers) {
+			exception.Throw(&UnwindStack{fmt.Errorf("impossible credential recursion detected - unwind stack")})
+		}
+	}
+	log.Trace("return credential source")
 	return credsrc, cur
 }