diff --git a/README.md b/README.md index 64a1845b85..bf74062c14 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The library currently supports the following [repository mappings](docs/ocm/inte - **Component Archive**: Compose the content of a component version on the filesystem. -For the usage of the library to access OCM repositories, handle configuratio and credentials see the [examples section](examples/lib/README.md). +For the usage of the library to access OCM repositories, handle configuration and credentials see the [examples section](examples/lib/README.md). Additionally, OCM provides a generic solution for how to: diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/cpi/helper.go b/cmds/ocm/commands/ocmcmds/common/inputs/cpi/helper.go index c6d3873e9f..7101caed9d 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/cpi/helper.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/cpi/helper.go @@ -157,7 +157,7 @@ func (s *MediaFileSpec) ValidateFile(fldPath *field.Path, ctx clictx.Context, in pathField := fldPath.Child("path") fileInfo, filePath, err := inputs.FileInfo(ctx, s.Path, inputFilePath) if err != nil { - allErrs = append(allErrs, field.Invalid(pathField, filePath, err.Error())) + allErrs = append(allErrs, field.Invalid(pathField, s.Path, err.Error())) } return fileInfo, filePath, allErrs } diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/docker/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/docker/spec.go index 9c93fa5f3a..b2c70b30cc 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/docker/spec.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/docker/spec.go @@ -11,9 +11,7 @@ import ( "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/cpi" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/ociimage" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" - "github.com/open-component-model/ocm/pkg/contexts/oci" - "github.com/open-component-model/ocm/pkg/contexts/oci/annotations" - "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockerdaemon" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" ) @@ -53,25 +51,7 @@ func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (bloba if err != nil { return nil, "", err } - spec := docker.NewRepositorySpec() - repo, err := ctx.OCIContext().RepositoryForSpec(spec) - if err != nil { - return nil, "", err - } - ns, err := repo.LookupNamespace(locator) - if err != nil { - return nil, "", err - } - - if version == "" || version == "latest" { - version = info.ComponentVersion.GetVersion() - } - blob, err := artifactset.SynthesizeArtifactBlob(ns, version, - func(art oci.ArtifactAccess) error { - art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String()) - return nil - }, - ) + blob, version, err := dockerdaemon.BlobAccessForImageFromDockerDaemon(s.Path, dockerdaemon.WithVersion(info.ComponentVersion.GetVersion()), dockerdaemon.WithOrigin(info.ComponentVersion)) if err != nil { return nil, "", err } diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti/spec.go index 5b82a26608..d1fe9b904a 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti/spec.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti/spec.go @@ -9,21 +9,17 @@ import ( . "github.com/open-component-model/ocm/pkg/finalizer" - "github.com/opencontainers/go-digest" "k8s.io/apimachinery/pkg/util/validation/field" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/ociimage" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockermulti" "github.com/open-component-model/ocm/pkg/contexts/clictx" "github.com/open-component-model/ocm/pkg/contexts/oci" - "github.com/open-component-model/ocm/pkg/contexts/oci/annotations" - "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" - "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" - "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/runtime" ) @@ -100,77 +96,12 @@ func (s *Spec) getVariant(ctx clictx.Context, finalize *Finalizer, variant strin } func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blobaccess.BlobAccess, string, error) { - index := artdesc.NewIndexArtifact() - i := 0 - - index.SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String()) - - feedback := func(blob blobaccess.BlobAccess, art cpi.ArtifactAccess) error { - desc := artdesc.DefaultBlobDescriptor(blob) - if art.IsManifest() { - cfgBlob, err := art.ManifestAccess().GetConfigBlob() - if err != nil { - return errors.Wrapf(err, "cannot get config blob") - } - cfg, err := artdesc.ParseImageConfig(cfgBlob) - if err != nil { - return errors.Wrapf(err, "cannot parse config blob") - } - if cfg.Architecture != "" { - desc.Platform = &artdesc.Platform{ - Architecture: cfg.Architecture, - OS: cfg.OS, - Variant: cfg.Variant, - } - } - } - index.Index().AddManifest(desc) - return nil - } - - blob, err := artifactset.SynthesizeArtifactBlobFor(info.ComponentVersion.GetVersion(), func() (fac artifactset.ArtifactFactory, main bool, err error) { - var art cpi.ArtifactAccess - var blob blobaccess.BlobAccess - - switch { - case i > len(s.Variants): - // end loop - case i == len(s.Variants): - // provide index (main) artifact - ctx.Printf("image %d: INDEX\n", i) - fac = func(set *artifactset.ArtifactSet) (digest.Digest, string, error) { - art, err = set.NewArtifact(index) - if err != nil { - return "", "", errors.Wrapf(err, "cannot create index artifact") - } - defer art.Close() - blob, err = set.AddArtifact(art) - if err != nil { - return "", "", errors.Wrapf(err, "cannot add index artifact") - } - defer blob.Close() - return blob.Digest(), blob.MimeType(), nil - } - main = true - default: - // provide variant - ctx.Printf("image %d: %s\n", i, s.Variants[i]) - var finalize Finalizer - - art, err = s.getVariant(ctx, &finalize, s.Variants[i]) - - if err == nil { - art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String()) - blob, err = art.Blob() - if err == nil { - finalize.Close(art) - fac = artifactset.ArtifactTransferCreator(art, &finalize, feedback) - } - } - } - i++ - return - }) + blob, err := dockermulti.BlobAccessForMultiImageFromDockerDaemon( + dockermulti.WithContext(ctx), + dockermulti.WithPrinter(ctx.Printer()), + dockermulti.WithVariants(s.Variants...), + dockermulti.WithOrigin(info.ComponentVersion), + dockermulti.WithVersion(info.ComponentVersion.GetVersion())) if err != nil { return nil, "", err } diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/helm/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/helm/spec.go index d1cd8b67d7..24a62a5b4d 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/helm/spec.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/helm/spec.go @@ -5,19 +5,14 @@ package helm import ( - "github.com/mandelsoft/vfs/pkg/vfs" "k8s.io/apimachinery/pkg/util/validation/field" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/cpi" - "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" - ocihelm "github.com/open-component-model/ocm/pkg/contexts/oci/ociutils/helm" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/helm" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/helm" - "github.com/open-component-model/ocm/pkg/helm/identity" - "github.com/open-component-model/ocm/pkg/helm/loader" ) type Spec struct { @@ -93,60 +88,34 @@ func (s *Spec) Validate(fldPath *field.Path, ctx inputs.Context, inputFilePath s } func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blob blobaccess.BlobAccess, hint string, err error) { - var chartLoader loader.Loader + path := s.Path if s.HelmRepository == "" { - _, inputPath, err := inputs.FileInfo(ctx, s.Path, info.InputFilePath) + _, inputPath, err := inputs.FileInfo(ctx, path, info.InputFilePath) if err != nil { return nil, "", errors.Wrapf(err, "cannot handle input path %q", s.Path) } - chartLoader = loader.VFSLoader(inputPath, ctx.FileSystem()) - - } else { - cert := []byte(s.CACert) - if s.CACertFile != "" { - _, certPath, err := inputs.FileInfo(ctx, s.CACertFile, info.InputFilePath) - if err != nil { - return nil, "", err - } - cert, err = vfs.ReadFile(ctx.FileSystem(), certPath) - if err != nil { - return nil, "", errors.Wrapf(err, "cannot read root certificates from %q", s.CACertFile) - } - } - - acc, err := helm.DownloadChart(common.NewPrinter(ctx.StdOut()), ctx, s.Path, s.Version, s.HelmRepository, - helm.WithCredentials(identity.GetCredentials(ctx, s.HelmRepository, s.Path)), - helm.WithRootCert([]byte(cert))) - if err != nil { - return nil, "", errors.Wrapf(err, "cannot download chart %s:%s from %s", s.Path, s.Version, s.HelmRepository) - } - chartLoader = loader.AccessLoader(acc) - } - - defer errors.PropagateError(&err, chartLoader.Close) - - chart, err := chartLoader.Chart() - if err != nil { - return nil, "", err - } - vers := chart.Metadata.Version - if s.Version != "" { - vers = s.Version + path = inputPath } + vers := s.Version + override := true if vers == "" { vers = info.ComponentVersion.GetVersion() + override = false } - hint = ociartifact.Hint(info.ComponentVersion, chart.Name(), s.Repository, vers) - blob, err = chartLoader.ChartArtefactSet() - if err != nil || blob != nil { - return blob, hint, err - } - blob, err = ocihelm.SynthesizeArtifactBlob(chartLoader) + blob, name, vers, err := helm.BlobAccessForHelmChart(path, + helm.WithContext(ctx), + helm.WithFileSystem(ctx.FileSystem()), + helm.WithPrinter(ctx.Printer()), + helm.WithVersionOverride(vers, override), + helm.WithCACert(s.CACert), + helm.WithCACertFile(s.CACertFile), + helm.WithHelmRepository(s.HelmRepository), + ) if err != nil { - return nil, "", errors.Wrapf(err, "cannot synthesize artifact blob") + return nil, "", err } - + hint = ociartifact.Hint(info.ComponentVersion, name, s.Repository, vers) return blob, hint, err } diff --git a/cmds/ocm/commands/ocmcmds/routingslips/add/cmd.go b/cmds/ocm/commands/ocmcmds/routingslips/add/cmd.go index 5aa6c690f7..3edfc60369 100644 --- a/cmds/ocm/commands/ocmcmds/routingslips/add/cmd.go +++ b/cmds/ocm/commands/ocmcmds/routingslips/add/cmd.go @@ -223,5 +223,5 @@ func (a *action) Out() error { } else { _, err = routingslip.AddEntry(cv, a.cmd.Name, a.cmd.Algorithm, a.cmd.Entry, links, digest.Digest(a.cmd.Digest)) } - return err + return cv.Update() } diff --git a/cmds/ocm/commands/ocmcmds/routingslips/get/cmd_test.go b/cmds/ocm/commands/ocmcmds/routingslips/get/cmd_test.go index 5e15a724ce..ba3025a7a6 100644 --- a/cmds/ocm/commands/ocmcmds/routingslips/get/cmd_test.go +++ b/cmds/ocm/commands/ocmcmds/routingslips/get/cmd_test.go @@ -49,6 +49,7 @@ var _ = Describe("Test Environment", func() { defer Close(cv) e1a = Must(routingslip.AddEntry(cv, PROVIDER, rsa.Algorithm, comment.New("first entry"), nil)) + MustBeSuccessful(cv.Update()) }) AfterEach(func() { @@ -66,6 +67,7 @@ var _ = Describe("Test Environment", func() { slip := Must(routingslip.GetSlip(cv, PROVIDER)) slip.Get(0).Signature.Value = slip.Get(0).Signature.Value[1:] + "0" MustBeSuccessful(routingslip.SetSlip(cv, slip)) + MustBeSuccessful(cv.Update()) MustBeSuccessful(finalize.Finalize()) buf := bytes.NewBuffer(nil) @@ -110,6 +112,8 @@ test.de/x:v1 acme.org comment ` + e1a.Timestamp.String() + ` Comment: first "status", "passed", )) e2c = Must(routingslip.AddEntry(cv, OTHER, rsa.Algorithm, te, nil)) + + MustBeSuccessful(cv.Update()) }) It("gets different slips", func() { @@ -168,6 +172,7 @@ acme.org/test ` + digests(e2c, e2b) + ` name: unit-tests, status: passed Name: PROVIDER, Digest: e1a.Digest, }})) + MustBeSuccessful(cv.Update()) }) It("gets dedicated wide slip with link", func() { buf := bytes.NewBuffer(nil) diff --git a/components/ecrplugin/Makefile b/components/ecrplugin/Makefile index c8eaf5648e..cb4dc4cb39 100644 --- a/components/ecrplugin/Makefile +++ b/components/ecrplugin/Makefile @@ -60,6 +60,11 @@ $(GEN)/ca.done: $(GEN)/.exists $(GEN)/build resources.yaml $(CHARTSRCS) $(OCM) add resources --templater=spiff --file $(GEN)/ca NAME="$(NAME)" VERSION="$(VERSION)" COMMIT="$(COMMIT)" GEN="$(GEN)" PLATFORMS="$(PLATFORMS)" resources.yaml @touch $(GEN)/ca.done +.PHONY: plain-ca +plain-ca: $(GEN)/.exists resources.yaml $(CHARTSRCS) + $(OCM) create ca -f $(COMPONENT) "$(VERSION)" --provider $(PROVIDER) --file $(GEN)/ca + $(OCM) add resources --templater=spiff --file $(GEN)/ca NAME="$(NAME)" VERSION="$(VERSION)" COMMIT="$(COMMIT)" GEN="$(GEN)" PLATFORMS="$(PLATFORMS)" resources.yaml + .PHONY: push push: $(GEN)/ctf $(GEN)/push.$(NAME) diff --git a/components/helminstaller/Makefile b/components/helminstaller/Makefile index d33ad2bab9..a58c11f337 100644 --- a/components/helminstaller/Makefile +++ b/components/helminstaller/Makefile @@ -56,7 +56,7 @@ $(GEN)/ca: $(GEN)/.exists $(GEN)/image.$(NAME)$(FLAGSUF) resources.yaml executor .PHONY: plain-ca plain-ca: $(GEN)/.exists - $(OCM) create ca -f $(COMPONENT) "$(VERSION)"--provider $(PROVIDER) --file $(GEN)/ca + $(OCM) create ca -f $(COMPONENT) "$(VERSION)" --provider $(PROVIDER) --file $(GEN)/ca $(OCM) add resources --templater spiff $(GEN)/ca $(ATTRIBUTES) resources.yaml @touch $(GEN)/ca diff --git a/components/helminstaller/a.yaml b/components/helminstaller/a.yaml new file mode 100644 index 0000000000..21ceec38d8 --- /dev/null +++ b/components/helminstaller/a.yaml @@ -0,0 +1 @@ +flag: (( flag ? "yes" :"no" )) diff --git a/components/helminstaller/resources.yaml b/components/helminstaller/resources.yaml index e3db4cc04a..9a6f2ef0eb 100644 --- a/components/helminstaller/resources.yaml +++ b/components/helminstaller/resources.yaml @@ -3,10 +3,10 @@ name: toiimage type: ociImage version: (( values.VERSION )) input: - type: (( values.MULTI ? "dockermulti" :"docker" )) + type: (( bool(values.MULTI) ? "dockermulti" :"docker" )) repository: (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE )) - variants: (( values.MULTI ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) - path: (( !values.MULTI ? values.IMAGE :~~ )) + variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) + path: (( !bool(values.MULTI) ? values.IMAGE :~~ )) --- name: toiexecutor type: toiExecutor diff --git a/components/ocmcli/resources.yaml b/components/ocmcli/resources.yaml index d60093d235..312ab25ff4 100644 --- a/components/ocmcli/resources.yaml +++ b/components/ocmcli/resources.yaml @@ -21,10 +21,10 @@ helper: type: ociImage version: (( values.VERSION )) input: - type: (( values.MULTI ? "dockermulti" :"docker" )) + type: (( bool(values.MULTI) ? "dockermulti" :"docker" )) repository: (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE )) - variants: (( values.MULTI ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) - path: (( !values.MULTI ? values.IMAGE :~~ )) + variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) + path: (( !bool(values.MULTI) ? values.IMAGE :~~ )) resources: (( map[split(" ", values.PLATFORMS)|p|-> *helper.executable] *helper.image )) diff --git a/examples/lib/area-ocm.png b/examples/lib/area-ocm.png new file mode 100755 index 0000000000..9a6379b392 Binary files /dev/null and b/examples/lib/area-ocm.png differ diff --git a/examples/lib/contexts.md b/examples/lib/contexts.md index b135b0ea73..6a61614ec2 100644 --- a/examples/lib/contexts.md +++ b/examples/lib/contexts.md @@ -19,18 +19,21 @@ A context provides access to a set of basic root elements of the functional area it is responsible for. The root elements can then be used to access nested or dependent objects. +![context](functionalarea.png) + The basic elements of most context types are *Specifications* and *Repositories*. A specification object provides serializable attributes used to describe dedicated elements in the functional area. They are typed. There might be different types (with different attribute sets) used to -describe instances provided by different implementations. +describe instances provided by different implementations. Those descriptions +are called descriptors. The root element below the context object is typically a *Repository* object, which provides access to elements hosted by this repository. The context itself manages all the specification and element types and -acts as an entry point to deserialize JSON/YAML based specification -descriptions and to gain access to the described effective root elements. +acts as a central entry point to deserialize JSON/YAML based specification +descriptors and to gain access to the described effective root elements. Those context extension points are managed by *Registries* used to configure implementations and specification types by globally unique names. These registration features are provided for every extension @@ -52,7 +55,7 @@ complex object ecosystem, including more kinds of specification and object types. Typically, a context is based on contexts of required functional areas, -for example an OCI contexts uses a credential context to gain access to +for example an OCI context uses a credential context to gain access to required credentials, which again uses a configuration context to configure itself from a central configuration provided by a shared configuration context. @@ -61,10 +64,31 @@ configure itself from a central configuration provided by a shared configuration The following context types are provided: - `config`: configuration management of all parts of the OCM library. -- `credentials`: credential management -- `oci`: working with OCI registries -- `ocm`: working withOCM repositories -- `clictx`: command line interface + It provides a uniform but generic way to manage configuration and its + serialized form for all kinds of configuration cosumers by supporting + typed descriptors which, are mapped to objects capable to configure + dedicated configuration targets. +- `credentials`: credential management. It acts as factory to provide + credentials for consumption targets in various environments (for example, + GitHub, OCI registries, S3 repositories, etc). Those targets are described + by so-called *comsumer ids*, which are mapped to credentials by the credentials context. + + It includes a *config* context. +- `oci`: working with OCI registries. + It acts as central access point to instantiate OCI registry view for + different backends hosting OCI images and OCU artifacts, like an + OCI registry, a filesystem representation or a docker daemon. + + It includes a *credentials* context. + +- **`ocm`**: working withOCM repositories. + This is the central context type providing access to the elements + of the open component model. + + It includes an *oci* context. +- `clictx`: command line interface. + It acts as configuration container for the OCM command line client, + hosting an OCM context. - `datacontext`: base context functionality used for all kinds of contexts. To just use the library without special configuration the complete standard @@ -78,7 +102,8 @@ All functional areas supported by contexts can be found as sub packages of `github.com/open-component-model/ocm/pkg/contexts`. A context package directly contains the typical user API for the functional -area. The most important interface is the interface `Context`. It acts as main entry point to access the functionality of the functional area. +area. The most important interface is the interface `Context`. It acts as main +entry point to access the functionality of the functional area. Elements required to provide own extension point implementations (for example new specification and repository types) can be found in the @@ -100,6 +125,51 @@ If a context type supports multiple extension types there is typically a dedicated sub package for this type (for example `github.com/open-component-model/pkg/contexts/ocm/accessmethods`), which again contains the various implementation types in sub packages. +### The OCM context organization + +The following diagram shows the element structured used for the OCM context as +an instance of the general context blueprint described above. + +![ocm context](area-ocm.png) + +Besides the standard sub packages, there are extension packages for + +- `blobhandler`: Handlers responsible to upload local blobs. By default, + local blobs are kept as local blobs when addedd to an OCM repository. + *Upload handlers* can be used to influence this behaviour and export + the content of dedicated blobs again into local repositories according + to their native technologies, for example OCI images, S3 blobs, etc. + +- `download`: Handlers understanding dedicated artifact and mime types + capable to provide then as filesystem content consumable by their + native tool set, for example Helm charts. + +- `labels`: The handling of predefined label content. +- `valuemergehandler`: Merge algorithms used to support the merge + of volatile labels during a transport step (used by the transport tool) + +The package `plugin` contains the plugin interface of the library, +which can be used to provide various kinds of extension points in form of +separate executables. +The embedding of handlers provided by plugins can always be found +in the package `plugin` below the dedicated extension point handler +package. + +Additionally, there are more functional +packages hosting predefined applications based on the open component model: + +- `transfer`: A generic transport tool, which can be used to transport + component versions from one repository environment to another one. + It is completely configurable by *transport handler* and *blob upload handlers* (configured as extensions in the used OCM context). A transport + handler may control the recursion behaviour for following component + references and the way described artifacts are transported into the target + environment. +- `signing`: a standard tool used to sign and verify component versions using different + signing, hashing and normalization procedures. + +- `routingslip`: A standard application using OCM labels to implement + routing slips. + ## Getting access to a context object There is always a `DefaultContext()` in the context package of a diff --git a/examples/lib/functionalarea.png b/examples/lib/functionalarea.png new file mode 100755 index 0000000000..d415026a1c Binary files /dev/null and b/examples/lib/functionalarea.png differ diff --git a/pkg/common/accessio/blobaccess/compress.go b/pkg/common/accessio/blobaccess/compress.go new file mode 100644 index 0000000000..3270b2b7e5 --- /dev/null +++ b/pkg/common/accessio/blobaccess/compress.go @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blobaccess + +import ( + "bytes" + "compress/gzip" + "io" + "sync" + + "github.com/opencontainers/go-digest" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + compression2 "github.com/open-component-model/ocm/pkg/common/compression" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/mime" +) + +//////////////////////////////////////////////////////////////////////////////// + +type compression struct { + blob BlobAccess +} + +var _ spi.BlobAccessBase = (*compression)(nil) + +func (c *compression) Close() error { + return c.blob.Close() +} + +func (c *compression) Get() ([]byte, error) { + r, err := c.blob.Reader() + if err != nil { + return nil, err + } + defer r.Close() + rr, _, err := compression2.AutoDecompress(r) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + + w := gzip.NewWriter(buf) + _, err = io.Copy(w, rr) + w.Close() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +type reader struct { + wait sync.WaitGroup + io.ReadCloser + err error +} + +func (r *reader) Close() error { + err := r.ReadCloser.Close() + r.wait.Wait() + return errors.Join(err, r.err) +} + +func (c *compression) Reader() (io.ReadCloser, error) { + r, err := c.blob.Reader() + if err != nil { + return nil, err + } + defer r.Close() + rr, _, err := compression2.AutoDecompress(r) + if err != nil { + return nil, err + } + pr, pw := io.Pipe() + cw := gzip.NewWriter(pw) + + outr := &reader{ReadCloser: pr} + outr.wait.Add(1) + + go func() { + _, err := io.Copy(cw, rr) + outr.err = errors.Join(err, cw.Close(), pw.Close()) + outr.wait.Done() + }() + return outr, nil +} + +func (c *compression) Digest() digest.Digest { + return BLOB_UNKNOWN_DIGEST +} + +func (c *compression) MimeType() string { + m := c.blob.MimeType() + if mime.IsGZip(m) { + return m + } + return m + "+gzip" +} + +func (c *compression) DigestKnown() bool { + return false +} + +func (c *compression) Size() int64 { + return BLOB_UNKNOWN_SIZE +} + +func WithCompression(blob BlobAccess) (BlobAccess, error) { + b, err := blob.Dup() + if err != nil { + return nil, err + } + return spi.NewBlobAccessForBase(&compression{ + blob: b, + }), nil +} + +//////////////////////////////////////////////////////////////////////////////// + +type decompression struct { + blob BlobAccess +} + +var _ spi.BlobAccessBase = (*decompression)(nil) + +func (c *decompression) Close() error { + return c.blob.Close() +} + +func (c *decompression) Get() ([]byte, error) { + r, err := c.blob.Reader() + if err != nil { + return nil, err + } + defer r.Close() + rr, _, err := compression2.AutoDecompress(r) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + _, err = io.Copy(buf, rr) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (c *decompression) Reader() (io.ReadCloser, error) { + r, err := c.blob.Reader() + if err != nil { + return nil, err + } + defer r.Close() + rr, _, err := compression2.AutoDecompress(r) + return rr, err +} + +func (c *decompression) Digest() digest.Digest { + return BLOB_UNKNOWN_DIGEST +} + +func (c *decompression) MimeType() string { + m := c.blob.MimeType() + if !mime.IsGZip(m) { + return m + } + return m[:len(m)-5] +} + +func (c *decompression) DigestKnown() bool { + return false +} + +func (c *decompression) Size() int64 { + return BLOB_UNKNOWN_SIZE +} + +func WithDecompression(blob BlobAccess) (BlobAccess, error) { + b, err := blob.Dup() + if err != nil { + return nil, err + } + return spi.NewBlobAccessForBase(&decompression{ + blob: b, + }), nil +} diff --git a/pkg/common/accessio/blobaccess/compress_test.go b/pkg/common/accessio/blobaccess/compress_test.go new file mode 100644 index 0000000000..6b2c207f28 --- /dev/null +++ b/pkg/common/accessio/blobaccess/compress_test.go @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blobaccess_test + +import ( + "bytes" + "compress/gzip" + "io" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/mime" +) + +var _ = Describe("temp file management", func() { + + Context("compress", func() { + It("compress access", func() { + blob := blobaccess.ForString(mime.MIME_TEXT, "testdata") + defer blob.Close() + + comp := Must(blobaccess.WithCompression(blob)) + defer comp.Close() + + Expect(comp.MimeType()).To(Equal(mime.MIME_TEXT + "+gzip")) + data := Must(comp.Get()) + Expect(len(data)).To(Not(Equal(8))) + + uncomp := Must(io.ReadAll(Must(gzip.NewReader(bytes.NewReader(data))))) + Expect(string(uncomp)).To(Equal("testdata")) + }) + + It("compress reader access", func() { + blob := blobaccess.ForString(mime.MIME_TEXT, "testdata") + defer blob.Close() + + comp := Must(blobaccess.WithCompression(blob)) + defer comp.Close() + + r := Must(comp.Reader()) + data := Must(io.ReadAll(r)) + Expect(len(data)).To(Not(Equal(8))) + + uncomp := Must(io.ReadAll(Must(gzip.NewReader(bytes.NewReader(data))))) + Expect(string(uncomp)).To(Equal("testdata")) + }) + }) + + Context("uncompress", func() { + buf := bytes.NewBuffer(nil) + cw := gzip.NewWriter(buf) + MustBeSuccessful(io.WriteString(cw, "testdata")) + cw.Close() + + It("uncompress access", func() { + blob := blobaccess.ForData(mime.MIME_TEXT+"+gzip", buf.Bytes()) + defer blob.Close() + + comp := Must(blobaccess.WithDecompression(blob)) + defer comp.Close() + Expect(comp.MimeType()).To(Equal(mime.MIME_TEXT)) + + data := Must(comp.Get()) + Expect(string(data)).To(Equal("testdata")) + }) + + It("compress reader access", func() { + blob := blobaccess.ForData(mime.MIME_TEXT+"+gzip", buf.Bytes()) + defer blob.Close() + + comp := Must(blobaccess.WithDecompression(blob)) + defer comp.Close() + + r := Must(comp.Reader()) + data := Must(io.ReadAll(r)) + Expect(string(data)).To(Equal("testdata")) + }) + }) + +}) diff --git a/pkg/common/accessio/blobaccess/dirtree/access.go b/pkg/common/accessio/blobaccess/dirtree/access.go index 3058930859..2ba6163f75 100644 --- a/pkg/common/accessio/blobaccess/dirtree/access.go +++ b/pkg/common/accessio/blobaccess/dirtree/access.go @@ -10,6 +10,7 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/utils" "github.com/open-component-model/ocm/pkg/utils/tarutils" ) @@ -23,12 +24,9 @@ func DataAccessForDirTree(path string, opts ...Option) (spi.DataAccess, error) { } func BlobAccessForDirTree(path string, opts ...Option) (_ spi.BlobAccess, rerr error) { - var eff Options - for _, opt := range opts { - opt.ApplyToDirtreeOptions(&eff) - } - + eff := optionutils.EvalOptions(opts...) fs := utils.FileSystem(eff.FileSystem) + ok, err := vfs.IsDir(fs, path) if err != nil { return nil, err @@ -71,3 +69,9 @@ func BlobAccessForDirTree(path string, opts ...Option) (_ spi.BlobAccess, rerr e } return temp.AsBlob(eff.MimeType), nil } + +func BlobAccessProviderForDirTree(path string, opts ...Option) spi.BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + return BlobAccessForDirTree(path, opts...) + }) +} diff --git a/pkg/common/accessio/blobaccess/dirtree/options.go b/pkg/common/accessio/blobaccess/dirtree/options.go index eeed8108c9..5793b9e60d 100644 --- a/pkg/common/accessio/blobaccess/dirtree/options.go +++ b/pkg/common/accessio/blobaccess/dirtree/options.go @@ -4,12 +4,11 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "golang.org/x/exp/slices" + "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/utils" ) -type Option interface { - ApplyToDirtreeOptions(*Options) -} +type Option = optionutils.Option[*Options] type Options struct { // FileSystem defines the file system that contains the specified directory. @@ -31,7 +30,7 @@ type Options struct { FollowSymlinks *bool `json:"followSymlinks,omitempty"` } -func (o *Options) ApplyToDirtreeOptions(opts *Options) { +func (o *Options) ApplyTo(opts *Options) { if opts == nil { return } @@ -58,11 +57,13 @@ func (o *Options) ApplyToDirtreeOptions(opts *Options) { } } +//////////////////////////////////////////////////////////////////////////////// + type fileSystem struct { fs vfs.FileSystem } -func (o *fileSystem) ApplyToDirtreeOptions(opts *Options) { +func (o *fileSystem) ApplyTo(opts *Options) { opts.FileSystem = o.fs } @@ -70,9 +71,11 @@ func WithFileSystem(fs vfs.FileSystem) Option { return &fileSystem{fs: fs} } +//////////////////////////////////////////////////////////////////////////////// + type mimeType string -func (o mimeType) ApplyToDirtreeOptions(opts *Options) { +func (o mimeType) ApplyTo(opts *Options) { opts.MimeType = string(o) } @@ -82,7 +85,7 @@ func WithMimeType(mime string) Option { type compressWithGzip bool -func (o compressWithGzip) ApplyToDirtreeOptions(opts *Options) { +func (o compressWithGzip) ApplyTo(opts *Options) { opts.CompressWithGzip = utils.BoolP(o) } @@ -92,7 +95,7 @@ func WithCompressWithGzip(b ...bool) Option { type preserveDir bool -func (o preserveDir) ApplyToDirtreeOptions(opts *Options) { +func (o preserveDir) ApplyTo(opts *Options) { opts.PreserveDir = utils.BoolP(o) } @@ -102,7 +105,7 @@ func WithPreserveDir(b ...bool) Option { type includeFiles []string -func (o includeFiles) ApplyToDirtreeOptions(opts *Options) { +func (o includeFiles) ApplyTo(opts *Options) { opts.IncludeFiles = slices.Clone(o) } @@ -112,7 +115,7 @@ func WithIncludeFiles(files []string) Option { type excludeFiles []string -func (o excludeFiles) ApplyToDirtreeOptions(opts *Options) { +func (o excludeFiles) ApplyTo(opts *Options) { opts.ExcludeFiles = slices.Clone(o) } @@ -122,7 +125,7 @@ func WithExcludeFiles(files []string) Option { type followSymlinks bool -func (o followSymlinks) ApplyToDirtreeOptions(opts *Options) { +func (o followSymlinks) ApplyTo(opts *Options) { opts.FollowSymlinks = utils.BoolP(o) } diff --git a/pkg/common/accessio/blobaccess/dockerdaemon/access.go b/pkg/common/accessio/blobaccess/dockerdaemon/access.go new file mode 100644 index 0000000000..3f315bb623 --- /dev/null +++ b/pkg/common/accessio/blobaccess/dockerdaemon/access.go @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dockerdaemon + +import ( + "fmt" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/oci/annotations" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +func (o *Options) OCIContext() oci.Context { + if o.Context == nil { + return oci.DefaultContext() + } + return o.Context +} + +func ImageInfoFor(name string, opts ...Option) (locator string, version string, err error) { + eff := optionutils.EvalOptions(opts...) + + locator, version, err = docker.ParseGenericRef(name) + if err != nil { + return "", "", err + } + + if version == "" || version == "latest" || optionutils.AsValue(eff.OverrideVersion) { + version = eff.Version + } + if version == "" { + return "", "", fmt.Errorf("no version specified") + } + return locator, version, nil +} + +func BlobAccessProviderForImageFromDockerDaemon(name string, opts ...Option) spi.BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + b, _, err := BlobAccessForImageFromDockerDaemon(name, opts...) + return b, err + }) +} + +func BlobAccessForImageFromDockerDaemon(name string, opts ...Option) (blobaccess.BlobAccess, string, error) { + eff := optionutils.EvalOptions(opts...) + ctx := eff.OCIContext() + + locator, version, err := ImageInfoFor(name, eff) + if err != nil { + return nil, "", err + } + spec := docker.NewRepositorySpec() + repo, err := ctx.RepositoryForSpec(spec) + if err != nil { + return nil, "", err + } + ns, err := repo.LookupNamespace(locator) + if err != nil { + return nil, "", err + } + blob, err := artifactset.SynthesizeArtifactBlob(ns, version, + func(art oci.ArtifactAccess) error { + if eff.Origin != nil { + art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, eff.Origin.String()) + } + return nil + }, + ) + if err != nil { + return nil, "", err + } + return blob, version, nil +} diff --git a/pkg/common/accessio/blobaccess/dockerdaemon/options.go b/pkg/common/accessio/blobaccess/dockerdaemon/options.go new file mode 100644 index 0000000000..f44bb884fa --- /dev/null +++ b/pkg/common/accessio/blobaccess/dockerdaemon/options.go @@ -0,0 +1,109 @@ +package dockerdaemon + +import ( + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/optionutils" + "github.com/open-component-model/ocm/pkg/utils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + Context oci.Context + Name string + Version string + OverrideVersion *bool + Origin *common.NameVersion +} + +func (o *Options) ApplyTo(opts *Options) { + if opts == nil { + return + } + if o.Context != nil { + opts.Context = o.Context + } + if o.Name != "" { + opts.Name = o.Name + } + if o.Version != "" { + opts.Version = o.Version + } + if o.OverrideVersion != nil { + opts.OverrideVersion = o.OverrideVersion + } + if o.Origin != nil { + opts.Origin = o.Origin + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type context struct { + oci.Context +} + +func (o context) ApplyTo(opts *Options) { + opts.Context = o +} + +func WithContext(ctx oci.ContextProvider) Option { + return context{ctx.OCIContext()} +} + +//////////////////////////////////////////////////////////////////////////////// + +type name string + +func (o name) ApplyTo(opts *Options) { + opts.Name = string(o) +} + +func WithName(n string) Option { + return name(n) +} + +//////////////////////////////////////////////////////////////////////////////// + +type version string + +func (o version) ApplyTo(opts *Options) { + opts.Version = string(o) +} + +func WithVersion(v string) Option { + return version(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type override struct { + flag bool + version string +} + +func (o *override) ApplyTo(opts *Options) { + opts.OverrideVersion = utils.BoolP(o.flag) + opts.Version = o.version +} + +func WithVersionOverride(v string, flag ...bool) Option { + return &override{ + version: v, + flag: utils.OptionalDefaultedBool(true, flag...), + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type compvers common.NameVersion + +func (o compvers) ApplyTo(opts *Options) { + n := common.NameVersion(o) + opts.Origin = &n +} + +func WithOrigin(o common.NameVersion) Option { + return compvers(o) +} diff --git a/pkg/common/accessio/blobaccess/dockermulti/access.go b/pkg/common/accessio/blobaccess/dockermulti/access.go new file mode 100644 index 0000000000..44e76db3e3 --- /dev/null +++ b/pkg/common/accessio/blobaccess/dockermulti/access.go @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dockermulti + +import ( + "fmt" + + . "github.com/open-component-model/ocm/pkg/finalizer" + + "github.com/opencontainers/go-digest" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/oci/annotations" + "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" + "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +func (o *Options) OCIContext() oci.Context { + if o.Context == nil { + return oci.DefaultContext() + } + return o.Context +} + +func (s *Options) getVariant(ctx oci.Context, finalize *Finalizer, variant string) (oci.ArtifactAccess, error) { + locator, version, err := docker.ParseGenericRef(variant) + if err != nil { + return nil, err + } + if version == "" { + return nil, fmt.Errorf("artifact version required") + } + spec := docker.NewRepositorySpec() + repo, err := ctx.RepositoryForSpec(spec) + if err != nil { + return nil, err + } + finalize.Close(repo) + ns, err := repo.LookupNamespace(locator) + if err != nil { + return nil, err + } + finalize.Close(ns) + + art, err := ns.GetArtifact(version) + if err != nil { + return nil, artifactset.GetArtifactError{Original: err, Ref: locator + ":" + version} + } + finalize.Close(art) + return art, nil +} + +func BlobAccessForMultiImageFromDockerDaemon(opts ...Option) (blobaccess.BlobAccess, error) { + eff := optionutils.EvalOptions(opts...) + ctx := eff.OCIContext() + + index := artdesc.NewIndexArtifact() + i := 0 + + version := eff.Version + if eff.Origin != nil { + if version == "" { + version = eff.Origin.GetVersion() + } + index.SetAnnotation(annotations.COMPVERS_ANNOTATION, eff.Origin.String()) + } + if version == "" { + return nil, fmt.Errorf("no versio specified") + } + + feedback := func(blob blobaccess.BlobAccess, art cpi.ArtifactAccess) error { + desc := artdesc.DefaultBlobDescriptor(blob) + if art.IsManifest() { + cfgBlob, err := art.ManifestAccess().GetConfigBlob() + if err != nil { + return errors.Wrapf(err, "cannot get config blob") + } + cfg, err := artdesc.ParseImageConfig(cfgBlob) + if err != nil { + return errors.Wrapf(err, "cannot parse config blob") + } + if cfg.Architecture != "" { + desc.Platform = &artdesc.Platform{ + Architecture: cfg.Architecture, + OS: cfg.OS, + Variant: cfg.Variant, + } + } + } + index.Index().AddManifest(desc) + return nil + } + + blob, err := artifactset.SynthesizeArtifactBlobFor(version, func() (fac artifactset.ArtifactFactory, main bool, err error) { + var art cpi.ArtifactAccess + var blob blobaccess.BlobAccess + + switch { + case i > len(eff.Variants): + // end loop + case i == len(eff.Variants): + // provide index (main) artifact + if eff.Printer != nil { + eff.Printer.Printf("image %d: INDEX\n", i) + } + fac = func(set *artifactset.ArtifactSet) (digest.Digest, string, error) { + art, err = set.NewArtifact(index) + if err != nil { + return "", "", errors.Wrapf(err, "cannot create index artifact") + } + defer art.Close() + blob, err = set.AddArtifact(art) + if err != nil { + return "", "", errors.Wrapf(err, "cannot add index artifact") + } + defer blob.Close() + return blob.Digest(), blob.MimeType(), nil + } + main = true + default: + // provide variant + if eff.Printer != nil { + eff.Printer.Printf("image %d: %s\n", i, eff.Variants[i]) + } + var finalize Finalizer + + art, err = eff.getVariant(ctx, &finalize, eff.Variants[i]) + + if err == nil { + if eff.Origin != nil { + art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, eff.Origin.String()) + } + blob, err = art.Blob() + if err == nil { + finalize.Close(art) + fac = artifactset.ArtifactTransferCreator(art, &finalize, feedback) + } + } + } + i++ + return + }) + if err != nil { + return nil, err + } + return blob, nil +} + +func BlobAccessProviderForMultiImageFromDockerDaemon(opts ...Option) spi.BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + return BlobAccessForMultiImageFromDockerDaemon(opts...) + }) +} diff --git a/pkg/common/accessio/blobaccess/dockermulti/options.go b/pkg/common/accessio/blobaccess/dockermulti/options.go new file mode 100644 index 0000000000..ef46077973 --- /dev/null +++ b/pkg/common/accessio/blobaccess/dockermulti/options.go @@ -0,0 +1,93 @@ +package dockermulti + +import ( + "golang.org/x/exp/slices" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + Context oci.Context + Version string + Variants []string + Origin *common.NameVersion + Printer common.Printer +} + +func (o *Options) ApplyTo(opts *Options) { + if opts == nil { + return + } + if o.Version != "" { + opts.Version = o.Version + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type context struct { + oci.Context +} + +func (o context) ApplyTo(opts *Options) { + opts.Context = o +} + +func WithContext(ctx oci.ContextProvider) Option { + return context{ctx.OCIContext()} +} + +//////////////////////////////////////////////////////////////////////////////// + +type version string + +func (o version) ApplyTo(opts *Options) { + opts.Version = string(o) +} + +func WithVersion(v string) Option { + return version(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type compvers common.NameVersion + +func (o compvers) ApplyTo(opts *Options) { + n := common.NameVersion(o) + opts.Origin = &n +} + +func WithOrigin(o common.NameVersion) Option { + return compvers(o) +} + +//////////////////////////////////////////////////////////////////////////////// + +type variants []string + +func (o variants) ApplyTo(opts *Options) { + opts.Variants = append(opts.Variants, []string(o)...) +} + +func WithVariants(v ...string) Option { + return variants(slices.Clone(v)) +} + +//////////////////////////////////////////////////////////////////////////////// + +type printer struct { + common.Printer +} + +func (o printer) ApplyTo(opts *Options) { + opts.Printer = o +} + +func WithPrinter(p common.Printer) Option { + return printer{p} +} diff --git a/pkg/common/accessio/blobaccess/helm/options.go b/pkg/common/accessio/blobaccess/helm/options.go new file mode 100644 index 0000000000..39930e06d7 --- /dev/null +++ b/pkg/common/accessio/blobaccess/helm/options.go @@ -0,0 +1,164 @@ +package helm + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/optionutils" + "github.com/open-component-model/ocm/pkg/utils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + Context oci.Context + FileSystem vfs.FileSystem + Version string + OverrideVersion *bool + HelmRepository string + CACert string + CACertFile string + + Printer common.Printer +} + +func (o *Options) ApplyTo(opts *Options) { + if opts == nil { + return + } + if o.Context != nil { + opts.Context = o.Context + } + if o.FileSystem != nil { + opts.FileSystem = o.FileSystem + } + if o.Version != "" { + opts.Version = o.Version + } + if o.OverrideVersion != nil { + opts.OverrideVersion = o.OverrideVersion + } + if o.HelmRepository != "" { + opts.HelmRepository = o.HelmRepository + } + if o.CACert != "" { + opts.CACert = o.CACert + } + if o.CACertFile != "" { + opts.CACertFile = o.CACertFile + } + if o.Printer != nil { + opts.Printer = o.Printer + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type context struct { + oci.Context +} + +func (o context) ApplyTo(opts *Options) { + opts.Context = o +} + +func WithContext(ctx oci.ContextProvider) Option { + return context{ctx.OCIContext()} +} + +//////////////////////////////////////////////////////////////////////////////// + +type fileSystem struct { + fs vfs.FileSystem +} + +func (o *fileSystem) ApplyTo(opts *Options) { + opts.FileSystem = o.fs +} + +func WithFileSystem(fs vfs.FileSystem) Option { + return &fileSystem{fs: fs} +} + +//////////////////////////////////////////////////////////////////////////////// + +type version string + +func (o version) ApplyTo(opts *Options) { + opts.Version = string(o) +} + +func WithVersion(v string) Option { + return version(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type override struct { + flag bool + version string +} + +func (o *override) ApplyTo(opts *Options) { + opts.OverrideVersion = utils.BoolP(o.flag) + opts.Version = o.version +} + +func WithVersionOverride(v string, flag ...bool) Option { + return &override{ + version: v, + flag: utils.OptionalDefaultedBool(true, flag...), + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type helmrepo string + +func (o helmrepo) ApplyTo(opts *Options) { + opts.HelmRepository = string(o) +} + +// WithHelmRepository defines the helm repository to read from. +func WithHelmRepository(v string) Option { + return helmrepo(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type cacert string + +func (o cacert) ApplyTo(opts *Options) { + opts.CACert = string(o) +} + +func WithCACert(v string) Option { + return cacert(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type cacertfile string + +func (o cacertfile) ApplyTo(opts *Options) { + opts.CACertFile = string(o) +} + +func WithCACertFile(v string) Option { + return cacertfile(v) +} + +//////////////////////////////////////////////////////////////////////////////// + +type printer struct { + common.Printer +} + +func (o printer) ApplyTo(opts *Options) { + opts.Printer = o +} + +func WithPrinter(p common.Printer) Option { + return printer{p} +} diff --git a/pkg/common/accessio/blobaccess/helm/resource.go b/pkg/common/accessio/blobaccess/helm/resource.go new file mode 100644 index 0000000000..7e08440d85 --- /dev/null +++ b/pkg/common/accessio/blobaccess/helm/resource.go @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package helm + +import ( + "fmt" + + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + "github.com/open-component-model/ocm/pkg/contexts/oci" + ocihelm "github.com/open-component-model/ocm/pkg/contexts/oci/ociutils/helm" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/helm" + "github.com/open-component-model/ocm/pkg/helm/identity" + "github.com/open-component-model/ocm/pkg/helm/loader" + "github.com/open-component-model/ocm/pkg/optionutils" + "github.com/open-component-model/ocm/pkg/utils" +) + +func (o *Options) OCIContext() oci.Context { + if o.Context == nil { + return oci.DefaultContext() + } + return o.Context +} + +func BlobAccessForHelmChart(path string, opts ...Option) (blob blobaccess.BlobAccess, name, version string, err error) { + eff := optionutils.EvalOptions(opts...) + ctx := eff.OCIContext() + fs := utils.FileSystem(eff.FileSystem) + printer := eff.Printer + if printer == nil { + printer = common.NewPrinter(nil) + } + + var chartLoader loader.Loader + if eff.HelmRepository == "" { + if ok, err := vfs.Exists(fs, path); !ok || err != nil { + return nil, "", "", errors.NewEf(err, "invalid file path %q", path) + } + chartLoader = loader.VFSLoader(path, fs) + } else { + cert := []byte(eff.CACert) + if eff.CACertFile != "" { + cert, err = vfs.ReadFile(fs, eff.CACertFile) + if err != nil { + return nil, "", "", errors.Wrapf(err, "cannot read root certificates from %q", eff.CACertFile) + } + } + + acc, err := helm.DownloadChart(printer, ctx, path, eff.Version, eff.HelmRepository, + helm.WithCredentials(identity.GetCredentials(ctx, eff.HelmRepository, path)), + helm.WithRootCert(cert)) + if err != nil { + return nil, "", "", errors.Wrapf(err, "cannot download chart %s:%s from %s", path, eff.Version, eff.HelmRepository) + } + chartLoader = loader.AccessLoader(acc) + } + + defer errors.PropagateError(&err, chartLoader.Close) + + chart, err := chartLoader.Chart() + if err != nil { + return nil, "", "", err + } + vers := chart.Metadata.Version + if vers == "" || optionutils.AsValue(eff.OverrideVersion) { + vers = eff.Version + } + if vers == "" { + return nil, "", "", fmt.Errorf("no version found or specified") + } + + blob, err = chartLoader.ChartArtefactSet() + if err == nil && blob == nil { + blob, err = ocihelm.SynthesizeArtifactBlob(chartLoader) + if err != nil { + return nil, "", "", errors.Wrapf(err, "cannot synthesize artifact blob") + } + } + return blob, chart.Name(), vers, err +} + +func BlobAccessProviderForHelmChart(name string, opts ...Option) spi.BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + b, _, _, err := BlobAccessForHelmChart(name, opts...) + return b, err + }) +} diff --git a/pkg/common/accessio/blobaccess/interface.go b/pkg/common/accessio/blobaccess/interface.go index ce2263683e..16aeae3ecf 100644 --- a/pkg/common/accessio/blobaccess/interface.go +++ b/pkg/common/accessio/blobaccess/interface.go @@ -23,7 +23,8 @@ type ( ) type ( - BlobAccess = internal.BlobAccess + BlobAccess = internal.BlobAccess + BlobAccessProvider = internal.BlobAccessProvider DigestSource = internal.DigestSource MimeType = internal.MimeType diff --git a/pkg/common/accessio/blobaccess/internal/interface.go b/pkg/common/accessio/blobaccess/internal/interface.go index f542113761..0c1b33dabe 100644 --- a/pkg/common/accessio/blobaccess/internal/interface.go +++ b/pkg/common/accessio/blobaccess/internal/interface.go @@ -80,3 +80,7 @@ type FileLocation interface { FileSystem() vfs.FileSystem Path() string } + +type BlobAccessProvider interface { + BlobAccess() (BlobAccess, error) +} diff --git a/pkg/common/accessio/blobaccess/spi/interface.go b/pkg/common/accessio/blobaccess/spi/interface.go index 9c6c3e48a7..5028f3abd9 100644 --- a/pkg/common/accessio/blobaccess/spi/interface.go +++ b/pkg/common/accessio/blobaccess/spi/interface.go @@ -22,11 +22,18 @@ var ErrClosed = refmgmt.ErrClosed type DataAccess = internal.DataAccess type ( - BlobAccess = internal.BlobAccess - BlobAccessBase = internal.BlobAccessBase + BlobAccess = internal.BlobAccess + BlobAccessBase = internal.BlobAccessBase + BlobAccessProvider = internal.BlobAccessProvider DigestSource = internal.DigestSource MimeType = internal.MimeType ) type FileLocation = internal.FileLocation + +type BlobAccessProviderFunction func() (BlobAccess, error) + +func (p BlobAccessProviderFunction) BlobAccess() (BlobAccess, error) { + return p() +} diff --git a/pkg/common/accessio/blobaccess/spi/utils.go b/pkg/common/accessio/blobaccess/spi/utils.go index 1621a27f75..789d026ba5 100644 --- a/pkg/common/accessio/blobaccess/spi/utils.go +++ b/pkg/common/accessio/blobaccess/spi/utils.go @@ -187,6 +187,10 @@ func (s *staticBlobAccess) Dup() (BlobAccess, error) { return s, nil } +func (s *staticBlobAccess) Close() error { + return nil +} + // ForStaticDataAccess is used for a data access using no closer. // They don't require a finalization and can be used // as long as they exist. Therefore, no ref counting diff --git a/pkg/common/accessio/blobaccess/spi/view.go b/pkg/common/accessio/blobaccess/spi/view.go index 58797f3a3b..078bf1601d 100644 --- a/pkg/common/accessio/blobaccess/spi/view.go +++ b/pkg/common/accessio/blobaccess/spi/view.go @@ -33,6 +33,10 @@ func (b *blobAccessView) base() BlobAccessBase { return b.access } +func (b *blobAccessView) Close() error { + return b.View.Close() +} + func (b *blobAccessView) Validate() error { return utils.ValidateObject(b.access) } @@ -41,9 +45,8 @@ func (b *blobAccessView) Get() (result []byte, err error) { return result, b.Execute(func() error { result, err = b.access.Get() if err != nil { - return fmt.Errorf("unable to get access: %w", err) + return err } - return nil }) } diff --git a/pkg/common/accessio/blobaccess/standard.go b/pkg/common/accessio/blobaccess/standard.go index 5a55f8dde4..87c0f2de94 100644 --- a/pkg/common/accessio/blobaccess/standard.go +++ b/pkg/common/accessio/blobaccess/standard.go @@ -10,6 +10,7 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" "github.com/open-component-model/ocm/pkg/common/accessio/refmgmt" "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/utils" ) @@ -44,14 +45,62 @@ func Validate(o BlobAccess) error { //////////////////////////////////////////////////////////////////////////////// +type blobprovider struct { + blob BlobAccess +} + +var _ BlobAccessProvider = (*blobprovider)(nil) + +func (b *blobprovider) BlobAccess() (BlobAccess, error) { + return b.blob.Dup() +} + +func (b *blobprovider) Close() error { + return b.blob.Close() +} + +// ProviderForBlobAccess provides subsequent bloc accesses +// as long as the given blob access is not closed. +// If required the blob can be closed with the additionally +// provided Close method. +// ATTENTION: the underlying BlobAccess wil not be closed +// as long as the provider is not closed, but the BlobProvider +// interface is no io.Closer. +// To be on the safe side, this method should only be called +// with static blob access, featuring a NOP closer without +// anny attached external resources, which should be released. +func ProviderForBlobAccess(blob BlobAccess) *blobprovider { + return &blobprovider{blob} +} + +//////////////////////////////////////////////////////////////////////////////// + // ForString wraps a string into a BlobAccess, which does not need a close. -func ForString(mime string, data string) BlobAccess { - return ForData(mime, []byte(data)) +func ForString(media string, data string) BlobAccess { + if media == "" { + media = mime.MIME_TEXT + } + return ForData(media, []byte(data)) +} + +func ProviderForString(mime, data string) BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + return ForString(mime, data), nil + }) } // ForData wraps data into a BlobAccess, which does not need a close. -func ForData(mime string, data []byte) BlobAccess { - return spi.ForStaticDataAccessAndMeta(mime, DataAccessForBytes(data), digest.FromBytes(data), int64(len(data))) +func ForData(media string, data []byte) BlobAccess { + if media == "" { + media = mime.MIME_OCTET + } + return spi.ForStaticDataAccessAndMeta(media, DataAccessForBytes(data), digest.FromBytes(data), int64(len(data))) +} + +func ProviderForData(mime string, data []byte) BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + return ForData(mime, data), nil + }) } type ( @@ -121,6 +170,12 @@ func ForFile(mime string, path string, fss ...vfs.FileSystem) BlobAccess { } } +func ProviderForFile(mime string, path string, fss ...vfs.FileSystem) BlobAccessProvider { + return spi.BlobAccessProviderFunction(func() (spi.BlobAccess, error) { + return ForFile(mime, path, fss...), nil + }) +} + type fileBlobAccessView struct { _blobAccess access *fileDataAccess diff --git a/pkg/common/accessio/refmgmt/refcloser.go b/pkg/common/accessio/refmgmt/refcloser.go index bdb9ad6bf1..9588f4d033 100644 --- a/pkg/common/accessio/refmgmt/refcloser.go +++ b/pkg/common/accessio/refmgmt/refcloser.go @@ -17,6 +17,8 @@ var ErrClosed = errors.ErrClosed() // If the last view is closed, the basic closer is finally closed. type ReferencableCloser interface { Allocatable + + RefCount() int UnrefLast() error IsClosed() bool @@ -66,7 +68,23 @@ type LazyMode interface { Lazy() } +// ToLazy resets the main view flag +// of closer views to enable +// dark release of resources even if the +// first/main view has been closed. +// Otherwise, closing the main view will +// fail, if there are still subsequent views. +func ToLazy[T any](o T, err error) (T, error) { + if err == nil { + Lazy(o) + } + return o, err +} + func Lazy(o interface{}) bool { + if o == nil { + return false + } if l, ok := o.(LazyMode); ok { l.Lazy() return true @@ -123,6 +141,9 @@ func (v *view) Execute(f func() error) error { return f() } +// Release will release the view. +// With releasing the last view +// the underlying object will be closed. func (v *view) Release() error { v.lock.Lock() defer v.lock.Unlock() @@ -133,6 +154,10 @@ func (v *view) Release() error { return v.ref.Unref() } +// Finalize will try to finalize the +// underlying object. This is only +// possible if no further view is +// still pending. func (v *view) Finalize() error { v.lock.Lock() defer v.lock.Unlock() diff --git a/pkg/common/accessio/refmgmt/refmgmt.go b/pkg/common/accessio/refmgmt/refmgmt.go index f964865b62..ea8154db25 100644 --- a/pkg/common/accessio/refmgmt/refmgmt.go +++ b/pkg/common/accessio/refmgmt/refmgmt.go @@ -25,6 +25,7 @@ type RefMgmt interface { Allocatable UnrefLast() error IsClosed() bool + RefCount() int WithName(name string) RefMgmt } @@ -95,6 +96,12 @@ func (c *refMgmt) Unref() error { return nil } +func (c *refMgmt) RefCount() int { + c.lock.Lock() + defer c.lock.Unlock() + return c.refcount +} + func (c *refMgmt) UnrefLast() error { c.lock.Lock() defer c.lock.Unlock() diff --git a/pkg/common/accessio/resource/resource.go b/pkg/common/accessio/resource/resource.go index 46e9bf16fa..936fc218af 100644 --- a/pkg/common/accessio/resource/resource.go +++ b/pkg/common/accessio/resource/resource.go @@ -55,6 +55,7 @@ type Dup[T any] interface { // ViewManager is the interface of the reference manager, which // can be used to gain new views to a managed resource. type ViewManager[T any] interface { + RefCount() int View(main ...bool) (T, error) IsClosed() bool } @@ -91,6 +92,10 @@ func NewResource[T any, I ResourceImplementation[T]](impl I, c ResourceViewCreat return t } +func (i *viewManager[T, I]) RefCount() int { + return i.refs.RefCount() +} + func (i *viewManager[T, I]) View(main ...bool) (T, error) { var _nil T @@ -217,6 +222,10 @@ func (b *ResourceImplBase[T]) SetViewManager(m ViewManager[T]) { b.refs = m } +func (b *ResourceImplBase[T]) RefCount() int { + return b.refs.RefCount() +} + func (b *ResourceImplBase[T]) View(main ...bool) (T, error) { return b.refs.View(main...) } diff --git a/pkg/contexts/oci/repositories/artifactset/artifactset.go b/pkg/contexts/oci/repositories/artifactset/artifactset.go index d01fefd0d2..2e0d4fd858 100644 --- a/pkg/contexts/oci/repositories/artifactset/artifactset.go +++ b/pkg/contexts/oci/repositories/artifactset/artifactset.go @@ -27,6 +27,19 @@ const ( OCITAG_ANNOTATION = "org.opencontainers.image.ref.name" ) +func RetrieveMainArtifactFromIndex(index *artdesc.Index) string { + if index.Annotations != nil { + main := index.Annotations[MAINARTIFACT_ANNOTATION] + if main != "" { + return main + } + } + if len(index.Manifests) == 1 { + return index.Manifests[0].Digest.String() + } + return "" +} + func RetrieveMainArtifact(m map[string]string) string { return m[MAINARTIFACT_ANNOTATION] } @@ -88,6 +101,12 @@ func (a *ArtifactSet) HasAnnotation(name string) bool { return a.container.HasAnnotation(name) } +func (a *ArtifactSet) SetMainArtifact(version string) { + if version != "" { + a.Annotate(MAINARTIFACT_ANNOTATION, version) + } +} + func AsArtifactSet(ns cpi.NamespaceAccess) (*ArtifactSet, error) { i, err := cpi.GetNamespaceAccessImplementation(ns) if err != nil { diff --git a/pkg/contexts/oci/repositories/ctf/synthesis_test.go b/pkg/contexts/oci/repositories/ctf/synthesis_test.go index e32a79296f..b53d19dec8 100644 --- a/pkg/contexts/oci/repositories/ctf/synthesis_test.go +++ b/pkg/contexts/oci/repositories/ctf/synthesis_test.go @@ -42,6 +42,10 @@ func (d *DummyMethod) GetKind() string { return localblob.Type } +func (d *DummyMethod) IsLocal() bool { + return true +} + func (d *DummyMethod) AccessSpec() cpi.AccessSpec { return nil } diff --git a/pkg/contexts/ocm/accessmethods/compose/method.go b/pkg/contexts/ocm/accessmethods/compose/method.go new file mode 100644 index 0000000000..b11b552596 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/compose/method.go @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package compose + +import ( + "fmt" + "io" + "sync/atomic" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + cpi "github.com/open-component-model/ocm/pkg/contexts/ocm/internal" // avoid cycle + "github.com/open-component-model/ocm/pkg/runtime" +) + +// Type is the access type of GitHub registry. +const ( + Type = "compose" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) + +func Is(spec cpi.AccessSpec) bool { + return spec != nil && spec.GetKind() == Type +} + +// AccessSpec describes the access for a GitHub registry. +type AccessSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // Id is the internal id to identify the content + Id string `json:"id"` + + // MediaType is the media type of the object represented by the blob + MediaType string `json:"mediaType"` + + // GlobalAccess is an optional field describing a possibility + // for a global access. If given, it MUST describe a global access method. + GlobalAccess *cpi.AccessSpecRef `json:"globalAccess,omitempty"` + // ReferenceName is an optional static name the object should be + // use in a local repository context. It is use by a repository + // to optionally determine a globally referencable access according + // to the OCI distribution spec. The result will be stored + // by the repository in the field ImageReference. + // The value is typically an OCI repository name optionally + // followed by a colon ':' and a tag + ReferenceName string `json:"referenceName,omitempty"` +} + +var ( + _ cpi.AccessSpec = (*AccessSpec)(nil) + _ cpi.HintProvider = (*AccessSpec)(nil) + _ cpi.GlobalAccessProvider = (*AccessSpec)(nil) +) + +// New creates a new GitHub registry access spec version v1. +func New(hint string, mediaType string, global cpi.AccessSpec) *AccessSpec { + id := fmt.Sprintf("compose-%d", number.Add(1)) + s := &AccessSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + Id: id, + ReferenceName: hint, + MediaType: mediaType, + GlobalAccess: cpi.NewAccessSpecRef(global), + } + return s +} + +var number atomic.Int64 + +func (a *AccessSpec) Describe(ctx cpi.Context) string { + return fmt.Sprintf("Composition blob %s", a.Id) +} + +func (_ *AccessSpec) IsLocal(cpi.Context) bool { + return true +} + +func (a *AccessSpec) GetReferenceHint(cv cpi.ComponentVersionAccess) string { + return a.ReferenceName +} + +func (a *AccessSpec) GlobalAccessSpec(ctx cpi.Context) cpi.AccessSpec { + if g, err := ctx.AccessSpecForSpec(a.GlobalAccess); err == nil { + return g + } + return a.GlobalAccess.Unwrap() +} + +func (_ *AccessSpec) GetType() string { + return Type +} + +func (a *AccessSpec) AccessMethod(cv cpi.ComponentVersionAccess) (cpi.AccessMethod, error) { + return cv.AccessMethod(a) +} + +func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access cpi.ComponentVersionAccess) string { + return a.Id +} + +type accessMethod struct { + access blobaccess.BlobAccess + + spec *AccessSpec +} + +var _ cpi.AccessMethod = (*accessMethod)(nil) + +func NewMethod(spec *AccessSpec, blob blobaccess.BlobAccess) (cpi.AccessMethod, error) { + if blob.MimeType() != spec.MediaType { + return nil, fmt.Errorf("mimetype mismatch (spec=%s, blob=%s)", spec.MediaType, blob.MimeType()) + } + b, err := blob.Dup() + if err != nil { + return nil, err + } + return &accessMethod{ + access: b, + spec: spec, + }, nil +} + +func (_ *accessMethod) IsLocal() bool { + return true +} + +func (m *accessMethod) GetKind() string { + return Type +} + +func (m *accessMethod) MimeType() string { + return m.access.MimeType() +} + +func (m *accessMethod) AccessSpec() cpi.AccessSpec { + return m.spec +} + +func (m *accessMethod) Get() ([]byte, error) { + return m.access.Get() +} + +func (m *accessMethod) Reader() (io.ReadCloser, error) { + return m.access.Reader() +} + +func (m *accessMethod) Close() error { + if m.access == nil { + return nil + } + return m.access.Close() +} diff --git a/pkg/contexts/ocm/accessmethods/github/method.go b/pkg/contexts/ocm/accessmethods/github/method.go index 0bda080218..de6365ec6b 100644 --- a/pkg/contexts/ocm/accessmethods/github/method.go +++ b/pkg/contexts/ocm/accessmethods/github/method.go @@ -234,6 +234,10 @@ func getCreds(serverurl, path string, cctx credentials.Context) (string, error) return creds.GetProperty(credentials.ATTR_TOKEN), nil } +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return Type } diff --git a/pkg/contexts/ocm/accessmethods/helm/method.go b/pkg/contexts/ocm/accessmethods/helm/method.go index b2caad6f59..bf794561fa 100644 --- a/pkg/contexts/ocm/accessmethods/helm/method.go +++ b/pkg/contexts/ocm/accessmethods/helm/method.go @@ -115,6 +115,10 @@ type accessMethod struct { var _ cpi.AccessMethod = (*accessMethod)(nil) +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return Type } diff --git a/pkg/contexts/ocm/accessmethods/localblob/method.go b/pkg/contexts/ocm/accessmethods/localblob/method.go index e41006c42a..e6b2f8b777 100644 --- a/pkg/contexts/ocm/accessmethods/localblob/method.go +++ b/pkg/contexts/ocm/accessmethods/localblob/method.go @@ -104,9 +104,10 @@ type AccessSpec struct { } var ( - _ json.Marshaler = (*AccessSpec)(nil) - _ cpi.HintProvider = (*AccessSpec)(nil) - _ cpi.AccessSpec = (*AccessSpec)(nil) + _ json.Marshaler = (*AccessSpec)(nil) + _ cpi.HintProvider = (*AccessSpec)(nil) + _ cpi.GlobalAccessProvider = (*AccessSpec)(nil) + _ cpi.AccessSpec = (*AccessSpec)(nil) ) func (a AccessSpec) MarshalJSON() ([]byte, error) { @@ -126,7 +127,7 @@ func (a *AccessSpec) GlobalAccessSpec(ctx cpi.Context) cpi.AccessSpec { if g, err := ctx.AccessSpecForSpec(a.GlobalAccess); err == nil { return g } - return a.GlobalAccess + return a.GlobalAccess.Unwrap() } func (a *AccessSpec) GetMimeType() string { diff --git a/pkg/contexts/ocm/accessmethods/none/method.go b/pkg/contexts/ocm/accessmethods/none/method.go index 4f0de2965b..b56aae3b16 100644 --- a/pkg/contexts/ocm/accessmethods/none/method.go +++ b/pkg/contexts/ocm/accessmethods/none/method.go @@ -74,6 +74,10 @@ type accessMethod struct { var _ cpi.AccessMethod = (*accessMethod)(nil) +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return Type } diff --git a/pkg/contexts/ocm/accessmethods/ociartifact/method.go b/pkg/contexts/ocm/accessmethods/ociartifact/method.go index 2540e478ef..5656281270 100644 --- a/pkg/contexts/ocm/accessmethods/ociartifact/method.go +++ b/pkg/contexts/ocm/accessmethods/ociartifact/method.go @@ -152,6 +152,10 @@ func NewMethod(ctx cpi.ContextProvider, a cpi.AccessSpec, ref string, repo ...oc }, nil } +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return m.spec.GetKind() } diff --git a/pkg/contexts/ocm/accessmethods/ociartifact/utils.go b/pkg/contexts/ocm/accessmethods/ociartifact/utils.go index 6fd62443c1..b85b27a833 100644 --- a/pkg/contexts/ocm/accessmethods/ociartifact/utils.go +++ b/pkg/contexts/ocm/accessmethods/ociartifact/utils.go @@ -16,16 +16,29 @@ func Hint(nv common.NameVersion, locator, repo, version string) string { if i := strings.LastIndex(version, "@"); i >= 0 { version = version[:i] // remove digest } - repository := fmt.Sprintf("%s/%s", nv.GetName(), locator) + repository := repoName(nv, locator) if repo != "" { if strings.HasPrefix(repo, grammar.RepositorySeparator) { repository = repo[1:] } else { - repository = fmt.Sprintf("%s/%s", nv.GetName(), repo) + repository = repoName(nv, repo) } } - if !strings.Contains(repository, ":") { - repository = fmt.Sprintf("%s:%s", repository, version) + if repository != "" && version != "" { + if !strings.Contains(repository, ":") { + repository = fmt.Sprintf("%s:%s", repository, version) + } } return repository } + +func repoName(nv common.NameVersion, locator string) string { + if nv.GetName() == "" { + return locator + } else { + if locator == "" { + return nv.GetName() + } + return fmt.Sprintf("%s/%s", nv.GetName(), locator) + } +} diff --git a/pkg/contexts/ocm/accessmethods/ociblob/method.go b/pkg/contexts/ocm/accessmethods/ociblob/method.go index fd766774df..78b28d2113 100644 --- a/pkg/contexts/ocm/accessmethods/ociblob/method.go +++ b/pkg/contexts/ocm/accessmethods/ociblob/method.go @@ -97,6 +97,10 @@ type accessMethod struct { var _ cpi.AccessMethod = (*accessMethod)(nil) +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return Type } diff --git a/pkg/contexts/ocm/accessmethods/plugin/method.go b/pkg/contexts/ocm/accessmethods/plugin/method.go index c89e3b6166..7569c47c8e 100644 --- a/pkg/contexts/ocm/accessmethods/plugin/method.go +++ b/pkg/contexts/ocm/accessmethods/plugin/method.go @@ -86,6 +86,10 @@ func newMethod(p *PluginHandler, spec *AccessSpec, ctx ocm.Context, info *ppi.Ac } } +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return m.spec.GetKind() } diff --git a/pkg/contexts/ocm/accessmethods/s3/method.go b/pkg/contexts/ocm/accessmethods/s3/method.go index 089f732287..d79e9dbd42 100644 --- a/pkg/contexts/ocm/accessmethods/s3/method.go +++ b/pkg/contexts/ocm/accessmethods/s3/method.go @@ -161,6 +161,10 @@ func getCreds(a *AccessSpec, cctx credentials.Context) (credentials.Credentials, return identity.GetCredentials(cctx, "", a.Bucket, a.Key, a.Version) } +func (_ *accessMethod) IsLocal() bool { + return false +} + func (m *accessMethod) GetKind() string { return Type } diff --git a/pkg/contexts/ocm/attrs/compositionmodeattr/attr.go b/pkg/contexts/ocm/attrs/compositionmodeattr/attr.go new file mode 100644 index 0000000000..ec13c9a775 --- /dev/null +++ b/pkg/contexts/ocm/attrs/compositionmodeattr/attr.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package compositionmodeattr + +import ( + "fmt" + + "github.com/open-component-model/ocm/pkg/contexts/datacontext" + "github.com/open-component-model/ocm/pkg/runtime" +) + +// UseCompositionMode enables the support of the new Composition mode for +// Component versions. It disabled the direct write-through and update-on-close +// to the underlying repository. Instead, an explicit call to AddVersion call +// s required to persist a composed change on a new as well as queried +// component version object. +const UseCompositionMode = false + +const ( + ATTR_KEY = "ocm.software/compositionmode" + ATTR_SHORT = "compositionmode" +) + +func init() { + datacontext.RegisterAttributeType(ATTR_KEY, AttributeType{}, ATTR_SHORT) +} + +type AttributeType struct{} + +func (a AttributeType) Name() string { + return ATTR_KEY +} + +func (a AttributeType) Description() string { + return ` +*bool* (default: ` + fmt.Sprintf("%t", UseCompositionMode) + ` +Composition mode decouples a component version provided by a repository +implemention from the backened persistence. Added local blobs will +and other changes witll not be forwarded to the backend repository until +an AddVersion is called on the component. +If composition mode is disabled blobs will directly be forwarded to +the backend and descriptor updated will be persisted on AddVersion +or closing a provided existing component version. +` +} + +func (a AttributeType) Encode(v interface{}, marshaller runtime.Marshaler) ([]byte, error) { + if _, ok := v.(bool); !ok { + return nil, fmt.Errorf("boolean required") + } + return marshaller.Marshal(v) +} + +func (a AttributeType) Decode(data []byte, unmarshaller runtime.Unmarshaler) (interface{}, error) { + var value bool + err := unmarshaller.Unmarshal(data, &value) + return value, err +} + +//////////////////////////////////////////////////////////////////////////////// + +func Get(ctx datacontext.Context) bool { + a := ctx.GetAttributes().GetAttribute(ATTR_KEY) + if a == nil { + return UseCompositionMode + } + return a.(bool) +} + +func Set(ctx datacontext.Context, flag bool) error { + return ctx.GetAttributes().SetAttribute(ATTR_KEY, flag) +} diff --git a/pkg/contexts/ocm/attrs/compositionmodeattr/attr_test.go b/pkg/contexts/ocm/attrs/compositionmodeattr/attr_test.go new file mode 100644 index 0000000000..b2576a33a2 --- /dev/null +++ b/pkg/contexts/ocm/attrs/compositionmodeattr/attr_test.go @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package compositionmodeattr_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/config" + "github.com/open-component-model/ocm/pkg/contexts/credentials" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + me "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" + "github.com/open-component-model/ocm/pkg/runtime" +) + +var _ = Describe("attribute", func() { + var ctx ocm.Context + var cfgctx config.Context + + BeforeEach(func() { + cfgctx = config.WithSharedAttributes(datacontext.New(nil)).New() + credctx := credentials.WithConfigs(cfgctx).New() + ocictx := oci.WithCredentials(credctx).New() + ctx = ocm.WithOCIRepositories(ocictx).New() + }) + It("local setting", func() { + Expect(me.Get(ctx)).To(Equal(me.UseCompositionMode)) + Expect(me.Set(ctx, true)).To(Succeed()) + Expect(me.Get(ctx)).To(BeTrue()) + Expect(me.Set(ctx, false)).To(Succeed()) + Expect(me.Get(ctx)).To(BeFalse()) + }) + + It("global setting", func() { + Expect(me.Get(cfgctx)).To(Equal(me.UseCompositionMode)) + Expect(me.Set(cfgctx, true)).To(Succeed()) + Expect(me.Get(cfgctx)).To(BeTrue()) + Expect(me.Set(cfgctx, false)).To(Succeed()) + Expect(me.Get(cfgctx)).To(BeFalse()) + }) + + It("parses string", func() { + Expect(me.AttributeType{}.Decode([]byte("true"), runtime.DefaultJSONEncoding)).To(BeTrue()) + }) +}) diff --git a/pkg/contexts/ocm/attrs/compositionmodeattr/suite_test.go b/pkg/contexts/ocm/attrs/compositionmodeattr/suite_test.go new file mode 100644 index 0000000000..cafab85bd1 --- /dev/null +++ b/pkg/contexts/ocm/attrs/compositionmodeattr/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package compositionmodeattr_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM Composition Mode Attribute") +} diff --git a/pkg/contexts/ocm/compdesc/access/artifacts.go b/pkg/contexts/ocm/compdesc/access/artifacts.go deleted file mode 100644 index 9c05d36e71..0000000000 --- a/pkg/contexts/ocm/compdesc/access/artifacts.go +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - -package access - -import ( - "github.com/open-component-model/ocm/pkg/contexts/ocm" - "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" -) - -type baseAccess struct { - cv ocm.ComponentVersionAccess - access compdesc.AccessSpec -} - -func (r *baseAccess) ComponentVersion() ocm.ComponentVersionAccess { - return r.cv -} - -func (r *baseAccess) Access() (ocm.AccessSpec, error) { - return r.cv.GetContext().AccessSpecForSpec(r.access) -} - -func (r *baseAccess) AccessMethod() (ocm.AccessMethod, error) { - acc, err := r.Access() - if err != nil { - return nil, err - } - return r.cv.AccessMethod(acc) -} - -type resourceAccess struct { - baseAccess - resource *compdesc.Resource -} - -var _ ocm.ResourceAccess = (*resourceAccess)(nil) - -func NewResourceAccess(cv ocm.ComponentVersionAccess, rsc *compdesc.Resource) ocm.ResourceAccess { - return &resourceAccess{ - baseAccess: baseAccess{ - cv: cv, - access: rsc.Access, - }, - resource: rsc, - } -} - -func (r *resourceAccess) Meta() *compdesc.ResourceMeta { - return &r.resource.ResourceMeta -} - -type sourceAccess struct { - baseAccess - source *compdesc.Source -} - -var _ ocm.SourceAccess = (*sourceAccess)(nil) - -func NewSourceAccess(cv ocm.ComponentVersionAccess, sc *compdesc.Source) ocm.SourceAccess { - return &sourceAccess{ - baseAccess: baseAccess{ - cv: cv, - access: sc.Access, - }, - source: sc, - } -} - -func (r *sourceAccess) Meta() *compdesc.SourceMeta { - return &r.source.SourceMeta -} diff --git a/pkg/contexts/ocm/compdesc/componentdescriptor.go b/pkg/contexts/ocm/compdesc/componentdescriptor.go index 4fdeeab2b3..ea4299fc9a 100644 --- a/pkg/contexts/ocm/compdesc/componentdescriptor.go +++ b/pkg/contexts/ocm/compdesc/componentdescriptor.go @@ -128,6 +128,13 @@ type ElementMetaAccess interface { type ArtifactMetaAccess interface { ElementMetaAccess GetType() string + SetType(string) +} + +// ArtifactMetaPointer is a pointer to an artifact meta object. +type ArtifactMetaPointer[P any] interface { + ArtifactMetaAccess + *P } // ElementMeta defines a object that is uniquely identified by its identity. @@ -179,8 +186,8 @@ func (o *ElementMeta) SetLabels(labels []metav1.Label) { // SetLabel sets a single label to an effective value. // If the value is no byte slice, it is marshaled. -func (o *ElementMeta) SetLabel(name string, value interface{}) error { - return o.Labels.Set(name, value) +func (o *ElementMeta) SetLabel(name string, value interface{}, opts ...metav1.LabelOption) error { + return o.Labels.Set(name, value, opts...) } // RemoveLabel removes a single label. @@ -343,6 +350,7 @@ type ElementAccessor interface { // ElementArtifactAccessor provides access to generic artifact information of an element. type ElementArtifactAccessor interface { ElementMetaAccessor + GetType() string GetAccess() AccessSpec SetAccess(a AccessSpec) } @@ -488,6 +496,13 @@ func (o *SourceMeta) Copy() *SourceMeta { } } +func NewSourceMeta(name, typ string) *SourceMeta { + return &SourceMeta{ + ElementMeta: ElementMeta{Name: name}, + Type: typ, + } +} + // SourceRef defines a reference to a source // +k8s:deepcopy-gen=true // +k8s:openapi-gen=true @@ -663,22 +678,13 @@ func (o *ResourceMeta) GetType() string { } // SetType sets the type of the object. -func (o *ResourceMeta) SetType(ttype string) *ResourceMeta { +func (o *ResourceMeta) SetType(ttype string) { o.Type = ttype - return o } // SetDigest sets the digest of the object. -func (o *ResourceMeta) SetDigest(d *metav1.DigestSpec) *ResourceMeta { +func (o *ResourceMeta) SetDigest(d *metav1.DigestSpec) { o.Digest = d - return o -} - -// SetLabel sets a label of the object. -func (o *ResourceMeta) SetLabel(name string, value interface{}, opts ...metav1.LabelOption) *ResourceMeta { - // assure chainability - _ = o.Labels.Set(name, value, opts...) - return o } // Copy copies a resource meta. diff --git a/pkg/contexts/ocm/cpi/compose_test.go b/pkg/contexts/ocm/cpi/compose_test.go new file mode 100644 index 0000000000..18055a6c92 --- /dev/null +++ b/pkg/contexts/ocm/cpi/compose_test.go @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package cpi_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/contexts/ocm/testhelper" + . "github.com/open-component-model/ocm/pkg/finalizer" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/mime" +) + +const COMPONENT = "github.com/mandelsoft/ocm" +const VERSION = "1.0.0" + +var _ = Describe("access method", func() { + var fs vfs.FileSystem + var ctx ocm.Context + + BeforeEach(func() { + ctx = ocm.New(datacontext.MODE_EXTENDED) + fs = memoryfs.New() + }) + + DescribeTable("composes cv in one repo", func(mode bool) { + final := Finalizer{} + defer Defer(final.Finalize) + + compositionmodeattr.Set(ctx, mode) + a := Must(ctf.Create(ctx, accessobj.ACC_WRITABLE|accessobj.ACC_CREATE, "ctf", 0o700, accessio.PathFileSystem(fs))) + final.Close(a) + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv) + + // add resource + MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) + Expect(Must(cv.GetResource(compdesc.NewIdentity("text1"))).Meta().Digest).To(Equal(DS_TESTDATA)) + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(final.Finalize()) + + a = Must(ctf.Open(ctx, accessobj.ACC_READONLY, "ctf", 0o700, accessio.PathFileSystem(fs))) + final.Close(a) + + cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) + final.Close(cv) + + Expect(Must(cv.GetResourcesByName("text1"))[0].Meta().Digest).To(Equal(DS_TESTDATA)) + }, + Entry("direct", false), + Entry("compose", true), + ) + + DescribeTable("composes cv in one repo and add it to another", func(mode bool) { + final := Finalizer{} + defer Defer(final.Finalize) + + compositionmodeattr.Set(ctx, mode) + a := Must(ctf.Create(ctx, accessobj.ACC_WRITABLE|accessobj.ACC_CREATE, "ctf", 0o700, accessio.PathFileSystem(fs))) + final.Close(a) + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv) + + // add resource + MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) + Expect(Must(cv.GetResource(compdesc.NewIdentity("text1"))).Meta().Digest).To(Equal(DS_TESTDATA)) + + a2 := Must(ctf.Create(ctx, accessobj.ACC_WRITABLE|accessobj.ACC_CREATE, "ctf2", 0o700, accessio.PathFileSystem(fs))) + final.Close(a2) + c2 := Must(a2.LookupComponent(COMPONENT)) + final.Close(c2) + + MustBeSuccessful(c2.AddVersion(cv)) + MustBeSuccessful(final.Finalize()) + + a = Must(ctf.Open(ctx, accessobj.ACC_READONLY, "ctf", 0o700, accessio.PathFileSystem(fs))) + final.Close(a) + ExpectError(a.LookupComponentVersion(COMPONENT, VERSION)).To(MatchError(`component version "github.com/mandelsoft/ocm:1.0.0" not found: oci artifact "1.0.0" not found in component-descriptors/github.com/mandelsoft/ocm`)) + + a2 = Must(ctf.Open(ctx, accessobj.ACC_READONLY, "ctf2", 0o700, accessio.PathFileSystem(fs))) + final.Close(a2) + + cv = Must(a2.LookupComponentVersion(COMPONENT, VERSION)) + final.Close(cv) + + Expect(Must(cv.GetResourcesByName("text1"))[0].Meta().Digest).To(Equal(DS_TESTDATA)) + }, + Entry("direct", false), + Entry("compose", true), + ) +}) diff --git a/pkg/contexts/ocm/cpi/dummy.go b/pkg/contexts/ocm/cpi/dummy.go index 134024532d..9ae0a2b0a1 100644 --- a/pkg/contexts/ocm/cpi/dummy.go +++ b/pkg/contexts/ocm/cpi/dummy.go @@ -111,6 +111,10 @@ func (d *DummyComponentVersionAccess) GetInexpensiveContentVersionIdentity(spec panic("implement me") } +func (d *DummyComponentVersionAccess) Update() error { + panic("implement me") +} + func (d *DummyComponentVersionAccess) AddBlob(blob BlobAccess, arttype, refName string, global AccessSpec) (AccessSpec, error) { panic("implement me") } @@ -127,6 +131,10 @@ func (d *DummyComponentVersionAccess) SetResource(meta *ResourceMeta, spec compd panic("implement me") } +func (d *DummyComponentVersionAccess) SetResourceByAccess(art ResourceAccess, modopts ...ModificationOption) error { + panic("implement me") +} + func (d *DummyComponentVersionAccess) SetSourceBlob(meta *SourceMeta, blob BlobAccess, refname string, global AccessSpec) error { panic("implement me") } @@ -135,6 +143,10 @@ func (d *DummyComponentVersionAccess) SetSource(meta *SourceMeta, spec compdesc. panic("implement me") } +func (d *DummyComponentVersionAccess) SetSourceByAccess(art SourceAccess) error { + panic("implement me") +} + func (d *DummyComponentVersionAccess) SetReference(ref *ComponentReference) error { panic("implement me") } @@ -146,6 +158,10 @@ func (d *DummyComponentVersionAccess) IsPersistent() bool { return false } +func (d *DummyComponentVersionAccess) UseDirectAccess() bool { + return true +} + func (d *DummyComponentVersionAccess) GetResourcesByIdentitySelectors(selectors ...compdesc.IdentitySelector) ([]internal.ResourceAccess, error) { return nil, nil } diff --git a/pkg/contexts/ocm/cpi/interface.go b/pkg/contexts/ocm/cpi/interface.go index 627f62742a..2de44c2ab0 100644 --- a/pkg/contexts/ocm/cpi/interface.go +++ b/pkg/contexts/ocm/cpi/interface.go @@ -73,6 +73,8 @@ type ( ComponentReference = internal.ComponentReference ) +type ArtifactAccess[M any] internal.ArtifactAccess[M] + type ( BlobHandler = internal.BlobHandler BlobHandlerOption = internal.BlobHandlerOption @@ -220,6 +222,9 @@ func RepositoryPrefix(spec RepositorySpec) string { // artifacts. type HintProvider internal.HintProvider +// GlobalAccessProvider is able to provide a non-local access specification. +type GlobalAccessProvider internal.GlobalAccessProvider + func ArtifactNameHint(spec AccessSpec, cv ComponentVersionAccess) string { if h, ok := spec.(HintProvider); ok { return h.GetReferenceHint(cv) diff --git a/pkg/contexts/ocm/cpi/method.go b/pkg/contexts/ocm/cpi/method.go index 6c6874dc66..075b67a648 100644 --- a/pkg/contexts/ocm/cpi/method.go +++ b/pkg/contexts/ocm/cpi/method.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/utils" ) //////////////////////////////////////////////////////////////////////////////// @@ -21,18 +22,20 @@ type DefaultAccessMethod struct { comp ComponentVersionAccess spec AccessSpec mime string + local bool } var _ AccessMethod = (*DefaultAccessMethod)(nil) type BlobAccessFactory func() (BlobAccess, error) -func NewDefaultMethod(c ComponentVersionAccess, a AccessSpec, mime string, fac BlobAccessFactory) AccessMethod { +func NewDefaultMethod(c ComponentVersionAccess, a AccessSpec, mime string, fac BlobAccessFactory, local ...bool) AccessMethod { return &DefaultAccessMethod{ spec: a, comp: c, mime: mime, factory: fac, + local: utils.Optional(local...), } } @@ -49,6 +52,10 @@ func (m *DefaultAccessMethod) getAccess() (blobaccess.BlobAccess, error) { return m.access, nil } +func (m *DefaultAccessMethod) IsLocal() bool { + return m.local +} + func (m *DefaultAccessMethod) GetKind() string { return m.spec.GetKind() } diff --git a/pkg/contexts/ocm/cpi/methodview.go b/pkg/contexts/ocm/cpi/methodview.go index 4707d12235..b62a21d02f 100644 --- a/pkg/contexts/ocm/cpi/methodview.go +++ b/pkg/contexts/ocm/cpi/methodview.go @@ -15,12 +15,7 @@ import ( // AccessMethodView can be used map wrap an access method // into a managed method with multiple views. The original method // object is closed once the last view is closed. -type AccessMethodView interface { - AccessMethod - - Base() interface{} - Dup() (AccessMethodView, error) -} +type AccessMethodView = internal.AccessMethodView // AccessMethodAsView wrap an access method object into // a multi-view version. The original method is closed when @@ -32,6 +27,16 @@ func AccessMethodAsView(acc AccessMethod, closer ...io.Closer) AccessMethodView return refmgmt.WithView[AccessMethod, AccessMethodView](acc, accessMethodViewCreator, closer...) } +// BlobAccessForAccessSpec provide a blob access for an access specification. +func BlobAccessForAccessSpec(spec AccessSpec, cv ComponentVersionAccess) (blobaccess.BlobAccess, error) { + m, err := AccessMethodViewForSpec(spec, cv) + if err != nil { + return nil, err + } + defer m.Close() + return BlobAccessForAccessMethod(m) +} + func AccessMethodViewForSpec(spec AccessSpec, cv ComponentVersionAccess) (AccessMethodView, error) { m, err := spec.AccessMethod(cv) if err != nil { @@ -61,6 +66,10 @@ func (a *accessMethodView) Base() interface{} { return a.access } +func (a *accessMethodView) IsLocal() bool { + return a.access.IsLocal() +} + func (a *accessMethodView) Get() ([]byte, error) { var result []byte err := a.Execute(func() (err error) { diff --git a/pkg/contexts/ocm/cpi/suite_test.go b/pkg/contexts/ocm/cpi/suite_test.go new file mode 100644 index 0000000000..7fc7504c2f --- /dev/null +++ b/pkg/contexts/ocm/cpi/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package cpi_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM CPI Test Suite") +} diff --git a/pkg/contexts/ocm/cpi/support/compversaccess.go b/pkg/contexts/ocm/cpi/support/compversaccess.go index a61702afc6..4f88d512d3 100644 --- a/pkg/contexts/ocm/cpi/support/compversaccess.go +++ b/pkg/contexts/ocm/cpi/support/compversaccess.go @@ -15,12 +15,12 @@ type _ComponentVersionAccessImplBase = cpi.ComponentVersionAccessImplBase type ComponentVersionAccessImpl interface { cpi.ComponentVersionAccessImpl EnablePersistence() bool - Update(final bool) error } type componentVersionAccessImpl struct { *_ComponentVersionAccessImplBase lazy bool + directAccess bool persistent bool discardChanges bool base ComponentVersionContainer @@ -28,18 +28,24 @@ type componentVersionAccessImpl struct { var _ ComponentVersionAccessImpl = (*componentVersionAccessImpl)(nil) -func GetComponentVersionContainer(cv cpi.ComponentVersionAccess) (ComponentVersionContainer, error) { +func GetComponentVersionContainer[T ComponentVersionContainer](cv cpi.ComponentVersionAccess) (T, error) { + var _nil T + impl, err := cpi.GetComponentVersionAccessImplementation(cv) if err != nil { - return nil, err + return _nil, err } if mine, ok := impl.(*componentVersionAccessImpl); ok { - return mine.base, nil + cont, ok := mine.base.(T) + if ok { + return cont, nil + } + return _nil, errors.Newf("non-matching component version implementation %T", mine.base) } - return nil, errors.Newf("non-matching component version implementation %T", impl) + return _nil, errors.Newf("non-matching component version implementation %T", impl) } -func NewComponentVersionAccessImpl(name, version string, container ComponentVersionContainer, lazy bool, persistent bool) (cpi.ComponentVersionAccessImpl, error) { +func NewComponentVersionAccessImpl(name, version string, container ComponentVersionContainer, lazy, persistent, direct bool) (cpi.ComponentVersionAccessImpl, error) { base, err := cpi.NewComponentVersionAccessImplBase(container.GetContext(), name, version, container.GetParentViewManager()) if err != nil { return nil, err @@ -48,6 +54,7 @@ func NewComponentVersionAccessImpl(name, version string, container ComponentVers _ComponentVersionAccessImplBase: base, lazy: lazy, persistent: persistent, + directAccess: direct, base: container, } container.SetImplementation(impl) @@ -66,12 +73,17 @@ func (a *componentVersionAccessImpl) IsPersistent() bool { return a.persistent } +func (d *componentVersionAccessImpl) UseDirectAccess() bool { + return d.directAccess +} + func (a *componentVersionAccessImpl) DiscardChanges() { a.discardChanges = true } func (a *componentVersionAccessImpl) Close() error { - return errors.ErrListf("closing component version access %s/%s", a.GetName(), a.GetVersion()).Add(a.Update(true), a.base.Close(), a._ComponentVersionAccessImplBase.Close()).Result() + list := errors.ErrListf("closing component version access %s/%s", a.GetName(), a.GetVersion()) + return list.Add(a.base.Close(), a._ComponentVersionAccessImplBase.Close()).Result() } func (a *componentVersionAccessImpl) Repository() cpi.Repository { @@ -105,8 +117,18 @@ func (a *componentVersionAccessImpl) AddBlobFor(storagectx cpi.StorageContext, b return a.base.AddBlobFor(storagectx, blob, refName, global) } +func (a *componentVersionAccessImpl) ShouldUpdate(final bool) bool { + if a.discardChanges { + return false + } + if final { + return a.persistent + } + return !a.lazy && a.directAccess && a.persistent +} + func (a *componentVersionAccessImpl) Update(final bool) error { - if (final || !a.lazy) && !a.discardChanges && a.persistent { + if a.ShouldUpdate(final) { return a.base.Update() } return nil diff --git a/pkg/contexts/ocm/cpi/utils.go b/pkg/contexts/ocm/cpi/utils.go index 283e1af9f1..1a7915c116 100644 --- a/pkg/contexts/ocm/cpi/utils.go +++ b/pkg/contexts/ocm/cpi/utils.go @@ -8,6 +8,7 @@ import ( "io" "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/contexts/ocm/internal" ) type AccessMethodSource interface { @@ -57,3 +58,18 @@ func ResourceData(s AccessMethodSource) ([]byte, error) { defer meth.Close() return meth.Get() } + +func ReferenceHint(spec AccessSpec, cv ComponentVersionAccess) string { + if h, ok := spec.(internal.HintProvider); ok { + return h.GetReferenceHint(cv) + } + return "" +} + +func GlobalAccess(spec AccessSpec, ctx Context) AccessSpec { + g := spec.GlobalAccessSpec(ctx) + if g != nil && g.IsLocal(ctx) { + g = nil + } + return g +} diff --git a/pkg/contexts/ocm/cpi/view.go b/pkg/contexts/ocm/cpi/view.go index 933f79fc7a..7da21f6ff6 100644 --- a/pkg/contexts/ocm/cpi/view.go +++ b/pkg/contexts/ocm/cpi/view.go @@ -13,14 +13,19 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/common/accessio" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/refmgmt" "github.com/open-component-model/ocm/pkg/common/accessio/resource" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/compose" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/keepblobattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" "github.com/open-component-model/ocm/pkg/contexts/ocm/internal" + "github.com/open-component-model/ocm/pkg/contexts/ocm/plugin/descriptor" "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/finalizer" "github.com/open-component-model/ocm/pkg/utils" "github.com/open-component-model/ocm/pkg/utils/selector" ) @@ -110,15 +115,15 @@ func (r *repositoryView) GetIdentityMatcher() string { return credentials.GetProvidedIdentityMatcher(r.impl) } -func (r *repositoryView) GetSpecification() internal.RepositorySpec { +func (r *repositoryView) GetSpecification() RepositorySpec { return r.impl.GetSpecification() } -func (r *repositoryView) GetContext() internal.Context { +func (r *repositoryView) GetContext() Context { return r.impl.GetContext() } -func (r *repositoryView) ComponentLister() internal.ComponentLister { +func (r *repositoryView) ComponentLister() ComponentLister { return r.impl.ComponentLister() } @@ -146,6 +151,26 @@ func (r *repositoryView) LookupComponent(name string) (acc ComponentAccess, err return acc, err } +func (r *repositoryView) NewVersion(comp, vers string, overrides ...bool) (ComponentVersionAccess, error) { + c, err := refmgmt.ToLazy(r.LookupComponent(comp)) + if err != nil { + return nil, err + } + defer c.Close() + + return c.NewVersion(vers, overrides...) +} + +func (r *repositoryView) AddVersion(cv ComponentVersionAccess, overrides ...bool) error { + c, err := refmgmt.ToLazy(r.LookupComponent(cv.GetName())) + if err != nil { + return err + } + defer c.Close() + + return c.AddVersion(cv, overrides...) +} + //////////////////////////////////////////////////////////////////////////////// type _ComponentAccessView interface { @@ -160,6 +185,10 @@ type ComponentAccessImpl interface { IsReadOnly() bool GetName() string + + IsOwned(access ComponentVersionAccess) bool + + AddVersion(cv ComponentVersionAccess) error } type _ComponentAccessImplBase = resource.ResourceImplBase[ComponentAccess] @@ -239,17 +268,117 @@ func (c *componentAccessView) LookupVersion(version string) (acc ComponentVersio return acc, err } -func (c *componentAccessView) AddVersion(acc ComponentVersionAccess) error { +func (c *componentAccessView) AddVersion(acc ComponentVersionAccess, overrides ...bool) error { if acc.GetName() != c.GetName() { return errors.ErrInvalid("component name", acc.GetName()) } return c.Execute(func() error { - return c.addVersion(acc) + return c.addVersion(acc, overrides...) }) } -func (c *componentAccessView) addVersion(acc ComponentVersionAccess) error { - return c.impl.AddVersion(acc) +func (c *componentAccessView) addVersion(acc ComponentVersionAccess, overrides ...bool) (ferr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagation(&ferr) + + ctx := acc.GetContext() + + impl, err := GetComponentVersionAccessImplementation(acc) + if err != nil { + return err + } + + var ( + d *compdesc.ComponentDescriptor + sel func(AccessSpec) bool + eff ComponentVersionAccess + ) + + if !c.impl.IsOwned(acc) { + // transfer all local blobs into a new owned version. + sel = func(spec AccessSpec) bool { return spec.IsLocal(ctx) } + + eff, err = c.impl.NewVersion(acc.GetVersion(), overrides...) + if err != nil { + return err + } + finalize.With(func() error { + return eff.Close() + }) + impl, err = GetComponentVersionAccessImplementation(eff) + if err != nil { + return err + } + + d = eff.GetDescriptor() + *d = *acc.GetDescriptor().Copy() + } else { + // transfer composition blobs into local blobs + sel = compose.Is + d = acc.GetDescriptor() + eff = acc + } + + err = setupLocalBobs(ctx, "resource", acc, eff, nil, impl, d.Resources, sel) + if err == nil { + err = setupLocalBobs(ctx, "source", acc, eff, nil, impl, d.Sources, sel) + } + if err != nil { + return err + } + + return c.impl.AddVersion(eff) +} + +func setupLocalBobs(ctx Context, kind string, src, tgt ComponentVersionAccess, accprov func(AccessSpec) (AccessMethod, error), tgtimpl ComponentVersionAccessImpl, it compdesc.ArtifactAccessor, sel func(AccessSpec) bool) (ferr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagation(&ferr) + + for i := 0; i < it.Len(); i++ { + nested := finalize.Nested() + a := it.GetArtifact(i) + spec, err := ctx.AccessSpecForSpec(a.GetAccess()) + if err != nil { + return errors.Wrapf(err, "%s %d", kind, i) + } + if sel(spec) { + blob, err := blobAccessForLocalAccessSpec(spec, src, accprov) + if err != nil { + return errors.Wrapf(err, "%s %d", kind, i) + } + nested.Close(blob) + effspec, err := addBlob(tgtimpl, tgt, a.GetType(), ReferenceHint(spec, src), blob, GlobalAccess(spec, ctx)) + if err != nil { + return errors.Wrapf(err, "cannot store %s %d", kind, i) + } + a.SetAccess(effspec) + } + err = nested.Finalize() + if err != nil { + return errors.Wrapf(err, "%s %d", kind, i) + } + } + return nil +} + +func blobAccessForLocalAccessSpec(spec AccessSpec, cv ComponentVersionAccess, accprov func(AccessSpec) (AccessMethod, error)) (blobaccess.BlobAccess, error) { + var m AccessMethod + var err error + if accprov != nil { + m, err = accprov(spec) + } else { + m, err = spec.AccessMethod(cv) + } + if err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + v := AccessMethodAsView(m) + defer v.Close() + return BlobAccessForAccessMethod(v) } func (c *componentAccessView) NewVersion(version string, overrides ...bool) (acc ComponentVersionAccess, err error) { @@ -281,7 +410,16 @@ type ComponentVersionAccessViewManager = resource.ViewManager[ComponentVersionAc type ComponentVersionAccessImpl interface { resource.ResourceImplementation[ComponentVersionAccess] - internal.ComponentVersionAccessImpl + common.VersionedElement + io.Closer + + GetContext() Context + Repository() Repository + + DiscardChanges() + IsPersistent() bool + + GetDescriptor() *compdesc.ComponentDescriptor AccessMethod(ComponentVersionAccess, AccessSpec) (AccessMethod, error) @@ -301,41 +439,56 @@ type ComponentVersionAccessImpl interface { AddBlobFor(storagectx StorageContext, blob BlobAccess, refName string, global AccessSpec) (AccessSpec, error) IsReadOnly() bool - Update(final bool) error + + // ShouldUpdate checks, whether an update is indicated + // by the state of object, considering persistence, lazy, discard + // and update mode state + ShouldUpdate(final bool) bool // GetBlobCache retieves the blob cache used to store preliminary // blob accesses for freshly generated local access specs not directly // usable until a component version is finally added to the repository. GetBlobCache() BlobCache + + // UseDirectAccess returns true if composition should be directly + // forwarded to the repository backend., + UseDirectAccess() bool + + // Update persists the current state of the component version to the + // underlying repository backend. + Update(final bool) error } -type BlobCacheEntry = blobaccess.BlobAccess +type ( + BlobCacheEntry = blobaccess.BlobAccess + BlobCacheKey = interface{} +) type BlobCache interface { // AddBlobFor stores blobs for added blobs not yet accessible // by generated access method until version is finally added. - AddBlobFor(acc compdesc.AccessSpec, blob BlobCacheEntry) error + AddBlobFor(acc BlobCacheKey, blob BlobCacheEntry) error // GetBlobFor retrieves the original blob access for // a given access specification. - GetBlobFor(acc compdesc.AccessSpec) BlobCacheEntry + GetBlobFor(acc BlobCacheKey) BlobCacheEntry - RemoveBlobFor(acc compdesc.AccessSpec) + RemoveBlobFor(acc BlobCacheKey) Clear() error } type blobCache struct { lock sync.Mutex - blobcache map[interface{}]BlobCacheEntry + blobcache map[BlobCacheKey]BlobCacheEntry } func NewBlobCache() BlobCache { return &blobCache{ - blobcache: map[interface{}]BlobCacheEntry{}, + blobcache: map[BlobCacheKey]BlobCacheEntry{}, } } -func (c *blobCache) RemoveBlobFor(acc compdesc.AccessSpec) { +func (c *blobCache) RemoveBlobFor(acc BlobCacheKey) { c.lock.Lock() defer c.lock.Unlock() if b := c.blobcache[acc]; b != nil { @@ -344,7 +497,10 @@ func (c *blobCache) RemoveBlobFor(acc compdesc.AccessSpec) { } } -func (c *blobCache) AddBlobFor(acc compdesc.AccessSpec, blob BlobCacheEntry) error { +func (c *blobCache) AddBlobFor(acc BlobCacheKey, blob BlobCacheEntry) error { + if s, ok := acc.(string); ok && s == "" { + return errors.ErrInvalid("blob key") + } c.lock.Lock() defer c.lock.Unlock() @@ -358,7 +514,7 @@ func (c *blobCache) AddBlobFor(acc compdesc.AccessSpec, blob BlobCacheEntry) err return nil } -func (c *blobCache) GetBlobFor(acc compdesc.AccessSpec) BlobCacheEntry { +func (c *blobCache) GetBlobFor(acc BlobCacheKey) BlobCacheEntry { c.lock.Lock() defer c.lock.Unlock() @@ -372,7 +528,7 @@ func (c *blobCache) Clear() error { for _, b := range c.blobcache { list.Add(b.Close()) } - c.blobcache = map[interface{}]BlobCacheEntry{} + c.blobcache = map[BlobCacheKey]BlobCacheEntry{} return list.Result() } @@ -450,6 +606,23 @@ func NewComponentVersionAccess(impl ComponentVersionAccessImpl) ComponentVersion } func (c *componentVersionAccessView) Close() error { + err := c.Execute(func() error { + // executed under local lock, if refcount is one, I'm the last user. + if c.impl.RefCount() == 1 { + // prepare artifact access for final close in + // direct access mode. + if !compositionmodeattr.Get(c.GetContext()) { + err := c.update(true) + if err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } return c._ComponentVersionAccessView.Close() } @@ -474,32 +647,107 @@ func (c *componentVersionAccessView) GetDescriptor() *compdesc.ComponentDescript } func (c *componentVersionAccessView) AccessMethod(spec AccessSpec) (meth AccessMethod, err error) { + spec, err = c.GetContext().AccessSpecForSpec(spec) + if err != nil { + return nil, err + } err = c.Execute(func() error { - if !spec.IsLocal(c.GetContext()) { - // fall back to original version - meth, err = spec.AccessMethod(c) - } else { - meth, err = c.impl.AccessMethod(c, spec) - } + var err error + meth, err = c.accessMethod(spec) return err }) return meth, err } +func (c *componentVersionAccessView) accessMethod(spec AccessSpec) (meth AccessMethod, err error) { + switch { + case compose.Is(spec): + cspec, ok := spec.(*compose.AccessSpec) + if !ok { + return nil, fmt.Errorf("invalid implementation (%T) for access method compose", spec) + } + blob := c.getLocalBlob(cspec) + if blob == nil { + return nil, errors.ErrUnknown(blobaccess.KIND_BLOB, cspec.Id, common.VersionedElementKey(c).String()) + } + meth, err = compose.NewMethod(cspec, blob) + case !spec.IsLocal(c.GetContext()): + meth, err = spec.AccessMethod(c) + default: + meth, err = c.impl.AccessMethod(c, spec) + if err == nil { + if blob := c.getLocalBlob(spec); blob != nil { + meth, err = newFakeMethod(meth, blob) + } + } + } + return meth, err +} + func (c *componentVersionAccessView) GetInexpensiveContentVersionIdentity(spec AccessSpec) string { + var err error + + spec, err = c.GetContext().AccessSpecForSpec(spec) + if err != nil { + return "" + } + var id string _ = c.Execute(func() error { - if !spec.IsLocal(c.GetContext()) { - // fall back to original version - id = spec.GetInexpensiveContentVersionIdentity(c) - } else { - id = c.impl.GetInexpensiveContentVersionIdentity(c, spec) - } + id = c.getInexpensiveContentVersionIdentity(spec) return nil }) return id } +func (c *componentVersionAccessView) getInexpensiveContentVersionIdentity(spec AccessSpec) string { + switch { + case compose.Is(spec): + fallthrough + case !spec.IsLocal(c.GetContext()): + // fall back to original version + return spec.GetInexpensiveContentVersionIdentity(c) + default: + return c.impl.GetInexpensiveContentVersionIdentity(c, spec) + } +} + +func (c *componentVersionAccessView) Update() error { + return c.Execute(func() error { + if !c.impl.IsPersistent() { + return fmt.Errorf("temporary component version cannot be updated") + } + return c.update(true) + }) +} + +func (c *componentVersionAccessView) update(final bool) error { + if !c.impl.ShouldUpdate(final) { + return nil + } + + ctx := c.GetContext() + d := c.GetDescriptor() + impl, err := GetComponentVersionAccessImplementation(c) + if err != nil { + return err + } + // TODO: exceute for separately lockable view + err = setupLocalBobs(ctx, "resource", c, c, c.accessMethod, impl, d.Resources, compose.Is) + if err == nil { + err = setupLocalBobs(ctx, "source", c, c, c.accessMethod, impl, d.Sources, compose.Is) + } + if err != nil { + return err + } + + err = c.impl.Update(true) + if err != nil { + return err + } + return c.impl.GetBlobCache().Clear() +} + func (c *componentVersionAccessView) AddBlob(blob cpi.BlobAccess, artType, refName string, global AccessSpec) (AccessSpec, error) { if blob == nil { return nil, errors.New("a resource has to be defined") @@ -517,34 +765,49 @@ func (c *componentVersionAccessView) AddBlob(blob cpi.BlobAccess, artType, refNa return nil, errors.Wrapf(err, "inavlid blob access") } - storagectx := c.impl.GetStorageContext(c) - h := c.GetContext().BlobHandlers().LookupHandler(storagectx.GetImplementationRepositoryType(), artType, blob.MimeType()) + var acc AccessSpec + if c.impl.UseDirectAccess() { + acc, err = addBlob(c.impl, c, artType, refName, blob, global) + } else { + // use local composition access to be added to the repository with AddVersion. + acc = compose.New(refName, blob.MimeType(), global) + } + if err == nil { + return c.cacheLocalBlob(acc, blob) + } + return acc, err +} + +func addBlob(impl ComponentVersionAccessImpl, cv ComponentVersionAccess, artType, refName string, blob BlobAccess, global AccessSpec) (AccessSpec, error) { + storagectx := impl.GetStorageContext(cv) + ctx := cv.GetContext() + h := ctx.BlobHandlers().LookupHandler(storagectx.GetImplementationRepositoryType(), artType, blob.MimeType()) if h != nil { acc, err := h.StoreBlob(blob, artType, refName, nil, storagectx) if err != nil { return nil, err } if acc != nil { - if !keepblobattr.Get(c.GetContext()) || acc.IsLocal(c.GetContext()) { - return c.cacheBlob(acc, blob) + if !keepblobattr.Get(ctx) || acc.IsLocal(ctx) { + return acc, nil } global = acc } } - acc, err := c.impl.AddBlobFor(storagectx, blob, refName, global) - if err != nil { - return nil, err - } - return c.cacheBlob(acc, blob) + return impl.AddBlobFor(storagectx, blob, refName, global) +} + +func (c *componentVersionAccessView) getLocalBlob(acc AccessSpec) BlobAccess { + return c.impl.GetBlobCache().GetBlobFor(c.getInexpensiveContentVersionIdentity(acc)) } -func (c *componentVersionAccessView) cacheBlob(acc AccessSpec, blob BlobAccess) (AccessSpec, error) { +func (c *componentVersionAccessView) cacheLocalBlob(acc AccessSpec, blob BlobAccess) (AccessSpec, error) { if acc.IsLocal(c.GetContext()) { // local blobs might not be accessible from the underlying // repository implementation if the component version is not // finally added (for example ghcr.io as OCI repository). - // Therefore, we keep a copy of the blo access for further usage. - err := c.impl.GetBlobCache().AddBlobFor(acc, blob) + // Therefore, we keep a copy of the blob access for further usage. + err := c.impl.GetBlobCache().AddBlobFor(c.getInexpensiveContentVersionIdentity(acc), blob) if err != nil { return nil, err } @@ -554,10 +817,10 @@ func (c *componentVersionAccessView) cacheBlob(acc AccessSpec, blob BlobAccess) func (c *componentVersionAccessView) AdjustResourceAccess(meta *ResourceMeta, acc compdesc.AccessSpec, opts ...internal.ModificationOption) error { cd := c.GetDescriptor() - if idx := cd.GetResourceIndex(meta); idx == -1 { - return errors.ErrUnknown(KIND_RESOURCE, meta.GetIdentity(cd.Resources).String()) + if idx := cd.GetResourceIndex(meta); idx >= 0 { + return c.SetResource(&cd.Resources[idx].ResourceMeta, acc, opts...) } - return c.SetResource(meta, acc, opts...) + return errors.ErrUnknown(KIND_RESOURCE, meta.GetIdentity(cd.Resources).String()) } // SetResourceBlob adds a blob resource to the component version. @@ -570,7 +833,6 @@ func (c *componentVersionAccessView) SetResourceBlob(meta *ResourceMeta, blob cp if err != nil { return fmt.Errorf("unable to add blob (component %s:%s resource %s): %w", c.GetName(), c.GetVersion(), meta.GetName(), err) } - defer c.impl.GetBlobCache().RemoveBlobFor(acc) if err := c.SetResource(meta, acc, append(opts, internal.ModifyResource())...); err != nil { return fmt.Errorf("unable to set resource: %w", err) @@ -581,10 +843,10 @@ func (c *componentVersionAccessView) SetResourceBlob(meta *ResourceMeta, blob cp func (c *componentVersionAccessView) AdjustSourceAccess(meta *SourceMeta, acc compdesc.AccessSpec) error { cd := c.GetDescriptor() - if idx := cd.GetSourceIndex(meta); idx == -1 { - return errors.ErrUnknown(KIND_RESOURCE, meta.GetIdentity(cd.Resources).String()) + if idx := cd.GetSourceIndex(meta); idx >= 0 { + return c.SetSource(&cd.Sources[idx].SourceMeta, acc) } - return c.SetSource(meta, acc) + return errors.ErrUnknown(KIND_RESOURCE, meta.GetIdentity(cd.Resources).String()) } func (c *componentVersionAccessView) SetSourceBlob(meta *SourceMeta, blob BlobAccess, refName string, global AccessSpec) error { @@ -596,7 +858,6 @@ func (c *componentVersionAccessView) SetSourceBlob(meta *SourceMeta, blob BlobAc if err != nil { return fmt.Errorf("unable to add blob: (component %s:%s source %s): %w", c.GetName(), c.GetVersion(), meta.GetName(), err) } - defer c.impl.GetBlobCache().RemoveBlobFor(acc) if err := c.SetSource(meta, acc); err != nil { return fmt.Errorf("unable to set source: %w", err) @@ -606,8 +867,51 @@ func (c *componentVersionAccessView) SetSourceBlob(meta *SourceMeta, blob BlobAc } type fakeMethod struct { - AccessMethod - blob blobaccess.BlobAccess + spec AccessSpec + local bool + mime string + blob blobaccess.BlobAccess +} + +var _ AccessMethod = (*fakeMethod)(nil) + +func newFakeMethod(m AccessMethod, blob BlobAccess) (AccessMethod, error) { + b, err := blob.Dup() + if err != nil { + return nil, errors.Wrapf(err, "cannot remember blob for access method") + } + f := &fakeMethod{ + spec: m.AccessSpec(), + local: m.IsLocal(), + mime: m.MimeType(), + blob: b, + } + err = m.Close() + if err != nil { + _ = b.Close() + return nil, errors.Wrapf(err, "closing access method") + } + return f, nil +} + +func (f *fakeMethod) MimeType() string { + return f.mime +} + +func (f *fakeMethod) IsLocal() bool { + return f.local +} + +func (f *fakeMethod) GetKind() string { + return f.spec.GetKind() +} + +func (f *fakeMethod) AccessSpec() internal.AccessSpec { + return f.spec +} + +func (f *fakeMethod) Close() error { + return f.blob.Close() } func (f *fakeMethod) Reader() (io.ReadCloser, error) { @@ -618,26 +922,70 @@ func (f *fakeMethod) Get() ([]byte, error) { return f.blob.Get() } -func (c *componentVersionAccessView) getMethod(acc compdesc.AccessSpec) (AccessMethod, error) { - spec, err := c.impl.GetContext().AccessSpecForSpec(acc) - if err != nil { - return nil, err +func setAccess[T any, A internal.ArtifactAccess[T]](c *componentVersionAccessView, kind string, art A, + set func(*T, compdesc.AccessSpec) error, + setblob func(*T, BlobAccess, string, AccessSpec) error, +) error { + if c.impl.IsReadOnly() { + return accessio.ErrReadOnly + } + meta := art.Meta() + if meta == nil { + return errors.Newf("no meta data provided by %s access", kind) + } + acc, err := art.Access() + if err != nil && !errors.IsErrNotFoundElem(err, "", descriptor.KIND_ACCESSMETHOD) { + return err } - meth, err := c.AccessMethod(spec) - if err != nil { - return nil, err + var ( + blob BlobAccess + hint string + global AccessSpec + ) + + if acc != nil { + if !acc.IsLocal(c.GetContext()) { + return set(meta, acc) + } + + blob, err = BlobAccessForAccessSpec(acc, c) + if err != nil && errors.IsErrNotFoundElem(err, "", blobaccess.KIND_BLOB) { + return err + } + hint = ReferenceHint(acc, c) + global = GlobalAccess(acc, c.GetContext()) } - if b := c.impl.GetBlobCache().GetBlobFor(acc); b != nil { - meth = &fakeMethod{ - AccessMethod: meth, - blob: b, + if blob == nil { + blob, err = art.BlobAccess() + if err != nil { + return err } + defer blob.Close() + } + if blob == nil { + return errors.Newf("neither access nor blob specified in %s access", kind) } - return meth, nil + if v := art.ReferenceHint(); v != "" { + hint = v + } + if v := art.GlobalAccess(); v != nil { + global = v + } + return setblob(meta, blob, hint, global) } -func (c *componentVersionAccessView) SetResource(meta *internal.ResourceMeta, acc compdesc.AccessSpec, modopts ...internal.ModificationOption) error { +func (c *componentVersionAccessView) SetResourceByAccess(art ResourceAccess, modopts ...ModificationOption) error { + return setAccess(c, "resource", art, + func(meta *ResourceMeta, acc compdesc.AccessSpec) error { + return c.SetResource(meta, acc, modopts...) + }, + func(meta *ResourceMeta, blob BlobAccess, hint string, global AccessSpec) error { + return c.SetResourceBlob(meta, blob, hint, global, modopts...) + }) +} + +func (c *componentVersionAccessView) SetResource(meta *internal.ResourceMeta, acc compdesc.AccessSpec, modopts ...ModificationOption) error { if c.impl.IsReadOnly() { return accessio.ErrReadOnly } @@ -651,7 +999,12 @@ func (c *componentVersionAccessView) SetResource(meta *internal.ResourceMeta, ac opts := internal.NewModificationOptions(modopts...) CompleteModificationOptions(ctx, opts) - meth, err := c.getMethod(acc) + spec, err := c.impl.GetContext().AccessSpecForSpec(acc) + if err != nil { + return err + } + + meth, err := c.AccessMethod(spec) if err != nil { return err } @@ -730,12 +1083,12 @@ func (c *componentVersionAccessView) SetResource(meta *internal.ResourceMeta, ac } else { cd.Resources[idx] = *res } - return c.impl.Update(false) + return c.update(false) }) } // evaluateResourceDigest evaluate given potentially partly set digest to determine defaults. -func (c *componentVersionAccessView) evaluateResourceDigest(res, old *compdesc.Resource, opts internal.ModificationOptions) (string, DigesterType, string) { +func (c *componentVersionAccessView) evaluateResourceDigest(res, old *compdesc.Resource, opts ModificationOptions) (string, DigesterType, string) { var digester DigesterType hashAlgo := opts.DefaultHashAlgorithm @@ -770,7 +1123,12 @@ func (c *componentVersionAccessView) evaluateResourceDigest(res, old *compdesc.R return hashAlgo, digester, value } -func (c *componentVersionAccessView) SetSource(meta *internal.SourceMeta, acc compdesc.AccessSpec) error { +func (c *componentVersionAccessView) SetSourceByAccess(art SourceAccess) error { + return setAccess(c, "source", art, + c.SetSource, c.SetSourceBlob) +} + +func (c *componentVersionAccessView) SetSource(meta *SourceMeta, acc compdesc.AccessSpec) error { if c.impl.IsReadOnly() { return accessio.ErrReadOnly } @@ -789,11 +1147,11 @@ func (c *componentVersionAccessView) SetSource(meta *internal.SourceMeta, acc co } else { cd.Sources[idx] = *res } - return c.impl.Update(false) + return c.update(false) }) } -func (c *componentVersionAccessView) SetReference(ref *internal.ComponentReference) error { +func (c *componentVersionAccessView) SetReference(ref *ComponentReference) error { return c.Execute(func() error { cd := c.impl.GetDescriptor() if idx := cd.GetComponentReferenceIndex(*ref); idx == -1 { @@ -801,7 +1159,7 @@ func (c *componentVersionAccessView) SetReference(ref *internal.ComponentReferen } else { cd.References[idx] = *ref } - return c.impl.Update(false) + return c.update(false) }) } @@ -813,6 +1171,10 @@ func (c *componentVersionAccessView) IsPersistent() bool { return c.impl.IsPersistent() } +func (c *componentVersionAccessView) UseDirectAccess() bool { + return c.impl.UseDirectAccess() +} + //////////////////////////////////////////////////////////////////////////////// // Standard Implementation for descriptor based methods @@ -821,7 +1183,7 @@ func (c *componentVersionAccessView) GetResource(id metav1.Identity) (ResourceAc if err != nil { return nil, err } - return newResourceAccess(c, r.Access, r.ResourceMeta), nil + return NewResourceAccess(c, r.Access, r.ResourceMeta), nil } func (c *componentVersionAccessView) GetResourceIndex(id metav1.Identity) int { @@ -833,7 +1195,7 @@ func (c *componentVersionAccessView) GetResourceByIndex(i int) (ResourceAccess, return nil, errors.ErrInvalid("resource index", strconv.Itoa(i)) } r := c.GetDescriptor().Resources[i] - return newResourceAccess(c, r.Access, r.ResourceMeta), nil + return NewResourceAccess(c, r.Access, r.ResourceMeta), nil } func (c *componentVersionAccessView) GetResourcesByName(name string, selectors ...compdesc.IdentitySelector) ([]ResourceAccess, error) { @@ -844,7 +1206,7 @@ func (c *componentVersionAccessView) GetResourcesByName(name string, selectors . result := []ResourceAccess{} for _, resource := range resources { - result = append(result, newResourceAccess(c, resource.Access, resource.ResourceMeta)) + result = append(result, NewResourceAccess(c, resource.Access, resource.ResourceMeta)) } return result, nil } @@ -852,7 +1214,7 @@ func (c *componentVersionAccessView) GetResourcesByName(name string, selectors . func (c *componentVersionAccessView) GetResources() []ResourceAccess { result := []ResourceAccess{} for _, r := range c.GetDescriptor().Resources { - result = append(result, newResourceAccess(c, r.Access, r.ResourceMeta)) + result = append(result, NewResourceAccess(c, r.Access, r.ResourceMeta)) } return result } @@ -906,7 +1268,7 @@ func (c *componentVersionAccessView) GetSource(id metav1.Identity) (SourceAccess if err != nil { return nil, err } - return newSourceAccess(c, r.Access, r.SourceMeta), nil + return NewSourceAccess(c, r.Access, r.SourceMeta), nil } func (c *componentVersionAccessView) GetSourceIndex(id metav1.Identity) int { @@ -918,13 +1280,13 @@ func (c *componentVersionAccessView) GetSourceByIndex(i int) (SourceAccess, erro return nil, errors.ErrInvalid("source index", strconv.Itoa(i)) } r := c.GetDescriptor().Sources[i] - return newSourceAccess(c, r.Access, r.SourceMeta), nil + return NewSourceAccess(c, r.Access, r.SourceMeta), nil } func (c *componentVersionAccessView) GetSources() []SourceAccess { result := []SourceAccess{} for _, r := range c.GetDescriptor().Sources { - result = append(result, newSourceAccess(c, r.Access, r.SourceMeta)) + result = append(result, NewSourceAccess(c, r.Access, r.SourceMeta)) } return result } diff --git a/pkg/contexts/ocm/cpi/view_rsc.go b/pkg/contexts/ocm/cpi/view_rsc.go index fc1d87bf36..bf22c15174 100644 --- a/pkg/contexts/ocm/cpi/view_rsc.go +++ b/pkg/contexts/ocm/cpi/view_rsc.go @@ -5,74 +5,157 @@ package cpi import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + ocm "github.com/open-component-model/ocm/pkg/contexts/ocm/context" + cpi "github.com/open-component-model/ocm/pkg/contexts/ocm/internal" + "github.com/open-component-model/ocm/pkg/contexts/ocm/plugin/descriptor" + "github.com/open-component-model/ocm/pkg/errors" ) //////////////////////////////////////////////////////////////////////////////// -type BaseAccess struct { +type ComponentVersionBasedAccessProvider struct { vers ComponentVersionAccess access compdesc.AccessSpec } -type baseAccess = BaseAccess +var _ AccessProvider = (*ComponentVersionBasedAccessProvider)(nil) -func NewBaseAccess(cv ComponentVersionAccess, acc compdesc.AccessSpec) *BaseAccess { - return &BaseAccess{vers: cv, access: acc} +// Deprecated: use ComponentVersionBasedAccessProvider. +type BaseAccess = ComponentVersionBasedAccessProvider + +type cvBaseAccess = ComponentVersionBasedAccessProvider + +func NewBaseAccess(cv ComponentVersionAccess, acc compdesc.AccessSpec) *ComponentVersionBasedAccessProvider { + return &ComponentVersionBasedAccessProvider{vers: cv, access: acc} +} + +func (r *ComponentVersionBasedAccessProvider) GetOCMContext() Context { + return r.vers.GetContext() +} + +func (r *ComponentVersionBasedAccessProvider) ReferenceHint() string { + if hp, ok := r.access.(cpi.HintProvider); ok { + return hp.GetReferenceHint(r.vers) + } + return "" } -func (r *BaseAccess) ComponentVersion() ComponentVersionAccess { - return r.vers +func (r *ComponentVersionBasedAccessProvider) GlobalAccess() AccessSpec { + acc, err := r.GetOCMContext().AccessSpecForSpec(r.access) + if err != nil { + return nil + } + return acc.GlobalAccessSpec(r.GetOCMContext()) } -func (r *BaseAccess) Access() (AccessSpec, error) { +func (r *ComponentVersionBasedAccessProvider) Access() (AccessSpec, error) { return r.vers.GetContext().AccessSpecForSpec(r.access) } -func (r *BaseAccess) AccessMethod() (AccessMethod, error) { +func (r *ComponentVersionBasedAccessProvider) AccessMethod() (AccessMethod, error) { acc, err := r.vers.GetContext().AccessSpecForSpec(r.access) if err != nil { return nil, err } - return r.vers.AccessMethod(acc) + return acc.AccessMethod(r.vers) +} + +func (r *ComponentVersionBasedAccessProvider) BlobAccess() (BlobAccess, error) { + m, err := r.AccessMethod() + if err != nil { + return nil, err + } + return BlobAccessForAccessMethod(AccessMethodAsView(m)) } //////////////////////////////////////////////////////////////////////////////// -type ResourceAccessImpl struct { - *baseAccess - meta ResourceMeta +type blobAccessProvider struct { + ctx ocm.Context + blobaccess.BlobAccessProvider + hint string + global AccessSpec } -var _ ResourceAccess = (*ResourceAccessImpl)(nil) +var _ AccessProvider = (*blobAccessProvider)(nil) -func newResourceAccess(componentVersion ComponentVersionAccess, accessSpec compdesc.AccessSpec, meta ResourceMeta) *ResourceAccessImpl { - return &ResourceAccessImpl{ - baseAccess: NewBaseAccess(componentVersion, accessSpec), - meta: meta, +func NewAccessProviderForBlobAccessProvider(ctx ocm.Context, prov blobaccess.BlobAccessProvider, hint string, global AccessSpec) AccessProvider { + return &blobAccessProvider{ + BlobAccessProvider: prov, + hint: hint, + global: global, + ctx: ctx, } } -func (r *ResourceAccessImpl) Meta() *ResourceMeta { - return &r.meta +func (b *blobAccessProvider) GetOCMContext() cpi.Context { + return b.ctx +} + +func (b *blobAccessProvider) ReferenceHint() string { + return b.hint +} + +func (b *blobAccessProvider) GlobalAccess() cpi.AccessSpec { + return b.global +} + +func (b blobAccessProvider) Access() (cpi.AccessSpec, error) { + return nil, errors.ErrNotFound(descriptor.KIND_ACCESSMETHOD) +} + +func (b *blobAccessProvider) AccessMethod() (cpi.AccessMethod, error) { + return nil, errors.ErrNotFound(descriptor.KIND_ACCESSMETHOD) } //////////////////////////////////////////////////////////////////////////////// -type SourceAccessImpl struct { - *baseAccess - meta SourceMeta +func NewArtifactAccessProviderForBlobAccessProvider[M any](ctx Context, meta *M, src blobAccessProvider, hint string, global AccessSpec) cpi.ArtifactAccess[M] { + return NewArtifactAccessForProvider(meta, NewAccessProviderForBlobAccessProvider(ctx, src, hint, global)) } -var _ SourceAccess = (*SourceAccessImpl)(nil) +//////////////////////////////////////////////////////////////////////////////// + +type accessProvider = AccessProvider + +type artifactAccessProvider[M any] struct { + accessProvider + meta *M +} -func newSourceAccess(componentVersion ComponentVersionAccess, accessSpec compdesc.AccessSpec, meta SourceMeta) *SourceAccessImpl { - return &SourceAccessImpl{ - baseAccess: NewBaseAccess(componentVersion, accessSpec), - meta: meta, +func NewArtifactAccessForProvider[M any](meta *M, prov AccessProvider) cpi.ArtifactAccess[M] { + return &artifactAccessProvider[M]{ + accessProvider: prov, + meta: meta, } } -func (r SourceAccessImpl) Meta() *SourceMeta { - return &r.meta +func (r *artifactAccessProvider[M]) Meta() *M { + return r.meta +} + +//////////////////////////////////////////////////////////////////////////////// + +var _ ResourceAccess = (*artifactAccessProvider[ResourceMeta])(nil) + +func NewResourceAccess(componentVersion ComponentVersionAccess, accessSpec compdesc.AccessSpec, meta ResourceMeta) ResourceAccess { + return NewResourceAccessForProvider(&meta, NewBaseAccess(componentVersion, accessSpec)) +} + +func NewResourceAccessForProvider(meta *ResourceMeta, prov AccessProvider) ResourceAccess { + return NewArtifactAccessForProvider(meta, prov) +} + +//////////////////////////////////////////////////////////////////////////////// + +var _ SourceAccess = (*artifactAccessProvider[SourceMeta])(nil) + +func NewSourceAccess(componentVersion ComponentVersionAccess, accessSpec compdesc.AccessSpec, meta SourceMeta) SourceAccess { + return NewSourceAccessForProvider(&meta, NewBaseAccess(componentVersion, accessSpec)) +} + +func NewSourceAccessForProvider(meta *SourceMeta, prov AccessProvider) SourceAccess { + return NewArtifactAccessForProvider(meta, prov) } diff --git a/pkg/contexts/ocm/digester/digesters/artifact/digester.go b/pkg/contexts/ocm/digester/digesters/artifact/digester.go index efe391b0f8..8ce43d1671 100644 --- a/pkg/contexts/ocm/digester/digesters/artifact/digester.go +++ b/pkg/contexts/ocm/digester/digesters/artifact/digester.go @@ -6,17 +6,15 @@ package artifact import ( "archive/tar" - "compress/gzip" "fmt" "io" - "strings" "github.com/opencontainers/go-digest" "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/compression" "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" - "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/errors" @@ -63,7 +61,7 @@ func (d *Digester) GetType() cpi.DigesterType { } func (d *Digester) DetermineDigest(reftyp string, acc cpi.AccessMethod, preferred signing.Hasher) (*cpi.DigestDescriptor, error) { - if acc.GetKind() == localblob.Type { + if acc.IsLocal() { mime := acc.MimeType() if !artdesc.IsOCIMediaType(mime) { return nil, nil @@ -74,13 +72,12 @@ func (d *Digester) DetermineDigest(reftyp string, acc cpi.AccessMethod, preferre } defer r.Close() - var reader io.Reader = r - if strings.HasSuffix(mime, "+gzip") { - reader, err = gzip.NewReader(reader) - if err != nil { - return nil, err - } + var reader io.ReadCloser + reader, _, err = compression.AutoDecompress(r) + if err != nil { + return nil, err } + defer reader.Close() tr := tar.NewReader(reader) var desc *cpi.DigestDescriptor @@ -124,7 +121,7 @@ func (d *Digester) DetermineDigest(reftyp string, acc cpi.AccessMethod, preferre if index == nil { return nil, fmt.Errorf("no main artifact found") } - main := artifactset.RetrieveMainArtifact(index.Annotations) + main := artifactset.RetrieveMainArtifactFromIndex(index) if main == "" { return nil, fmt.Errorf("no main artifact found") } diff --git a/pkg/contexts/ocm/download/handlers/helm/download.go b/pkg/contexts/ocm/download/handlers/helm/download.go index 3abd0d4e00..ec587b395c 100644 --- a/pkg/contexts/ocm/download/handlers/helm/download.go +++ b/pkg/contexts/ocm/download/handlers/helm/download.go @@ -57,6 +57,9 @@ func Download2(p common.Printer, ctx oci.Context, ref string, path string, fs vf return "", "", "", errors.Wrapf(err, "cannot create artifact set") } err = artifactset.TransferArtifact(art, ctf) + if err == nil { + ctf.Annotate(artifactset.MAINARTIFACT_ANNOTATION, art.Digest().String()) + } ctf.Close() if err != nil { fs.Remove(aset) diff --git a/pkg/contexts/ocm/download/handlers/ocirepo/handler.go b/pkg/contexts/ocm/download/handlers/ocirepo/handler.go index 42a93cc688..806e31da26 100644 --- a/pkg/contexts/ocm/download/handlers/ocirepo/handler.go +++ b/pkg/contexts/ocm/download/handlers/ocirepo/handler.go @@ -41,7 +41,7 @@ func (h *handler) Download(p common.Printer, racc cpi.ResourceAccess, path strin var finalize finalizer.Finalizer defer finalize.FinalizeWithErrorPropagationf(&err, "upload to OCI registry") - ctx := racc.ComponentVersion().GetContext() + ctx := racc.GetOCMContext() m, err := racc.AccessMethod() if err != nil { return false, "", err @@ -58,13 +58,11 @@ func (h *handler) Download(p common.Printer, racc cpi.ResourceAccess, path strin var repo oci.Repository - var namespace string var version string = "latest" aspec := m.AccessSpec() - if hp, ok := aspec.(cpi.HintProvider); ok { - namespace = hp.GetReferenceHint(racc.ComponentVersion()) - } else if l, ok := aspec.(*localblob.AccessSpec); ok { + namespace := racc.ReferenceHint() + if l, ok := aspec.(*localblob.AccessSpec); namespace == "" && ok { namespace = l.ReferenceName } diff --git a/pkg/contexts/ocm/elements/artifacts.go b/pkg/contexts/ocm/elements/artifacts.go new file mode 100644 index 0000000000..1bc8bdc05f --- /dev/null +++ b/pkg/contexts/ocm/elements/artifacts.go @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +type ArtifactOption interface { + ResourceMetaOption + SourceMetaOption +} diff --git a/pkg/contexts/ocm/elements/common.go b/pkg/contexts/ocm/elements/common.go new file mode 100644 index 0000000000..2ef428f00a --- /dev/null +++ b/pkg/contexts/ocm/elements/common.go @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" +) + +type CommonOption interface { + ResourceMetaOption + SourceMetaOption + ReferenceOption +} + +type commonOption struct { + apply func(meta *compdesc.ElementMeta) error +} + +type commonOptionI interface { + apply(*compdesc.ElementMeta) error +} + +func newCommonOption[T commonOptionI](e T) CommonOption { + return commonOption{e.apply} +} + +func (o commonOption) ApplyToResourceMeta(m *compdesc.ResourceMeta) error { + return o.apply(&m.ElementMeta) +} + +func (o commonOption) ApplyToSourceMeta(m *compdesc.SourceMeta) error { + return o.apply(&m.ElementMeta) +} + +func (o commonOption) ApplyToReference(m *compdesc.ComponentReference) error { + return o.apply(&m.ElementMeta) +} + +//////////////////////////////////////////////////////////////////////////////// + +type version string + +func (o version) apply(m *compdesc.ElementMeta) error { + m.Version = string(o) + return nil +} + +// WithVersion sets the version of the element. +func WithVersion(v string) CommonOption { + return newCommonOption(version(v)) +} + +//////////////////////////////////////////////////////////////////////////////// + +type extraIdentity struct { + id metav1.Identity +} + +func (o *extraIdentity) apply(m *compdesc.ElementMeta) error { + if m.ExtraIdentity == nil { + m.ExtraIdentity = o.id.Copy() + } else { + for n, v := range o.id { + m.ExtraIdentity.Set(n, v) + } + } + return nil +} + +// WithExtraIdentity adds extra identity properties. +func WithExtraIdentity(extras ...string) CommonOption { + return newCommonOption(&extraIdentity{compdesc.NewExtraIdentity(extras...)}) +} + +//////////////////////////////////////////////////////////////////////////////// + +type label struct { + name string + value interface{} + opts []metav1.LabelOption +} + +func (o *label) apply(m *compdesc.ElementMeta) error { + return m.Labels.Set(o.name, o.value, o.opts...) +} + +func WithLabel(name string, value interface{}, opts ...metav1.LabelOption) CommonOption { + return newCommonOption(&label{name, value, opts}) +} diff --git a/pkg/contexts/ocm/elements/digests.go b/pkg/contexts/ocm/elements/digests.go new file mode 100644 index 0000000000..690d51971c --- /dev/null +++ b/pkg/contexts/ocm/elements/digests.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" +) + +type ResourceReferenceOption interface { + ResourceMetaOption + ReferenceOption +} + +//////////////////////////////////////////////////////////////////////////////// + +type digest metav1.DigestSpec + +func (o *digest) ApplyToReference(m *compdesc.ComponentReference) error { + if !(*metav1.DigestSpec)(o).IsNone() { + m.Digest = (*metav1.DigestSpec)(o).Copy() + } + return nil +} + +func (o *digest) ApplyToResourceMeta(m *compdesc.ResourceMeta) error { + if !(*metav1.DigestSpec)(o).IsNone() { + m.Digest = (*metav1.DigestSpec)(o).Copy() + } + return nil +} + +// WithDigest sets digest information. +// at least one value should be set. +func WithDigest(algo, norm, value string) ResourceReferenceOption { + return &digest{HashAlgorithm: algo, NormalisationAlgorithm: norm, Value: value} +} diff --git a/pkg/contexts/ocm/elements/references.go b/pkg/contexts/ocm/elements/references.go new file mode 100644 index 0000000000..cbdf953354 --- /dev/null +++ b/pkg/contexts/ocm/elements/references.go @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/errors" +) + +type ReferenceOption interface { + ApplyToReference(reference *compdesc.ComponentReference) error +} + +func Reference(name, comp, vers string, opts ...ReferenceOption) (*compdesc.ComponentReference, error) { + m := compdesc.NewComponentReference(name, comp, vers, nil) + list := errors.ErrList() + for _, o := range opts { + if o != nil { + list.Add(o.ApplyToReference(m)) + } + } + return m, list.Result() +} diff --git a/pkg/contexts/ocm/elements/resources.go b/pkg/contexts/ocm/elements/resources.go new file mode 100644 index 0000000000..2f319e85e3 --- /dev/null +++ b/pkg/contexts/ocm/elements/resources.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/utils" +) + +type ResourceMetaOption interface { + ApplyToResourceMeta(*compdesc.ResourceMeta) error +} + +func ResourceMeta(name, typ string, opts ...ResourceMetaOption) (*compdesc.ResourceMeta, error) { + m := compdesc.NewResourceMeta(name, typ, metav1.LocalRelation) + list := errors.ErrList() + for _, o := range opts { + if o != nil { + list.Add(o.ApplyToResourceMeta(m)) + } + } + return m, list.Result() +} + +//////////////////////////////////////////////////////////////////////////////// + +type local bool + +func (o local) ApplyToResourceMeta(m *compdesc.ResourceMeta) error { + if o { + m.Relation = metav1.LocalRelation + } else { + m.Relation = metav1.ExternalRelation + } + return nil +} + +// WithLocalRelation sets the resource relation to metav1.LocalRelation. +func WithLocalRelation(flag ...bool) ResourceMetaOption { + return local(utils.OptionalDefaultedBool(true, flag...)) +} + +// WithExternalRelation sets the resource relation to metav1.ExternalRelation. +func WithExternalRelation(flag ...bool) ResourceMetaOption { + return local(!utils.OptionalDefaultedBool(true, flag...)) +} + +//////////////////////////////////////////////////////////////////////////////// + +type srcref struct { + ref metav1.StringMap + labels metav1.Labels + errlist errors.ErrorList +} + +var _ ResourceMetaOption = (*srcref)(nil) + +func (o *srcref) ApplyToResourceMeta(m *compdesc.ResourceMeta) error { + if err := o.errlist.Result(); err != nil { + return err + } + m.SourceRef = append(m.SourceRef, compdesc.SourceRef{IdentitySelector: o.ref.Copy(), Labels: o.labels.Copy()}) + return nil +} + +func (o *srcref) WithLabel(name string, value interface{}, opts ...metav1.LabelOption) *srcref { + r := &srcref{ref: o.ref, labels: o.labels.Copy()} + r.errlist.Add(r.labels.Set(name, value, opts...)) + return r +} + +// WithSourceRef adds a source reference to a resource meta object. +// this is a sequence of name/value pairs. +// Optionally, additional labels can be added with srcref.WithLabel. +func WithSourceRef(sel ...string) *srcref { + return &srcref{ref: metav1.StringMap(metav1.NewExtraIdentity(sel...))} +} diff --git a/pkg/contexts/ocm/elements/resources_test.go b/pkg/contexts/ocm/elements/resources_test.go new file mode 100644 index 0000000000..9fe084a0a1 --- /dev/null +++ b/pkg/contexts/ocm/elements/resources_test.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/contexts/ocm/digester/digesters/blob" + me "github.com/open-component-model/ocm/pkg/contexts/ocm/elements" + "github.com/open-component-model/ocm/pkg/signing/hasher/sha256" +) + +type value struct { + Field string `json:"field"` +} + +var _ = Describe("resources", func() { + It("configures resource meta", func() { + m := Must(me.ResourceMeta("name", "type", + me.WithVersion("v1"), + me.WithExtraIdentity("extra", "value"), + me.WithLabel("label", value{"value"}, metav1.WithSigning(), metav1.WithVersion("v1")), + me.WithSourceRef("name", "image").WithLabel("prop", "x"), + me.WithDigest(sha256.Algorithm, blob.GenericBlobDigestV1, "0815"), + )) + Expect(m).To(YAMLEqual(` +name: name +type: type +version: v1 +relation: local +extraIdentity: + extra: value +labels: + - name: label + version: v1 + value: + field: value + signing: true +srcRef: + - identitySelector: + name: image + labels: + - name: prop + value: x +digest: + hashAlgorithm: SHA-256 + normalisationAlgorithm: genericBlobDigest/v1 + value: "0815" +`)) + }) +}) diff --git a/pkg/contexts/ocm/elements/sources.go b/pkg/contexts/ocm/elements/sources.go new file mode 100644 index 0000000000..30080dcc02 --- /dev/null +++ b/pkg/contexts/ocm/elements/sources.go @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/errors" +) + +type SourceMetaOption interface { + ApplyToSourceMeta(*compdesc.SourceMeta) error +} + +func SourceMeta(name, typ string, opts ...SourceMetaOption) (*compdesc.SourceMeta, error) { + m := compdesc.NewSourceMeta(name, typ) + list := errors.ErrList() + for _, o := range opts { + if o != nil { + list.Add(o.ApplyToSourceMeta(m)) + } + } + return m, list.Result() +} diff --git a/pkg/contexts/ocm/elements/suite_test.go b/pkg/contexts/ocm/elements/suite_test.go new file mode 100644 index 0000000000..79af1da3b6 --- /dev/null +++ b/pkg/contexts/ocm/elements/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package elements_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Element builder") +} diff --git a/pkg/contexts/ocm/internal/accessspecref.go b/pkg/contexts/ocm/internal/accessspecref.go index 4144a53f46..5f30c43b39 100644 --- a/pkg/contexts/ocm/internal/accessspecref.go +++ b/pkg/contexts/ocm/internal/accessspecref.go @@ -55,6 +55,13 @@ func (a *AccessSpecRef) Set(spec AccessSpec) { } } +func (a *AccessSpecRef) Unwrap() AccessSpec { + if a == nil { + return nil + } + return a +} + func (a *AccessSpecRef) Describe(ctx Context) string { err := a.assure(ctx) if a.evaluated != nil { diff --git a/pkg/contexts/ocm/internal/accesstypes.go b/pkg/contexts/ocm/internal/accesstypes.go index ac35eeacd4..02f18adf1a 100644 --- a/pkg/contexts/ocm/internal/accesstypes.go +++ b/pkg/contexts/ocm/internal/accesstypes.go @@ -7,6 +7,7 @@ package internal import ( "encoding/json" "fmt" + "io" "github.com/modern-go/reflect2" @@ -64,6 +65,12 @@ type HintProvider interface { GetReferenceHint(cv ComponentVersionAccess) string } +// GlobalAccessProvider is used to provide a non-local access specification. +// It may optionally be provided by an access spec. +type GlobalAccessProvider interface { + GlobalAccessSpec(ctx Context) AccessSpec +} + // AccessMethod described the access to a dedicated resource // It can allocate external resources, which should be released // with the Close() call. @@ -71,12 +78,23 @@ type HintProvider interface { // via the DataAccess interface to avoid unnecessary effort // if the method object is just used to access meta data. type AccessMethod interface { + io.Closer DataAccess + MimeType + IsLocal() bool GetKind() string AccessSpec() AccessSpec - MimeType - Close() error +} + +// AccessMethodView can be used map wrap an access method +// into a managed method with multiple views. The original method +// object is closed once the last view is closed. +type AccessMethodView interface { + AccessMethod + + Base() interface{} + Dup() (AccessMethodView, error) } type AccessTypeScheme flagsetscheme.TypeScheme[AccessSpec, AccessType] diff --git a/pkg/contexts/ocm/internal/digesthandler_test.go b/pkg/contexts/ocm/internal/digesthandler_test.go index c17f7c06b7..1940081138 100644 --- a/pkg/contexts/ocm/internal/digesthandler_test.go +++ b/pkg/contexts/ocm/internal/digesthandler_test.go @@ -100,6 +100,10 @@ type AccessMethod struct { var _ internal.AccessMethod = (*AccessMethod)(nil) +func (_ AccessMethod) IsLocal() bool { + return false +} + func (a AccessMethod) GetKind() string { return "demo" } diff --git a/pkg/contexts/ocm/internal/repository.go b/pkg/contexts/ocm/internal/repository.go index 10f09a0bd1..63ad2c7b1c 100644 --- a/pkg/contexts/ocm/internal/repository.go +++ b/pkg/contexts/ocm/internal/repository.go @@ -34,8 +34,10 @@ type RepositoryImpl interface { type Repository interface { resource.ResourceView[Repository] - RepositoryImpl + + NewVersion(comp, version string, overrides ...bool) (ComponentVersionAccess, error) + AddVersion(cv ComponentVersionAccess, overrides ...bool) error } // ConsumerIdentityProvider is an interface for object requiring @@ -56,7 +58,6 @@ type ComponentAccessImpl interface { ListVersions() ([]string, error) LookupVersion(version string) (ComponentVersionAccess, error) HasVersion(vers string) (bool, error) - AddVersion(ComponentVersionAccess) error NewVersion(version string, overrides ...bool) (ComponentVersionAccess, error) Close() error @@ -66,48 +67,58 @@ type ComponentAccess interface { resource.ResourceView[ComponentAccess] ComponentAccessImpl + AddVersion(cv ComponentVersionAccess, overrides ...bool) error } -type ( - ResourceMeta = compdesc.ResourceMeta - ComponentReference = compdesc.ComponentReference -) - +// AccessProvider assembled methods provided +// and used for access methods. +// It is provided for resources in a component version +// with the base support implementation in package cpi. +// But it can also be provided by resource provisioners +// used to feed the ComponentVersionAccess.SetResourceByAccess +// or the ComponentVersionAccessSetSourceByAccess +// method. type AccessProvider interface { - ComponentVersion() ComponentVersionAccess + GetOCMContext() Context + ReferenceHint() string + Access() (AccessSpec, error) AccessMethod() (AccessMethod, error) + + GlobalAccess() AccessSpec + + blobaccess.BlobAccessProvider } -type ResourceAccess interface { - Meta() *ResourceMeta +type ArtifactAccess[M any] interface { + Meta() *M AccessProvider } -type SourceMeta = compdesc.SourceMeta +type ( + ResourceMeta = compdesc.ResourceMeta + ResourceAccess = ArtifactAccess[ResourceMeta] +) -type SourceAccess interface { - Meta() *SourceMeta - AccessProvider -} +type ( + SourceMeta = compdesc.SourceMeta + SourceAccess = ArtifactAccess[SourceMeta] +) + +type ComponentReference = compdesc.ComponentReference -type ComponentVersionAccessImpl interface { +type ComponentVersionAccess interface { + resource.ResourceView[ComponentVersionAccess] common.VersionedElement + io.Closer - Repository() Repository GetContext() Context + Repository() Repository GetDescriptor() *compdesc.ComponentDescriptor + DiscardChanges() IsPersistent() bool - io.Closer -} - -type ComponentVersionAccess interface { - resource.ResourceView[ComponentVersionAccess] - - ComponentVersionAccessImpl - GetResources() []ResourceAccess GetResource(meta metav1.Identity) (ResourceAccess, error) GetResourceIndex(meta metav1.Identity) int @@ -116,12 +127,14 @@ type ComponentVersionAccess interface { GetResourcesByIdentitySelectors(selectors ...compdesc.IdentitySelector) ([]ResourceAccess, error) GetResourcesByResourceSelectors(selectors ...compdesc.ResourceSelector) ([]ResourceAccess, error) SetResource(*ResourceMeta, compdesc.AccessSpec, ...ModificationOption) error + SetResourceByAccess(art ResourceAccess, modopts ...ModificationOption) error GetSources() []SourceAccess GetSource(meta metav1.Identity) (SourceAccess, error) GetSourceIndex(meta metav1.Identity) int GetSourceByIndex(i int) (SourceAccess, error) SetSource(*SourceMeta, compdesc.AccessSpec) error + SetSourceByAccess(art SourceAccess) error GetReference(meta metav1.Identity) (ComponentReference, error) GetReferenceIndex(meta metav1.Identity) int @@ -156,6 +169,9 @@ type ComponentVersionAccess interface { // calculating the digest from the bytes), which would defeat the purpose of caching. // It follows the same contract as AccessMethod. GetInexpensiveContentVersionIdentity(spec AccessSpec) string + + // Update adds the version with all changes to the component instance it has been created for. + Update() error } // ComponentLister provides the optional repository list functionality of diff --git a/pkg/contexts/ocm/labels/routingslip/transfer_test.go b/pkg/contexts/ocm/labels/routingslip/transfer_test.go index 3e5d61f648..782862a5a1 100644 --- a/pkg/contexts/ocm/labels/routingslip/transfer_test.go +++ b/pkg/contexts/ocm/labels/routingslip/transfer_test.go @@ -17,6 +17,7 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/labels/routingslip" "github.com/open-component-model/ocm/pkg/contexts/ocm/labels/routingslip/types/comment" "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" @@ -42,11 +43,12 @@ var _ = Describe("management", func() { env.RSAKeyPair(ORG, LOCAL) }) - It("transfers and updates", func() { + DescribeTable("transfers and updates", func(mode bool) { var finalize finalizer.Finalizer defer Defer(finalize.Finalize, "finalizer") + compositionmodeattr.Set(env.OCMContext(), mode) e1 := comment.New("start of routing slip") e2 := comment.New("additional entry") @@ -71,14 +73,19 @@ var _ = Describe("management", func() { transferring version "acme.org/routingslip:1.0.0"... ...adding component version... `)) - tcv := Must(target.LookupComponentVersion(COMPONENT, VERSION)) - slip, err := routingslip.GetSlip(tcv, ORG) - MustBeSuccessful(routingslip.AddEntry(tcv, LOCAL, rsa.Algorithm, e1, nil)) - MustBeSuccessful(tcv.Close()) - MustBeSuccessful(err) + nested := finalize.Nested() + tc := Must(target.LookupComponent(COMPONENT)) + nested.Close(tc, "target comp") + tcv := Must(tc.LookupVersion(VERSION)) + nested.Close(tcv) + slip := Must(routingslip.GetSlip(tcv, ORG)) + MustBeSuccessful(routingslip.AddEntry(tcv, LOCAL, rsa.Algorithm, e1, nil)) Expect(slip.Len()).To(Equal(1)) + MustBeSuccessful(tc.AddVersion(tcv)) + MustBeSuccessful(nested.Finalize()) + buf.Reset() MustBeSuccessful(routingslip.AddEntry(cv, ORG, rsa.Algorithm, e2, nil)) MustBeSuccessful(transfer.TransferVersion(pr, nil, cv, target, Must(standard.New()))) @@ -95,5 +102,8 @@ transferring version "acme.org/routingslip:1.0.0"... Expect(len(label[ORG])).To(Equal(2)) Expect(len(label[LOCAL])).To(Equal(1)) fmt.Printf("*** routing slips:\n%s\n", Must(runtime.DefaultYAMLEncoding.Marshal(label))) - }) + }, + Entry("with direct mode", false), + Entry("with composition mode", true), + ) }) diff --git a/pkg/contexts/ocm/repositories/comparch/accessmethod_localfs.go b/pkg/contexts/ocm/repositories/comparch/accessmethod_localfs.go index 007c2796cc..5baee6cd1e 100644 --- a/pkg/contexts/ocm/repositories/comparch/accessmethod_localfs.go +++ b/pkg/contexts/ocm/repositories/comparch/accessmethod_localfs.go @@ -34,6 +34,10 @@ func newLocalFilesystemBlobAccessMethod(a *localblob.AccessSpec, base support.Co } } +func (_ *localFilesystemBlobAccessMethod) IsLocal() bool { + return true +} + func (m *localFilesystemBlobAccessMethod) AccessSpec() cpi.AccessSpec { return m.spec } diff --git a/pkg/contexts/ocm/repositories/comparch/comparch_test.go b/pkg/contexts/ocm/repositories/comparch/comparch_test.go index 335240d742..ed710eec09 100644 --- a/pkg/contexts/ocm/repositories/comparch/comparch_test.go +++ b/pkg/contexts/ocm/repositories/comparch/comparch_test.go @@ -59,12 +59,12 @@ var _ = Describe("Repository", func() { octx := ocm.DefaultContext() spec := Must(comparch.NewRepositorySpec(accessobj.ACC_READONLY, TAR_COMPARCH)) repo := Must(spec.Repository(octx, nil)) - defer Close(repo) + defer Close(repo, "repo") cv := Must(repo.LookupComponentVersion(COMPONENT_NAME, COMPONENT_VERSION)) - defer Close(cv) + defer Close(cv, "compvers") res := Must(cv.GetResourcesByName(RESOURCE_NAME)) acc := Must(res[0].AccessMethod()) - defer Close(acc) + defer Close(acc, "method") bytesA := Must(acc.Get()) bytesB := Must(vfs.ReadFile(osfs.New(), filepath.Join(TAR_COMPARCH, "blobs", "sha256.3ed99e50092c619823e2c07941c175ea2452f1455f570c55510586b387ec2ff2"))) @@ -96,8 +96,11 @@ var _ = Describe("Repository", func() { octx := ocm.DefaultContext() memfs := memoryfs.New() + var finalize finalizer.Finalizer + defer Defer(finalize.Finalize) + arch := Must(comparch.Create(octx, accessobj.ACC_WRITABLE, "test", 0o0700, accessio.PathFileSystem(memfs))) - defer Close(arch, "comparch)") + finalize.Close(arch, "comparch)") arch.SetName("acme.org/test") arch.SetVersion("v1.0.1") @@ -111,6 +114,18 @@ var _ = Describe("Repository", func() { NormalisationAlgorithm: blob.GenericBlobDigestV1, Value: D_TESTDATA, })) + + MustBeSuccessful(finalize.Finalize()) + + arch = Must(comparch.Open(octx, accessobj.ACC_WRITABLE, "test", 0o0700, accessio.PathFileSystem(memfs))) + finalize.Close(arch, "comparch)") + + res = Must(arch.GetResourcesByName("blob")) + Expect(res[0].Meta().Digest).To(DeepEqual(&metav1.DigestSpec{ + HashAlgorithm: sha256.Algorithm, + NormalisationAlgorithm: blob.GenericBlobDigestV1, + Value: D_TESTDATA, + })) }) It("closing a resource before actually reading it", func() { @@ -146,4 +161,68 @@ var _ = Describe("Repository", func() { finalize.Close(cv, "cv") Expect(cv.GetDescriptor().Provider.Name).To(Equal(metav1.ProviderName("modified provider"))) }) + + It("component archive from spec with New/AddVersion", func() { + var finalize finalizer.Finalizer + defer Defer(finalize.Finalize) + + env := env.NewEnvironment(env.ModifiableTestData()) + octx := env.OCMContext() + spec := Must(comparch.NewRepositorySpec(accessobj.ACC_WRITABLE, TAR_COMPARCH, accessio.PathFileSystem(env))) + repo := Must(spec.Repository(octx, nil)) + finalize.Close(repo, "repo") + comp := Must(repo.LookupComponent(COMPONENT_NAME)) + finalize.Close(comp, "component") + cv := Must(comp.NewVersion(COMPONENT_VERSION, true)) + finalize.Close(cv, "compvers") + + MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), + blobaccess.ForString(mime.MIME_TEXT, S_OTHERDATA), "", nil)) + + MustBeSuccessful(comp.AddVersion(cv)) + + MustBeSuccessful(finalize.Finalize()) + + arch := Must(comparch.Open(octx, accessobj.ACC_READONLY, TAR_COMPARCH, 0o0700, accessio.PathFileSystem(env))) + finalize.Close(arch, "comparch)") + + res := Must(arch.GetResourcesByName("blob")) + Expect(res[0].Meta().Digest).To(DeepEqual(&metav1.DigestSpec{ + HashAlgorithm: sha256.Algorithm, + NormalisationAlgorithm: blob.GenericBlobDigestV1, + Value: D_OTHERDATA, + })) + }) + + It("handle multiple lookups", func() { + var finalize finalizer.Finalizer + defer Defer(finalize.Finalize) + + env := env.NewEnvironment(env.ModifiableTestData()) + octx := env.OCMContext() + spec := Must(comparch.NewRepositorySpec(accessobj.ACC_WRITABLE, TAR_COMPARCH, accessio.PathFileSystem(env))) + repo := Must(spec.Repository(octx, nil)) + finalize.Close(repo, "repo") + cv1 := Must(repo.LookupComponentVersion(COMPONENT_NAME, COMPONENT_VERSION)) + finalize.Close(cv1, "version1") + + cv2 := Must(repo.LookupComponentVersion(COMPONENT_NAME, COMPONENT_VERSION)) + + MustBeSuccessful(cv2.Close()) + + MustBeSuccessful(cv1.SetResourceBlob(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), + blobaccess.ForString(mime.MIME_TEXT, S_OTHERDATA), "", nil)) + + MustBeSuccessful(finalize.Finalize()) + + arch := Must(comparch.Open(octx, accessobj.ACC_READONLY, TAR_COMPARCH, 0o0700, accessio.PathFileSystem(env))) + finalize.Close(arch, "comparch)") + + res := Must(arch.GetResourcesByName("blob")) + Expect(res[0].Meta().Digest).To(DeepEqual(&metav1.DigestSpec{ + HashAlgorithm: sha256.Algorithm, + NormalisationAlgorithm: blob.GenericBlobDigestV1, + Value: D_OTHERDATA, + })) + }) }) diff --git a/pkg/contexts/ocm/repositories/comparch/componentarchive.go b/pkg/contexts/ocm/repositories/comparch/componentarchive.go index 5d6abba150..83acdcd543 100644 --- a/pkg/contexts/ocm/repositories/comparch/componentarchive.go +++ b/pkg/contexts/ocm/repositories/comparch/componentarchive.go @@ -50,7 +50,7 @@ func _Wrap(ctx cpi.ContextProvider, obj *accessobj.AccessObject, spec *Repositor ctx: ctx.OCMContext(), base: accessobj.NewFileSystemBlobAccess(obj), } - impl, err := support.NewComponentVersionAccessImpl(s.GetDescriptor().GetName(), s.GetDescriptor().GetVersion(), s, false, true) + impl, err := support.NewComponentVersionAccessImpl(s.GetDescriptor().GetName(), s.GetDescriptor().GetVersion(), s, false, true, true) if err != nil { return nil, err } diff --git a/pkg/contexts/ocm/repositories/comparch/repository.go b/pkg/contexts/ocm/repositories/comparch/repository.go index 86279b4cb2..32632ea13c 100644 --- a/pkg/contexts/ocm/repositories/comparch/repository.go +++ b/pkg/contexts/ocm/repositories/comparch/repository.go @@ -5,14 +5,23 @@ package comparch import ( + "fmt" "strings" "sync" "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessio/refmgmt" "github.com/open-component-model/ocm/pkg/contexts/datacontext/attrs/vfsattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localfsblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" + ocmhdlr "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/support" "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/utils" ) type _RepositoryImplBase = cpi.RepositoryImplBase @@ -37,6 +46,7 @@ func NewRepository(ctx cpi.Context, s *RepositorySpec) (cpi.Repository, error) { } func newRepository(a *ComponentArchive) (main cpi.Repository, nonref cpi.Repository) { + // close main cv abstraction on repository close -------v base := cpi.NewRepositoryImplBase(a.GetContext(), a.ComponentVersionAccess) impl := &RepositoryImpl{ _RepositoryImplBase: *base, @@ -111,13 +121,18 @@ func (r *RepositoryImpl) LookupComponentVersion(name string, version string) (cp r.lock.RLock() defer r.lock.RUnlock() ok, err := r.ExistsComponentVersion(name, version) - if ok { - return r.arch.Dup() + if !ok { + if err == nil { + err = errors.ErrNotFound(cpi.KIND_COMPONENTVERSION, common.NewNameVersion(name, version).String(), Type) + } + return nil, err } - if err == nil { - err = errors.ErrNotFound(cpi.KIND_COMPONENTVERSION, common.NewNameVersion(name, version).String(), Type) + c, err := newComponentAccess(r) + if err != nil { + return nil, err } - return nil, err + defer refmgmt.PropagateCloseTemporary(&err, c) // temporary component object not exposed. + return c.LookupVersion(version) } func (r *RepositoryImpl) LookupComponent(name string) (cpi.ComponentAccess, error) { @@ -167,14 +182,150 @@ func (c *ComponentAccessImpl) HasVersion(vers string) (bool, error) { return vers == c.repo.arch.GetVersion(), nil } -func (c *ComponentAccessImpl) LookupVersion(ref string) (cpi.ComponentVersionAccess, error) { - return c.repo.LookupComponentVersion(c.repo.arch.GetName(), ref) +func (c *ComponentAccessImpl) LookupVersion(version string) (cpi.ComponentVersionAccess, error) { + if version != c.repo.arch.GetVersion() { + return nil, errors.ErrNotFound(cpi.KIND_COMPONENTVERSION, fmt.Sprintf("%s:%s", c.GetName(), c.repo.arch.GetVersion())) + } + return newComponentVersionAccess(c, version, false) +} + +func (c *ComponentAccessImpl) container(access cpi.ComponentVersionAccess) *componentArchiveContainer { + mine, _ := support.GetComponentVersionContainer[*ComponentVersionContainer](access) + if mine == nil || mine.comp != c { + return nil + } + return mine.comp.repo.arch.container +} + +func (c *ComponentAccessImpl) IsOwned(access cpi.ComponentVersionAccess) bool { + return c.container(access) == c.repo.arch.container } func (c *ComponentAccessImpl) AddVersion(access cpi.ComponentVersionAccess) error { - return errors.ErrNotSupported(errors.KIND_FUNCTION, "add version", Type) + if access.GetName() != c.GetName() { + return errors.ErrInvalid("component name", access.GetName()) + } + mine := c.container(access) + if mine == nil { + return errors.Newf("component version not owned by component archive") + } + return nil } func (c *ComponentAccessImpl) NewVersion(version string, overrides ...bool) (cpi.ComponentVersionAccess, error) { - return nil, errors.ErrNotSupported(errors.KIND_FUNCTION, "new version", Type) + if version != c.repo.arch.GetVersion() { + return nil, errors.ErrNotSupported(cpi.KIND_COMPONENTVERSION, version, fmt.Sprintf("component archive %s:%s", c.GetName(), c.repo.arch.GetVersion())) + } + if !utils.Optional(overrides...) { + return nil, errors.ErrAlreadyExists(cpi.KIND_COMPONENTVERSION, fmt.Sprintf("%s:%s", c.GetName(), c.repo.arch.GetVersion())) + } + return newComponentVersionAccess(c, version, false) +} + +//////////////////////////////////////////////////////////////////////////////// + +type ComponentVersionContainer struct { + impl support.ComponentVersionAccessImpl + + comp *ComponentAccessImpl + + descriptor *compdesc.ComponentDescriptor +} + +var _ support.ComponentVersionContainer = (*ComponentVersionContainer)(nil) + +func newComponentVersionAccess(comp *ComponentAccessImpl, version string, persistent bool) (cpi.ComponentVersionAccess, error) { + c, err := newComponentVersionContainer(comp) + if err != nil { + return nil, err + } + impl, err := support.NewComponentVersionAccessImpl(comp.GetName(), version, c, true, persistent, !compositionmodeattr.Get(comp.GetContext())) + if err != nil { + c.Close() + return nil, err + } + return cpi.NewComponentVersionAccess(impl), nil +} + +func newComponentVersionContainer(comp *ComponentAccessImpl) (*ComponentVersionContainer, error) { + return &ComponentVersionContainer{ + comp: comp, + descriptor: comp.repo.arch.GetDescriptor(), + }, nil +} + +func (c *ComponentVersionContainer) SetImplementation(impl support.ComponentVersionAccessImpl) { + c.impl = impl +} + +func (c *ComponentVersionContainer) GetParentViewManager() cpi.ComponentAccessViewManager { + return c.comp +} + +func (c *ComponentVersionContainer) Close() error { + return nil +} + +func (c *ComponentVersionContainer) GetContext() cpi.Context { + return c.comp.GetContext() +} + +func (c *ComponentVersionContainer) Repository() cpi.Repository { + return c.comp.repo.arch.nonref +} + +func (c *ComponentVersionContainer) IsReadOnly() bool { + return c.comp.repo.arch.IsReadOnly() +} + +func (c *ComponentVersionContainer) Update() error { + desc := c.comp.repo.arch.GetDescriptor() + *desc = *c.descriptor.Copy() + return c.comp.repo.arch.container.Update() +} + +func (c *ComponentVersionContainer) GetDescriptor() *compdesc.ComponentDescriptor { + return c.descriptor +} + +func (c *ComponentVersionContainer) GetBlobData(name string) (cpi.DataAccess, error) { + return c.comp.repo.arch.container.GetBlobData(name) +} + +func (c *ComponentVersionContainer) GetStorageContext(cv cpi.ComponentVersionAccess) cpi.StorageContext { + return ocmhdlr.New(c.Repository(), cv, &BlobSink{c.comp.repo.arch.container.base}, Type) +} + +func (c *ComponentVersionContainer) AddBlobFor(storagectx cpi.StorageContext, blob cpi.BlobAccess, refName string, global cpi.AccessSpec) (cpi.AccessSpec, error) { + if blob == nil { + return nil, errors.New("a resource has to be defined") + } + err := c.comp.repo.arch.container.base.AddBlob(blob) + if err != nil { + return nil, err + } + return localblob.New(common.DigestToFileName(blob.Digest()), refName, blob.MimeType(), global), nil +} + +func (c *ComponentVersionContainer) AccessMethod(a cpi.AccessSpec) (cpi.AccessMethod, error) { + if a.GetKind() == localblob.Type || a.GetKind() == localfsblob.Type { + accessSpec, err := c.GetContext().AccessSpecForSpec(a) + if err != nil { + return nil, err + } + return newLocalFilesystemBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c), nil + } + return nil, errors.ErrNotSupported(errors.KIND_ACCESSMETHOD, a.GetType(), "component archive") +} + +func (c *ComponentVersionContainer) GetInexpensiveContentVersionIdentity(a cpi.AccessSpec) string { + if a.GetKind() == localblob.Type || a.GetKind() == localfsblob.Type { + accessSpec, err := c.GetContext().AccessSpecForSpec(a) + if err != nil { + return "" + } + digest, _ := accessio.Digest(newLocalFilesystemBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c)) + return digest.String() + } + return "" } diff --git a/pkg/contexts/ocm/repositories/composition/repository.go b/pkg/contexts/ocm/repositories/composition/repository.go new file mode 100644 index 0000000000..743b4e4cbd --- /dev/null +++ b/pkg/contexts/ocm/repositories/composition/repository.go @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package composition + +import ( + "fmt" + "reflect" + "sync" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/virtual" + "github.com/open-component-model/ocm/pkg/errors" +) + +//////////////////////////////////////////////////////////////////////////////// + +func NewRepository(ctx cpi.ContextProvider) cpi.Repository { + return virtual.NewRepository(ctx.OCMContext(), NewAccess()) +} + +type Index = virtual.Index[common.NameVersion] + +type Access struct { + lock sync.Mutex + index *Index + blobs map[string]blobaccess.BlobAccess +} + +func NewAccess() *Access { + return &Access{ + index: virtual.NewIndex[common.NameVersion](), + blobs: map[string]blobaccess.BlobAccess{}, + } +} + +func (a *Access) IsReadOnly() bool { + return false +} + +func (a *Access) ComponentLister() cpi.ComponentLister { + a.lock.Lock() + defer a.lock.Unlock() + + return a.index +} + +func (a *Access) ExistsComponentVersion(name string, version string) (bool, error) { + a.lock.Lock() + defer a.lock.Unlock() + + e := a.index.Get(name, version) + return e != nil, nil +} + +func (a *Access) ListVersions(comp string) ([]string, error) { + a.lock.Lock() + defer a.lock.Unlock() + + return a.index.GetVersions(comp), nil +} + +func (a *Access) GetComponentVersion(comp, version string) (virtual.VersionAccess, error) { + var cd *compdesc.ComponentDescriptor + + a.lock.Lock() + defer a.lock.Unlock() + + i := a.index.Get(comp, version) + if i == nil { + cd = compdesc.New(comp, version) + err := a.index.Add(cd, common.VersionedElementKey(cd)) + if err != nil { + return nil, err + } + } else { + cd = i.CD() + } + return &VersionAccess{a, cd.GetName(), cd.GetVersion(), cd.Copy()}, nil +} + +func (a *Access) GetBlob(name string) (blobaccess.BlobAccess, error) { + a.lock.Lock() + defer a.lock.Unlock() + b := a.blobs[name] + if b == nil { + return nil, errors.ErrNotFound(blobaccess.KIND_BLOB, name) + } + return b.Dup() +} + +func (a *Access) AddBlob(blob blobaccess.BlobAccess) (string, error) { + digest := blob.Digest() + if digest == blobaccess.BLOB_UNKNOWN_DIGEST { + return "", fmt.Errorf("unknown digest") + } + a.lock.Lock() + defer a.lock.Unlock() + b := a.blobs[digest.Encoded()] + if b == nil { + b, err := blob.Dup() + if err != nil { + return "", err + } + a.blobs[digest.Encoded()] = b + } + return digest.Encoded(), nil +} + +func (a *Access) Close() error { + list := errors.ErrorList{} + for _, b := range a.blobs { + list.Add(b.Close()) + } + return list.Result() +} + +var _ virtual.Access = (*Access)(nil) + +type VersionAccess struct { + access *Access + comp string + vers string + desc *compdesc.ComponentDescriptor +} + +func (v *VersionAccess) GetDescriptor() *compdesc.ComponentDescriptor { + return v.desc +} + +func (v *VersionAccess) GetBlob(name string) (cpi.DataAccess, error) { + return v.access.GetBlob(name) +} + +func (v *VersionAccess) AddBlob(blob cpi.BlobAccess) (string, error) { + return v.access.AddBlob(blob) +} + +func (v *VersionAccess) Update() error { + v.access.lock.Lock() + defer v.access.lock.Unlock() + + if v.desc.GetName() != v.comp || v.desc.GetVersion() != v.vers { + return errors.ErrInvalid(cpi.KIND_COMPONENTVERSION, common.VersionedElementKey(v.desc).String()) + } + i := v.access.index.Get(v.comp, v.vers) + if !reflect.DeepEqual(v.desc, i.CD()) { + v.access.index.Set(v.desc, i.Info()) + } + return nil +} + +func (v *VersionAccess) Close() error { + return v.Update() +} + +func (v *VersionAccess) IsReadOnly() bool { + return false +} + +func (v *VersionAccess) GetInexpensiveContentVersionIdentity(a cpi.AccessSpec) string { + switch a.GetKind() { //nolint:gocritic // to be extended + case localblob.Type: + return a.(*localblob.AccessSpec).LocalReference + } + return "" +} + +var _ virtual.VersionAccess = (*VersionAccess)(nil) diff --git a/pkg/contexts/ocm/repositories/composition/repository_test.go b/pkg/contexts/ocm/repositories/composition/repository_test.go new file mode 100644 index 0000000000..4e10b73df2 --- /dev/null +++ b/pkg/contexts/ocm/repositories/composition/repository_test.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package composition_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + me "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/composition" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + ocmutils "github.com/open-component-model/ocm/pkg/contexts/ocm/utils" + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/mime" +) + +const COMPONENT = "acme.org/testcomp" +const VERSION = "1.0.0" + +var _ = Describe("repository", func() { + var ctx = ocm.DefaultContext() + + It("handles cvs", func() { + finalize := finalizer.Finalizer{} + defer Defer(finalize.Finalize) + + nested := finalize.Nested() + + repo := me.NewRepository(ctx) + finalize.Close(repo, "source repo") + + c := Must(repo.LookupComponent(COMPONENT)) + finalize.Close(c, "src comp") + + cv := Must(c.NewVersion(VERSION)) + nested.Close(cv, "src vers") + + cv.GetDescriptor().Provider.Name = "acme.org" + // wrap a non-closer access into a ref counting access to check cleanup + blob := spi.NewBlobAccessForBase(blobaccess.ForString(mime.MIME_TEXT, "testdata")) + nested.Close(blob, "blob") + MustBeSuccessful(cv.SetResourceBlob(ocm.NewResourceMeta("test", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blob, "", nil)) + MustBeSuccessful(c.AddVersion(cv)) + + MustBeSuccessful(nested.Finalize()) + + cv = Must(c.LookupVersion(VERSION)) + finalize.Close(cv, "query") + rs := Must(cv.GetResourcesByName("test")) + Expect(len(rs)).To(Equal(1)) + data := Must(ocmutils.GetResourceData(rs[0])) + Expect(string(data)).To(Equal("testdata")) + }) +}) diff --git a/pkg/contexts/ocm/repositories/composition/suite_test.go b/pkg/contexts/ocm/repositories/composition/suite_test.go new file mode 100644 index 0000000000..7ec0cf7774 --- /dev/null +++ b/pkg/contexts/ocm/repositories/composition/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package composition_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM Composition Repository Test Suite") +} diff --git a/pkg/contexts/ocm/repositories/composition/version.go b/pkg/contexts/ocm/repositories/composition/version.go new file mode 100644 index 0000000000..e5fb7ded02 --- /dev/null +++ b/pkg/contexts/ocm/repositories/composition/version.go @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package composition + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/refmgmt" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" +) + +func NewComponentVersion(ctx cpi.ContextProvider, name, vers string) cpi.ComponentVersionAccess { + repo := NewRepository(ctx) + defer repo.Close() + if !refmgmt.Lazy(repo) { + panic("wrong composition repo implementation") + } + c, err := repo.LookupComponent(name) + if err != nil { + panic("wrong composition repo implementation: " + err.Error()) + } + defer c.Close() + cv, err := c.NewVersion(vers) + if err != nil { + panic("wrong composition repo implementation: " + err.Error()) + } + return cv +} diff --git a/pkg/contexts/ocm/repositories/composition/version_test.go b/pkg/contexts/ocm/repositories/composition/version_test.go new file mode 100644 index 0000000000..6823e39b0d --- /dev/null +++ b/pkg/contexts/ocm/repositories/composition/version_test.go @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package composition_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/spi" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + me "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/composition" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + ocmutils "github.com/open-component-model/ocm/pkg/contexts/ocm/utils" + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/mime" +) + +var _ = Describe("version", func() { + var ctx = ocm.DefaultContext() + + It("handles anonymous version", func() { + finalize := finalizer.Finalizer{} + defer Defer(finalize.Finalize) + + nested := finalize.Nested() + + // compose new version + cv := me.NewComponentVersion(ctx, COMPONENT, VERSION) + cv.GetDescriptor().Provider.Name = "acme.org" + nested.Close(cv, "composed version") + + // wrap a non-closer access into a ref counting access to check cleanup + blob := spi.NewBlobAccessForBase(blobaccess.ForString(mime.MIME_TEXT, "testdata")) + nested.Close(blob, "blob") + MustBeSuccessful(cv.SetResourceBlob(ocm.NewResourceMeta("test", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blob, "", nil)) + + // add version to repository + repo1 := me.NewRepository(ctx) + finalize.Close(repo1, "target repo1") + c := Must(repo1.LookupComponent(COMPONENT)) + finalize.Close(c, "src comp") + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(nested.Finalize()) + + // check result + cv = Must(c.LookupVersion(VERSION)) + nested.Close(cv, "query") + rs := Must(cv.GetResourcesByName("test")) + Expect(len(rs)).To(Equal(1)) + data := Must(ocmutils.GetResourceData(rs[0])) + Expect(string(data)).To(Equal("testdata")) + + // add this version again + repo2 := me.NewRepository(ctx) + finalize.Close(repo2, "target repo2") + MustBeSuccessful(repo2.AddVersion(cv)) + MustBeSuccessful(nested.Finalize()) + + // check result + cv = Must(repo2.LookupComponentVersion(COMPONENT, VERSION)) + finalize.Close(cv, "query") + rs = Must(cv.GetResourcesByName("test")) + Expect(len(rs)).To(Equal(1)) + data = Must(ocmutils.GetResourceData(rs[0])) + Expect(string(data)).To(Equal("testdata")) + }) +}) diff --git a/pkg/contexts/ocm/repositories/ctf/repo_test.go b/pkg/contexts/ocm/repositories/ctf/repo_test.go index 23b5e7a19f..4f5a6e2a43 100644 --- a/pkg/contexts/ocm/repositories/ctf/repo_test.go +++ b/pkg/contexts/ocm/repositories/ctf/repo_test.go @@ -53,11 +53,15 @@ var _ = Describe("access method", func() { Expect(Must(cv.GetResource(compdesc.NewIdentity("text1"))).Meta().Digest).To(Equal(DS_TESTDATA)) // add resource with digest - MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("text2", resourcetypes.PLAIN_TEXT, metav1.LocalRelation).SetDigest(DS_TESTDATA), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) + meta := compdesc.NewResourceMeta("text2", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) + meta.SetDigest(DS_TESTDATA) + MustBeSuccessful(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) Expect(Must(cv.GetResource(compdesc.NewIdentity("text2"))).Meta().Digest).To(Equal(DS_TESTDATA)) // reject resource with wrong digest - Expect(cv.SetResourceBlob(compdesc.NewResourceMeta("text3", resourcetypes.PLAIN_TEXT, metav1.LocalRelation).SetDigest(TextResourceDigestSpec("fake")), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)).To(MatchError("unable to set resource: digest mismatch: " + D_TESTDATA + " != fake")) + meta = compdesc.NewResourceMeta("text3", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) + meta.SetDigest(TextResourceDigestSpec("fake")) + Expect(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)).To(MatchError("unable to set resource: digest mismatch: " + D_TESTDATA + " != fake")) MustBeSuccessful(c.AddVersion(cv)) MustBeSuccessful(final.Finalize()) diff --git a/pkg/contexts/ocm/repositories/genericocireg/accessmethod_localblob.go b/pkg/contexts/ocm/repositories/genericocireg/accessmethod_localblob.go index 63c4210ede..6a50fe6f2c 100644 --- a/pkg/contexts/ocm/repositories/genericocireg/accessmethod_localblob.go +++ b/pkg/contexts/ocm/repositories/genericocireg/accessmethod_localblob.go @@ -36,6 +36,10 @@ func newLocalBlobAccessMethod(a *localblob.AccessSpec, ns oci.NamespaceAccess, a } } +func (_ *localBlobAccessMethod) IsLocal() bool { + return true +} + func (m *localBlobAccessMethod) GetKind() string { return m.spec.GetKind() } diff --git a/pkg/contexts/ocm/repositories/genericocireg/component.go b/pkg/contexts/ocm/repositories/genericocireg/component.go index 7aa5d76b7a..aacadd39f1 100644 --- a/pkg/contexts/ocm/repositories/genericocireg/component.go +++ b/pkg/contexts/ocm/repositories/genericocireg/component.go @@ -139,23 +139,32 @@ func (c *componentAccessImpl) LookupVersion(version string) (cpi.ComponentVersio return newComponentVersionAccess(accessobj.ACC_WRITABLE, c, version, acc, true) } +func (c *componentAccessImpl) versionContainer(access cpi.ComponentVersionAccess) *ComponentVersionContainer { + mine, _ := support.GetComponentVersionContainer[*ComponentVersionContainer](access) + if mine == nil || mine.comp != c { + return nil + } + return mine +} + +func (c *componentAccessImpl) IsOwned(access cpi.ComponentVersionAccess) bool { + return c.versionContainer(access) != nil +} + func (c *componentAccessImpl) AddVersion(access cpi.ComponentVersionAccess) error { if access.GetName() != c.GetName() { return errors.ErrInvalid("component name", access.GetName()) } - cont, err := support.GetComponentVersionContainer(access) - if err != nil { - return fmt.Errorf("cannot add component version: component version access %s not created for target", access.GetName()+":"+access.GetVersion()) - } - mine, ok := cont.(*ComponentVersionContainer) - if !ok || mine.comp != c { + mine := c.versionContainer(access) + if mine == nil { return fmt.Errorf("cannot add component version: component version access %s not created for target", access.GetName()+":"+access.GetVersion()) } - ok = mine.impl.EnablePersistence() + ok := mine.impl.EnablePersistence() if !ok { return fmt.Errorf("version has been discarded") } - return mine.impl.Update(false) + // delayed update in close is not done for composition mode + return mine.impl.Update(!mine.impl.UseDirectAccess()) } func (c *componentAccessImpl) NewVersion(version string, overrides ...bool) (cpi.ComponentVersionAccess, error) { diff --git a/pkg/contexts/ocm/repositories/genericocireg/componentversion.go b/pkg/contexts/ocm/repositories/genericocireg/componentversion.go index d747b5b1f2..85d35066fd 100644 --- a/pkg/contexts/ocm/repositories/genericocireg/componentversion.go +++ b/pkg/contexts/ocm/repositories/genericocireg/componentversion.go @@ -22,6 +22,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/relativeociref" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compatattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" ocihdlr "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/oci" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" @@ -36,7 +37,7 @@ func newComponentVersionAccess(mode accessobj.AccessMode, comp *componentAccessI if err != nil { return nil, err } - impl, err := support.NewComponentVersionAccessImpl(comp.GetName(), version, c, true, persistent) + impl, err := support.NewComponentVersionAccessImpl(comp.GetName(), version, c, true, persistent, !compositionmodeattr.Get(comp.GetContext())) if err != nil { c.Close() return nil, err diff --git a/pkg/contexts/ocm/repositories/genericocireg/repo_test.go b/pkg/contexts/ocm/repositories/genericocireg/repo_test.go index b77268b8f0..33334261ad 100644 --- a/pkg/contexts/ocm/repositories/genericocireg/repo_test.go +++ b/pkg/contexts/ocm/repositories/genericocireg/repo_test.go @@ -131,6 +131,12 @@ var _ = Describe("component repository mapping", func() { vers := finalizer.ClosingWith(&finalize, Must(comp.NewVersion("v1"))) acc := Must(vers.AddBlob(blob, "", "", nil)) + MustBeSuccessful(vers.SetResource(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), acc)) + MustBeSuccessful(comp.AddVersion(vers)) + + rs := Must(vers.GetResourceByIndex(0)) + acc = Must(rs.Access()) + // check provided actual access to be local blob Expect(acc.GetKind()).To(Equal(localblob.Type)) l, ok := acc.(*localblob.AccessSpec) @@ -143,9 +149,9 @@ var _ = Describe("component repository mapping", func() { MustBeSuccessful(vers.SetResource(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), acc)) MustBeSuccessful(comp.AddVersion(vers)) - rs := Must(vers.GetResourceByIndex(0)) - + rs = Must(vers.GetResourceByIndex(0)) spec := Must(rs.Access()) + Expect(spec.GetType()).To(Equal(localociblob.Type)) m := Must(rs.AccessMethod()) @@ -173,6 +179,11 @@ var _ = Describe("component repository mapping", func() { comp := finalizer.ClosingWith(&finalize, Must(repo.LookupComponent(COMPONENT))) vers := finalizer.ClosingWith(&finalize, Must(comp.NewVersion("v1"))) acc := Must(vers.AddBlob(blob, "", "", nil)) + MustBeSuccessful(vers.SetResource(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), acc)) + MustBeSuccessful(comp.AddVersion(vers)) + + res := Must(vers.GetResourceByIndex(0)) + acc = Must(res.Access()) // check provided actual access to be local blob Expect(acc.GetKind()).To(Equal(localblob.Type)) @@ -187,10 +198,7 @@ var _ = Describe("component repository mapping", func() { Expect(ok).To(BeTrue()) Expect(o.Digest).To(Equal(blob.Digest())) Expect(o.Reference).To(Equal(TESTBASE + "/" + componentmapping.ComponentDescriptorNamespace + "/" + COMPONENT)) - MustBeSuccessful(vers.SetResource(compdesc.NewResourceMeta("blob", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), acc)) - MustBeSuccessful(comp.AddVersion(vers)) - res := Must(vers.GetResourceByIndex(0)) Expect(res.Meta().Digest).NotTo(BeNil()) Expect(res.Meta().Digest.Value).To(Equal(ocmtesthelper.D_TESTDATA)) }) @@ -227,10 +235,12 @@ var _ = Describe("component repository mapping", func() { fmt.Printf("physical digest: %s\n", blob.Digest()) acc := Must(vers.AddBlob(blob, "", "artifact1", nil)) - Expect(acc.GetKind()).To(Equal(localblob.Type)) - MustBeSuccessful(vers.SetResource(cpi.NewResourceMeta("image", resourcetypes.OCI_IMAGE, metav1.LocalRelation), acc)) + MustBeSuccessful(comp.AddVersion(vers)) + res := Must(vers.GetResourceByIndex(0)) + acc = Must(res.Access()) + Expect(acc.GetKind()).To(Equal(localblob.Type)) rd := res.Meta().Digest Expect(rd).NotTo(BeNil()) Expect(rd.Value).To(Equal(testhelper.DIGEST_MANIFEST)) @@ -242,16 +252,18 @@ var _ = Describe("component repository mapping", func() { Expect(acc.GetKind()).To(Equal(ociartifact.Type)) o := acc.(*ociartifact.AccessSpec) Expect(o.ImageReference).To(Equal(TESTBASE + "/artifact1@sha256:" + testhelper.DIGEST_MANIFEST)) - MustBeSuccessful(comp.AddVersion(vers)) acc = Must(vers.AddBlob(blob, "", "artifact2:v1", nil)) + MustBeSuccessful(vers.SetResource(cpi.NewResourceMeta("image2", resourcetypes.OCI_IMAGE, metav1.LocalRelation), acc, cpi.ModifyResource())) + MustBeSuccessful(comp.AddVersion(vers)) + res = Must(vers.GetResourceByIndex(1)) + acc = Must(res.Access()) acc = acc.GlobalAccessSpec(ctx) Expect(acc).NotTo(BeNil()) Expect(acc.GetKind()).To(Equal(ociartifact.Type)) o = acc.(*ociartifact.AccessSpec) Expect(o.ImageReference).To(Equal(TESTBASE + "/artifact2:v1")) - MustBeSuccessful(comp.AddVersion(vers)) MustBeSuccessful(nested.Finalize()) @@ -283,10 +295,11 @@ var _ = Describe("component repository mapping", func() { MustBeSuccessful(nested.Finalize()) - // modify rsource in component + // modify resource in component vers = finalizer.ClosingWith(nested, Must(repo.LookupComponentVersion(COMPONENT, "v1"))) blob = blobaccess.ForString(mime.MIME_TEXT, "otherdata") MustBeSuccessful(vers.SetResourceBlob(m1, blob, "", nil)) + MustBeSuccessful(vers.Update()) MustBeSuccessful(nested.Finalize()) // check content diff --git a/pkg/contexts/ocm/repositories/virtual/accessmethod_localblob.go b/pkg/contexts/ocm/repositories/virtual/accessmethod_localblob.go index 8f10b676fb..ff1e8230f7 100644 --- a/pkg/contexts/ocm/repositories/virtual/accessmethod_localblob.go +++ b/pkg/contexts/ocm/repositories/virtual/accessmethod_localblob.go @@ -29,6 +29,10 @@ func newLocalBlobAccessMethod(a *localblob.AccessSpec, acc VersionAccess) *local } } +func (_ *localBlobAccessMethod) IsLocal() bool { + return true +} + func (m *localBlobAccessMethod) GetKind() string { return m.spec.GetKind() } @@ -72,8 +76,12 @@ func (m *localBlobAccessMethod) Reader() (io.ReadCloser, error) { return blob.Reader() } -func (m *localBlobAccessMethod) Get() ([]byte, error) { - return blobaccess.BlobData(m.getBlob()) +func (m *localBlobAccessMethod) Get() (data []byte, ferr error) { + b, err := m.getBlob() + if ferr != nil { + return nil, err + } + return blobaccess.BlobData(b) } func (m *localBlobAccessMethod) MimeType() string { diff --git a/pkg/contexts/ocm/repositories/virtual/component.go b/pkg/contexts/ocm/repositories/virtual/component.go index f072212760..3cd11b627e 100644 --- a/pkg/contexts/ocm/repositories/virtual/component.go +++ b/pkg/contexts/ocm/repositories/virtual/component.go @@ -63,20 +63,30 @@ func (c *componentAccessImpl) LookupVersion(version string) (cpi.ComponentVersio return newComponentVersionAccess(c, version, true) } +func (c *componentAccessImpl) versionContainer(access cpi.ComponentVersionAccess) *ComponentVersionContainer { + mine, _ := support.GetComponentVersionContainer[*ComponentVersionContainer](access) + if mine == nil || mine.comp != c { + return nil + } + return mine +} + +func (c *componentAccessImpl) IsOwned(access cpi.ComponentVersionAccess) bool { + return c.versionContainer(access) != nil +} + func (c *componentAccessImpl) AddVersion(access cpi.ComponentVersionAccess) error { if access.GetName() != c.GetName() { return errors.ErrInvalid("component name", access.GetName()) } - cont, err := support.GetComponentVersionContainer(access) - if err != nil { - return fmt.Errorf("cannot add component version: component version access %s not created for target", access.GetName()+":"+access.GetVersion()) - } - mine, ok := cont.(*ComponentVersionContainer) - if !ok || mine.comp != c { + mine := c.versionContainer(access) + if mine == nil { return fmt.Errorf("cannot add component version: component version access %s not created for target", access.GetName()+":"+access.GetVersion()) } mine.impl.EnablePersistence() - return mine.impl.Update(false) + + // delayed update in close is not done for composition mode + return mine.impl.Update(!mine.impl.UseDirectAccess()) } func (c *componentAccessImpl) NewVersion(version string, overrides ...bool) (cpi.ComponentVersionAccess, error) { diff --git a/pkg/contexts/ocm/repositories/virtual/componentversion.go b/pkg/contexts/ocm/repositories/virtual/componentversion.go index 5983a92fc1..5d8312f844 100644 --- a/pkg/contexts/ocm/repositories/virtual/componentversion.go +++ b/pkg/contexts/ocm/repositories/virtual/componentversion.go @@ -8,6 +8,7 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessio" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localfsblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" ocmhdlr "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" @@ -25,7 +26,7 @@ func newComponentVersionAccess(comp *componentAccessImpl, version string, persis if err != nil { return nil, err } - impl, err := support.NewComponentVersionAccessImpl(comp.GetName(), version, c, true, persistent) + impl, err := support.NewComponentVersionAccessImpl(comp.GetName(), version, c, true, persistent, !compositionmodeattr.Get(comp.GetContext())) if err != nil { c.Close() return nil, err diff --git a/pkg/contexts/ocm/repositories/virtual/repo_test.go b/pkg/contexts/ocm/repositories/virtual/repo_test.go index b2cfafc760..648986a88b 100644 --- a/pkg/contexts/ocm/repositories/virtual/repo_test.go +++ b/pkg/contexts/ocm/repositories/virtual/repo_test.go @@ -16,7 +16,9 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/compose" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/virtual" @@ -80,10 +82,11 @@ var _ = Describe("virtual repo", func() { repo = virtual.NewRepository(env.OCMContext(), access) }) - It("handles put", func() { + DescribeTable("handles put", func(mode bool, typ string) { var finalize finalizer.Finalizer defer Defer(finalize.Finalize) + compositionmodeattr.Set(env.OCMContext(), mode) comp := Must(repo.LookupComponent("acme.org/component/new")) finalize.Close(comp, "component") Expect(comp.ListVersions()).To(ConsistOf([]string{})) @@ -95,6 +98,11 @@ var _ = Describe("virtual repo", func() { r := Must(vers.GetResourceByIndex(0)) a := Must(r.Access()) + Expect(a.GetKind()).To(Equal(typ)) + + comp.AddVersion(vers) + r = Must(vers.GetResourceByIndex(0)) // re-read resource from component descriptor. + a = Must(r.Access()) Expect(a.GetKind()).To(Equal(localblob.Type)) dig := "fe81d80611e39a10f1d7d12f98ce0bc6fe745d08fef007d8eebddc0a21d17827" @@ -118,6 +126,9 @@ var _ = Describe("virtual repo", func() { a = Must(r.Access()) Expect(a.GetInexpensiveContentVersionIdentity(vers)).To(Equal("sha256:" + dig)) - }) + }, + Entry("with direct mode", false, localblob.Type), + Entry("with composition mode", true, compose.Type), + ) }) }) diff --git a/pkg/contexts/ocm/resourcetypes/data/options.go b/pkg/contexts/ocm/resourcetypes/data/options.go new file mode 100644 index 0000000000..53faf8cda7 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/data/options.go @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package data + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type compressionMode string + +const ( + COMPRESSION = compressionMode("compression") + DECOMPRESSION = compressionMode("decompression") + NONE = compressionMode("") +) + +type Options struct { + rpi.Options + MimeType string + Compression compressionMode +} + +var _ rpi.GeneralOptionsProvider = (*Options)(nil) + +func (o *Options) Apply(opts *Options) { + o.Options.ApplyTo(&opts.Options) + if o.MimeType != "" { + opts.MimeType = o.MimeType + } +} + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return rpi.WrapHint[Options](h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WrapGlobalAccess[Options](a) +} + +//////////////////////////////////////////////////////////////////////////////// +// Local Options + +type mimetype struct { + mime string +} + +func (o mimetype) ApplyTo(opts *Options) { + opts.MimeType = o.mime +} + +func WithMimeType(mime string) Option { + return mimetype{mime} +} + +//////////////////////////////////////////////////////////////////////////////// + +type compression struct { + mode compressionMode +} + +func (o compression) ApplyTo(opts *Options) { + opts.Compression = o.mode +} + +func WithCompression() Option { + return compression{COMPRESSION} +} + +func WithDecompression() Option { + return compression{DECOMPRESSION} +} diff --git a/pkg/contexts/ocm/resourcetypes/data/resource.go b/pkg/contexts/ocm/resourcetypes/data/resource.go new file mode 100644 index 0000000000..014bf9d30e --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/data/resource.go @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package data + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, blob []byte, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + + media := eff.MimeType + if media == "" { + media = mime.MIME_OCTET + } + + var blobprov blobaccess.BlobAccessProvider + switch eff.Compression { + case NONE: + blobprov = blobaccess.ProviderForData(media, blob) + case COMPRESSION: + blob := blobaccess.ForData(media, blob) + defer blob.Close() + blob, _ = blobaccess.WithCompression(blob) + blobprov = blobaccess.ProviderForBlobAccess(blob) + case DECOMPRESSION: + blob := blobaccess.ForData(media, blob) + defer blob.Close() + blob, _ = blobaccess.WithDecompression(blob) + blobprov = blobaccess.ProviderForBlobAccess(blob) + } + + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobprov, eff.Hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, media string, meta *ocm.ResourceMeta, blob []byte, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, blob, opts...) +} + +func SourceAccess(ctx ocm.Context, media string, meta *ocm.SourceMeta, blob []byte, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, blob, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/options.go b/pkg/contexts/ocm/resourcetypes/dirtree/options.go new file mode 100644 index 0000000000..a14f4319b0 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/options.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + base "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dirtree" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + rpi.Options + Blob base.Options +} + +var _ rpi.GeneralOptionsProvider = (*Options)(nil) + +func (o *Options) Apply(opts *Options) { + o.Options.ApplyTo(&opts.Options) + o.Blob.ApplyTo(&opts.Blob) +} + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return rpi.WrapHint[Options](h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WrapGlobalAccess[Options](a) +} + +//////////////////////////////////////////////////////////////////////////////// +// DirTree BlobAccess Options + +func mapBaseOption(opts *Options) *base.Options { + return &opts.Blob +} + +func wrapBase(o base.Option) Option { + return optionutils.OptionWrapperFunc[*base.Options, *Options](o, mapBaseOption) +} + +func WithFileSystem(fs vfs.FileSystem) Option { + return wrapBase(base.WithFileSystem(fs)) +} + +func WithExcludeFiles(files []string) Option { + return wrapBase(base.WithExcludeFiles(files)) +} + +func WithIncludeFiles(files []string) Option { + return wrapBase(base.WithIncludeFiles(files)) +} + +func WithFollowSymlinks(b ...bool) Option { + return wrapBase(base.WithFollowSymlinks(b...)) +} + +func WithPreserveDir(b ...bool) Option { + return wrapBase(base.WithPreserveDir(b...)) +} + +func WithCompressWithGzip(b ...bool) Option { + return wrapBase(base.WithCompressWithGzip(b...)) +} diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/resource.go b/pkg/contexts/ocm/resourcetypes/dirtree/resource.go new file mode 100644 index 0000000000..98cc7a8068 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/resource.go @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dirtree" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +const TYPE = resourcetypes.DIRECTORY_TREE + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, path string, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + if meta.GetType() == "" { + meta.SetType(TYPE) + } + blobprov := dirtree.BlobAccessProviderForDirTree(path, &eff.Blob) + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobprov, eff.Hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, meta *cpi.ResourceMeta, path string, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, path, opts...) +} + +func SourceAccess(ctx ocm.Context, meta *cpi.SourceMeta, path string, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, path, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/resource_test.go b/pkg/contexts/ocm/resourcetypes/dirtree/resource_test.go new file mode 100644 index 0000000000..dc4321220e --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/resource_test.go @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + me "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/dirtree" + "github.com/open-component-model/ocm/pkg/env" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +var _ = Describe("dir tree resource access", func() { + It("creates resource", func() { + env := env.NewEnvironment(env.TestData()) + + global := ociartifact.New("ghcr.io/mandelsoft/demo:v1.0.0") + + acc := me.ResourceAccess(env.OCMContext(), compdesc.NewResourceMeta("test", "", compdesc.LocalRelation), "testdata", + me.WithExcludeFiles([]string{"dir/a"}), + me.WithFileSystem(env.FileSystem()), + me.WithHint("demo"), + me.WithGlobalAccess(global), + ) + + Expect(acc.ReferenceHint()).To(Equal("demo")) + Expect(acc.GlobalAccess()).To(Equal(global)) + Expect(acc.Meta().Type).To(Equal(resourcetypes.DIRECTORY_TREE)) + + blob := Must(acc.BlobAccess()) + defer Defer(blob.Close, "blob") + Expect(blob.MimeType()).To(Equal(mime.MIME_TAR)) + + r := Must(blob.Reader()) + defer Defer(r.Close, "reader") + files := Must(tarutils.ListArchiveContentFromReader(r)) + Expect(files).To(ConsistOf([]string{ + "dir", + "dir/b", + "dir/c", + })) + }) + + It("adds resource", func() { + env := env.NewEnvironment(env.TestData()) + + global := ociartifact.New("ghcr.io/mandelsoft/demo:v1.0.0") + + acc := me.ResourceAccess(env.OCMContext(), compdesc.NewResourceMeta("test", "", compdesc.LocalRelation), "testdata", + me.WithExcludeFiles([]string{"dir/a"}), + me.WithFileSystem(env.FileSystem()), + me.WithHint("demo"), + me.WithGlobalAccess(global), + ) + + arch := Must(ctf.Create(env, accessobj.ACC_CREATE, "ctf", 0700, env, accessobj.FormatDirectory)) + c := Must(arch.LookupComponent("arcme.org/test")) + v := Must(c.NewVersion("v1.0.0")) + + MustBeSuccessful(v.SetResourceByAccess(acc)) + MustBeSuccessful(c.AddVersion(v)) + }) +}) diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/suite_test.go b/pkg/contexts/ocm/resourcetypes/dirtree/suite_test.go new file mode 100644 index 0000000000..e45ff816a7 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DirTree Resource") +} diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/a b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/a new file mode 100644 index 0000000000..20329687bb --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/a @@ -0,0 +1 @@ +content a \ No newline at end of file diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/b b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/b new file mode 100644 index 0000000000..50e9cdb03f --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/b @@ -0,0 +1 @@ +content b \ No newline at end of file diff --git a/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/c b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/c new file mode 100644 index 0000000000..20329687bb --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dirtree/testdata/dir/c @@ -0,0 +1 @@ +content a \ No newline at end of file diff --git a/pkg/contexts/ocm/resourcetypes/dockerdaemon/options.go b/pkg/contexts/ocm/resourcetypes/dockerdaemon/options.go new file mode 100644 index 0000000000..06e3b95dd9 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dockerdaemon/options.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dockerdaemon + +import ( + "github.com/open-component-model/ocm/pkg/common" + base "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockerdaemon" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + rpi.Options + Blob base.Options +} + +var _ rpi.GeneralOptionsProvider = (*Options)(nil) + +func (o *Options) Apply(opts *Options) { + o.Options.ApplyTo(&opts.Options) + o.Blob.ApplyTo(&opts.Blob) +} + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return rpi.WrapHint[Options](h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WrapGlobalAccess[Options](a) +} + +//////////////////////////////////////////////////////////////////////////////// +// Docker BlobAccess Options + +func mapBaseOption(opts *Options) *base.Options { + return &opts.Blob +} + +func wrapBase(o base.Option) Option { + return optionutils.OptionWrapperFunc[*base.Options, *Options](o, mapBaseOption) +} + +func WithName(n string) Option { + return wrapBase(base.WithName(n)) +} + +func WithVersion(v string) Option { + return wrapBase(base.WithVersion(v)) +} + +func WithVersionOverride(v string, flag ...bool) Option { + return wrapBase(base.WithVersionOverride(v, flag...)) +} + +func WithOrigin(o common.NameVersion) Option { + return wrapBase(base.WithOrigin(o)) +} diff --git a/pkg/contexts/ocm/resourcetypes/dockerdaemon/resource.go b/pkg/contexts/ocm/resourcetypes/dockerdaemon/resource.go new file mode 100644 index 0000000000..ec7a97da53 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/dockerdaemon/resource.go @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dockerdaemon + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockerdaemon" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +const TYPE = resourcetypes.OCI_IMAGE + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, name string, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + if meta.GetType() == "" { + meta.SetType(TYPE) + } + locator, version, err := dockerdaemon.ImageInfoFor(name, &eff.Blob) + if err == nil { + version = eff.Blob.Version + } + hint := ociartifact.Hint(optionutils.AsValue(eff.Blob.Origin), locator, eff.Hint, version) + blobprov := dockerdaemon.BlobAccessProviderForImageFromDockerDaemon(name, &eff.Blob) + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobprov, hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, meta *cpi.ResourceMeta, path string, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, path, opts...) +} + +func SourceAccess(ctx ocm.Context, meta *cpi.SourceMeta, path string, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, path, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/file/options.go b/pkg/contexts/ocm/resourcetypes/file/options.go new file mode 100644 index 0000000000..7a44099017 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/file/options.go @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type compressionMode string + +const ( + COMPRESSION = compressionMode("compression") + DECOMPRESSION = compressionMode("decompression") + NONE = compressionMode("") +) + +type Options struct { + rpi.Options + FileSystem vfs.FileSystem + Compression compressionMode +} + +var _ rpi.GeneralOptionsProvider = (*Options)(nil) + +func (o *Options) Apply(opts *Options) { + o.Options.ApplyTo(&opts.Options) + if o.FileSystem != nil { + opts.FileSystem = o.FileSystem + } +} + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return rpi.WrapHint[Options](h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WrapGlobalAccess[Options](a) +} + +//////////////////////////////////////////////////////////////////////////////// +// Local Options + +type filesystem struct { + fs vfs.FileSystem +} + +func (o filesystem) ApplyTo(opts *Options) { + opts.FileSystem = o.fs +} + +func WithFileSystem(fs vfs.FileSystem) Option { + return filesystem{fs} +} + +//////////////////////////////////////////////////////////////////////////////// + +type compression struct { + mode compressionMode +} + +func (o compression) ApplyTo(opts *Options) { + opts.Compression = o.mode +} + +func WithCompression() Option { + return compression{COMPRESSION} +} + +func WithDecompression() Option { + return compression{DECOMPRESSION} +} diff --git a/pkg/contexts/ocm/resourcetypes/file/resource.go b/pkg/contexts/ocm/resourcetypes/file/resource.go new file mode 100644 index 0000000000..10c9e6aa57 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/file/resource.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package dirtree + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +const TYPE = "blob" + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, media string, meta P, path string, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + + if meta.GetType() == "" { + meta.SetType(TYPE) + } + if media == "" { + media = mime.MIME_OCTET + } + + var blobprov blobaccess.BlobAccessProvider + switch eff.Compression { + case NONE: + blobprov = blobaccess.ProviderForFile(media, path, eff.FileSystem) + case COMPRESSION: + blob := blobaccess.ForFile(media, path, eff.FileSystem) + defer blob.Close() + blob, _ = blobaccess.WithCompression(blob) + blobprov = blobaccess.ProviderForBlobAccess(blob) + case DECOMPRESSION: + blob := blobaccess.ForFile(media, path, eff.FileSystem) + defer blob.Close() + blob, _ = blobaccess.WithDecompression(blob) + blobprov = blobaccess.ProviderForBlobAccess(blob) + } + + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobprov, eff.Hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, media string, meta *ocm.ResourceMeta, path string, opts ...Option) cpi.ResourceAccess { + return Access(ctx, media, meta, path, opts...) +} + +func SourceAccess(ctx ocm.Context, media string, meta *ocm.SourceMeta, path string, opts ...Option) cpi.SourceAccess { + return Access(ctx, media, meta, path, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/generic/options.go b/pkg/contexts/ocm/resourcetypes/generic/options.go new file mode 100644 index 0000000000..00e022a2e3 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/generic/options.go @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package generic + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" +) + +type ( + Options = rpi.Options + Option = rpi.Option +) + +func WithHint(h string) Option { + return rpi.WithHint(h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WithGlobalAccess(a) +} diff --git a/pkg/contexts/ocm/resourcetypes/generic/resource.go b/pkg/contexts/ocm/resourcetypes/generic/resource.go new file mode 100644 index 0000000000..676c5ae59a --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/generic/resource.go @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package generic + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, blob blobaccess.BlobAccess, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobaccess.ProviderForBlobAccess(blob), eff.Hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, media string, meta *ocm.ResourceMeta, blob blobaccess.BlobAccess, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, blob, opts...) +} + +func SourceAccess(ctx ocm.Context, media string, meta *ocm.SourceMeta, blob blobaccess.BlobAccess, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, blob, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/helm/options.go b/pkg/contexts/ocm/resourcetypes/helm/options.go new file mode 100644 index 0000000000..ef1591f1b2 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/helm/options.go @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package helm + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common" + base "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/helm" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + rpi.Options + Blob base.Options +} + +var _ rpi.GeneralOptionsProvider = (*Options)(nil) + +func (o *Options) Apply(opts *Options) { + o.Options.ApplyTo(&opts.Options) + o.Blob.ApplyTo(&opts.Blob) +} + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return rpi.WrapHint[Options](h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return rpi.WrapGlobalAccess[Options](a) +} + +//////////////////////////////////////////////////////////////////////////////// +// DirTree BlobAccess Options + +func mapBaseOption(opts *Options) *base.Options { + return &opts.Blob +} + +func wrapBase(o base.Option) Option { + return optionutils.OptionWrapperFunc[*base.Options, *Options](o, mapBaseOption) +} + +func WithFileSystem(fs vfs.FileSystem) Option { + return wrapBase(base.WithFileSystem(fs)) +} + +func WithContext(ctx oci.ContextProvider) Option { + return wrapBase(base.WithContext(ctx)) +} + +func WithIVersion(v string) Option { + return wrapBase(base.WithVersion(v)) +} + +func WithIVersionOverride(v string, flag ...bool) Option { + return wrapBase(base.WithVersionOverride(v, flag...)) +} + +func WithCACert(v string) Option { + return wrapBase(base.WithCACert(v)) +} + +func WithCACertFile(v string) Option { + return wrapBase(base.WithCACertFile(v)) +} + +func WithHelmRepository(v string) Option { + return wrapBase(base.WithHelmRepository(v)) +} + +func WithPrinter(v common.Printer) Option { + return wrapBase(base.WithPrinter(v)) +} diff --git a/pkg/contexts/ocm/resourcetypes/helm/resource.go b/pkg/contexts/ocm/resourcetypes/helm/resource.go new file mode 100644 index 0000000000..e048578df0 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/helm/resource.go @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package helm + +import ( + "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/helm" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +const TYPE = resourcetypes.HELM_CHART + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, path string, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(append(opts, WithContext(ctx))...) + if meta.GetType() == "" { + meta.SetType(TYPE) + } + hint := eff.Hint + blobprov := helm.BlobAccessProviderForHelmChart(path, &eff.Blob) + accprov := cpi.NewAccessProviderForBlobAccessProvider(ctx, blobprov, hint, eff.Global) + // strange type cast is required by Go compiler, meta has the correct type. + return cpi.NewArtifactAccessForProvider(generics.As[*M](meta), accprov) +} + +func ResourceAccess(ctx ocm.Context, meta *cpi.ResourceMeta, path string, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, path, opts...) +} + +func SourceAccess(ctx ocm.Context, meta *cpi.SourceMeta, path string, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, path, opts...) +} diff --git a/pkg/contexts/ocm/resourcetypes/rpi/options.go b/pkg/contexts/ocm/resourcetypes/rpi/options.go new file mode 100644 index 0000000000..d518d8f85f --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/rpi/options.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package rpi + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/optionutils" +) + +type ( + Option = optionutils.Option[*Options] + GeneralOptionsProvider = optionutils.NestedOptionsProvider[*Options] +) + +type Options struct { + Global cpi.AccessSpec + Hint string +} + +var ( + _ optionutils.NestedOptionsProvider[*Options] = (*Options)(nil) + _ optionutils.Option[*Options] = (*Options)(nil) +) + +func (w *Options) NestedOptions() *Options { + return w +} + +func (o *Options) ApplyTo(opts *Options) { + if o.Global != nil { + opts.Global = o.Global + } + if o.Hint != "" { + opts.Hint = o.Hint + } +} + +type hint string + +func (o hint) ApplyTo(opts *Options) { + opts.Hint = string(o) +} + +func WithHint(h string) Option { + return hint(h) +} + +func WrapHint[O any, P optionutils.OptionTargetProvider[*Options, O]](h string) optionutils.Option[P] { + return optionutils.OptionWrapper[*Options, O, P](WithHint(h)) +} + +//////////////////////////////////////////////////////////////////////////////// + +type global struct { + cpi.AccessSpec +} + +func (o global) ApplyTo(opts *Options) { + opts.Global = o.AccessSpec +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return global{a} +} + +func WrapGlobalAccess[O any, P optionutils.OptionTargetProvider[*Options, O]](a cpi.AccessSpec) optionutils.Option[P] { + return optionutils.OptionWrapper[*Options, O, P](WithGlobalAccess(a)) +} diff --git a/pkg/contexts/ocm/resourcetypes/text/options.go b/pkg/contexts/ocm/resourcetypes/text/options.go new file mode 100644 index 0000000000..363609e860 --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/text/options.go @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package text + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/data" +) + +type ( + Option = data.Option + Options = data.Options +) + +const ( + COMPRESSION = data.COMPRESSION + DECOMPRESSION = data.DECOMPRESSION + NONE = data.NONE +) + +//////////////////////////////////////////////////////////////////////////////// +// General Options + +func WithHint(h string) Option { + return data.WithHint(h) +} + +func WithGlobalAccess(a cpi.AccessSpec) Option { + return data.WithGlobalAccess(a) +} + +//////////////////////////////////////////////////////////////////////////////// +// Local Options + +func WithimeType(mime string) Option { + return data.WithMimeType(mime) +} + +func WithCompression() Option { + return data.WithCompression() +} + +func WithDecompression() Option { + return data.WithDecompression() +} diff --git a/pkg/contexts/ocm/resourcetypes/text/resource.go b/pkg/contexts/ocm/resourcetypes/text/resource.go new file mode 100644 index 0000000000..37a197faff --- /dev/null +++ b/pkg/contexts/ocm/resourcetypes/text/resource.go @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package text + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/data" +) + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, blob string, opts ...Option) cpi.ArtifactAccess[M] { + return data.Access(ctx, meta, []byte(blob), opts...) +} + +func ResourceAccess(ctx ocm.Context, media string, meta *ocm.ResourceMeta, blob string, opts ...Option) cpi.ResourceAccess { + return Access(ctx, meta, blob, opts...) +} + +func SourceAccess(ctx ocm.Context, media string, meta *ocm.SourceMeta, blob string, opts ...Option) cpi.SourceAccess { + return Access(ctx, meta, blob, opts...) +} diff --git a/pkg/contexts/ocm/signing/handle.go b/pkg/contexts/ocm/signing/handle.go index 0137f3760b..7a4f8c7462 100644 --- a/pkg/contexts/ocm/signing/handle.go +++ b/pkg/contexts/ocm/signing/handle.go @@ -319,6 +319,10 @@ func _apply(state WalkingState, nv common.NameVersion, cv ocm.ComponentVersionAc orig.Signatures = cd.Signatures ctx.Signed = true } + err := cv.Update() + if err != nil { + return nil, err + } } return ctx, nil } diff --git a/pkg/contexts/ocm/signing/signing_test.go b/pkg/contexts/ocm/signing/signing_test.go index 99b0dc4076..c1ae867fd9 100644 --- a/pkg/contexts/ocm/signing/signing_test.go +++ b/pkg/contexts/ocm/signing/signing_test.go @@ -24,6 +24,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/none" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" @@ -578,11 +579,12 @@ applying to version "github.com/mandelsoft/ref:v1"[github.com/mandelsoft/ref:v1] }) } - DescribeTable("hashes unsigned", func(c EntryCheck, mopts ...ocm.ModificationOption) { + DescribeTable("hashes unsigned", func(mode bool, c EntryCheck, mopts ...ocm.ModificationOption) { var finalizer Finalizer defer Defer(finalizer.Finalize) { + compositionmodeattr.Set(env.OCMContext(), mode) setup(mopts...) arch := finalizer.Nested() src, err := ctf.Open(env.OCMContext(), accessobj.ACC_WRITABLE, ARCH, 0, env) @@ -649,11 +651,17 @@ applying to version "github.com/mandelsoft/top:v1"[github.com/mandelsoft/top:v1] Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("github.com/mandelsoft/top:v1: failed applying to component reference refb[github.com/mandelsoft/ref:v1]: github.com/mandelsoft/top:v1->github.com/mandelsoft/ref:v1: failed applying to component reference ref[github.com/mandelsoft/test:v1]: github.com/mandelsoft/top:v1->github.com/mandelsoft/ref:v1->github.com/mandelsoft/test:v1: calculated resource digest ([{HashAlgorithm:SHA-256 NormalisationAlgorithm:genericBlobDigest/v1 Value:" + D_DATAA + "}]) mismatches existing digest (SHA-256:" + wrongDigest + "[genericBlobDigest/v1]) for data_a:v1 (Local blob sha256:" + D_DATAA + "[])")) }, - Entry(DIGESTMODE_TOP, &EntryTop{}), - Entry(DIGESTMODE_LOCAL, &EntryLocal{}), + Entry(DIGESTMODE_TOP, false, &EntryTop{}), + Entry(DIGESTMODE_LOCAL, false, &EntryLocal{}), - Entry("legacy "+DIGESTMODE_TOP, &EntryTop{}, ocm.SkipDigest()), - Entry("legacy "+DIGESTMODE_LOCAL, &EntryLocal{}, ocm.SkipDigest()), + Entry("legacy "+DIGESTMODE_TOP, false, &EntryTop{}, ocm.SkipDigest()), + Entry("legacy "+DIGESTMODE_LOCAL, false, &EntryLocal{}, ocm.SkipDigest()), + + Entry(DIGESTMODE_TOP+" with composition mode", true, &EntryTop{}), + Entry(DIGESTMODE_LOCAL+" with composition mode", true, &EntryLocal{}), + + Entry("legacy "+DIGESTMODE_TOP+" with composition mode", true, &EntryTop{}, ocm.SkipDigest()), + Entry("legacy "+DIGESTMODE_LOCAL+" with composition mode", true, &EntryLocal{}, ocm.SkipDigest()), ) DescribeTable("signs unsigned", func(c EntryCheck, mopts ...ocm.ModificationOption) { @@ -1209,6 +1217,7 @@ func (*EntryLocal) Check1CheckA(cva ocm.ComponentVersionAccess, d *metav1.Digest func (*EntryLocal) Check1Corrupt(cva ocm.ComponentVersionAccess, f *Finalizer, _ ocm.ComponentVersionAccess) { cva.GetDescriptor().Resources[0].Digest.Value = wrongDigest + MustBeSuccessful(cva.Update()) Check(f.Finalize) } diff --git a/pkg/contexts/ocm/signing/transport_test.go b/pkg/contexts/ocm/signing/transport_test.go index ade01d1d93..cc4bb2e9b6 100644 --- a/pkg/contexts/ocm/signing/transport_test.go +++ b/pkg/contexts/ocm/signing/transport_test.go @@ -207,6 +207,8 @@ var _ = Describe("transport and signing", func() { // change volatile data in origin modify(cv, tcv, merged) + MustBeSuccessful(tcv.Update()) + MustBeSuccessful(targetfinal.Finalize()) // transfer changed volatile data diff --git a/pkg/contexts/ocm/testhelper/resources.go b/pkg/contexts/ocm/testhelper/resources.go index a885370af5..03d5b07319 100644 --- a/pkg/contexts/ocm/testhelper/resources.go +++ b/pkg/contexts/ocm/testhelper/resources.go @@ -34,7 +34,7 @@ var DS_TESTDATA = TextResourceDigestSpec(D_TESTDATA) func TestDataResource(env *builder.Builder, funcs ...func()) { env.Resource("testdata", "", "PlainText", metav1.LocalRelation, func() { - env.BlobStringData(mime.MIME_TEXT, "testdata") + env.BlobStringData(mime.MIME_TEXT, S_TESTDATA) env.Configure(funcs...) }) } diff --git a/pkg/contexts/ocm/transfer/needs.go b/pkg/contexts/ocm/transfer/needs.go index 79a2ced2e5..a6fe3d30e2 100644 --- a/pkg/contexts/ocm/transfer/needs.go +++ b/pkg/contexts/ocm/transfer/needs.go @@ -9,7 +9,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" - "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/access" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" ) func needsResourceTransport(cv ocm.ComponentVersionAccess, s, t *compdesc.ComponentDescriptor, handler TransferHandler) bool { @@ -19,7 +19,7 @@ func needsResourceTransport(cv ocm.ComponentVersionAccess, s, t *compdesc.Compon return true } - sa := access.NewResourceAccess(cv, &r) + sa := cpi.NewResourceAccess(cv, r.Access, r.ResourceMeta) sacc, err := sa.Access() if err != nil { return true @@ -36,7 +36,7 @@ func needsResourceTransport(cv ocm.ComponentVersionAccess, s, t *compdesc.Compon return true } - sa := access.NewSourceAccess(cv, &r) + sa := cpi.NewSourceAccess(cv, r.Access, r.SourceMeta) sacc, err := sa.Access() if err != nil { diff --git a/pkg/contexts/ocm/transfer/transferhandler/standard/handler_test.go b/pkg/contexts/ocm/transfer/transferhandler/standard/handler_test.go index 8903ae9756..f8813682af 100644 --- a/pkg/contexts/ocm/transfer/transferhandler/standard/handler_test.go +++ b/pkg/contexts/ocm/transfer/transferhandler/standard/handler_test.go @@ -25,6 +25,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" @@ -108,7 +109,8 @@ var _ = Describe("Transfer handler", func() { env.Cleanup() }) - DescribeTable("it should copy a resource by value to a ctf file", func(acc string, topts ...transferhandler.TransferOption) { + DescribeTable("it should copy a resource by value to a ctf file", func(acc string, compose bool, topts ...transferhandler.TransferOption) { + compositionmodeattr.Set(env.OCMContext(), compose) src := Must(ctf.Open(env.OCMContext(), accessobj.ACC_WRITABLE, ARCH, 0, env)) defer Close(src, "source") cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) @@ -205,13 +207,24 @@ warning: version "github.com/mandelsoft/test:v1" already present, but differs }, Entry("without preserve global", - "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}"), + "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}", + false), Entry("with preserve global", "{\"globalAccess\":{\"imageReference\":\"alias.alias/ocm/value:v2.0\",\"type\":\"ociArtifact\"},\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}", + false, + standard.KeepGlobalAccess()), + + Entry("with composition and without preserve global", + "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}", + true), + Entry("with composition and with preserve global", + "{\"globalAccess\":{\"imageReference\":\"alias.alias/ocm/value:v2.0\",\"type\":\"ociArtifact\"},\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}", + true, standard.KeepGlobalAccess()), ) - DescribeTable("it should copy a resource by value to a ctf file for re-transport", func(acc string, topts ...transferhandler.TransferOption) { + DescribeTable("it should copy a resource by value to a ctf file for re-transport", func(acc string, mode bool, topts ...transferhandler.TransferOption) { + compositionmodeattr.Set(env.OCMContext(), mode) src := Must(ctf.Open(env.OCMContext(), accessobj.ACC_WRITABLE, ARCH, 0, env)) defer Close(src, "source") cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) @@ -333,9 +346,18 @@ warning: version "github.com/mandelsoft/test:v1" already present, but differs }, Entry("without preserve global", - "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}"), + "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}", + false), Entry("with preserve global", "{\"globalAccess\":{\"imageReference\":\"alias.alias/ocm/value:v2.0\",\"type\":\"ociArtifact\"},\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}", + false, + standard.KeepGlobalAccess()), + Entry("with composition and without preserve global", + "{\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\""+OCINAMESPACE+":"+OCIVERSION+"\",\"type\":\"localBlob\"}", + true), + Entry("with composition and with preserve global", + "{\"globalAccess\":{\"imageReference\":\"alias.alias/ocm/value:v2.0\",\"type\":\"ociArtifact\"},\"localReference\":\"%s\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}", + true, standard.KeepGlobalAccess()), ) @@ -401,7 +423,7 @@ warning: version "github.com/mandelsoft/test:v1" already present, but differs }) It("it should copy signatures", func() { - src, err := ctf.Open(env.OCMContext(), accessobj.ACC_READONLY, ARCH, 0, env) + src, err := ctf.Open(env.OCMContext(), accessobj.ACC_WRITABLE, ARCH, 0, env) Expect(err).To(Succeed()) cv, err := src.LookupComponentVersion(COMPONENT, VERSION) Expect(err).To(Succeed()) diff --git a/pkg/contexts/ocm/utils.go b/pkg/contexts/ocm/utils.go index ba03858be5..f1f4f047a4 100644 --- a/pkg/contexts/ocm/utils.go +++ b/pkg/contexts/ocm/utils.go @@ -125,3 +125,17 @@ func IsUnknownAccessSpec(s AccessSpec) bool { func WrapContextProvider(ctx LocalContextProvider) ContextProvider { return internal.WrapContextProvider(ctx) } + +func ReferenceHint(spec AccessSpec, cv ComponentVersionAccess) string { + if h, ok := spec.(internal.HintProvider); ok { + return h.GetReferenceHint(cv) + } + return "" +} + +func GlobalAccess(spec AccessSpec, ctx Context) AccessSpec { + if h, ok := spec.(internal.GlobalAccessProvider); ok { + return h.GlobalAccessSpec(ctx) + } + return nil +} diff --git a/pkg/env/builder/ocm_version_test.go b/pkg/env/builder/ocm_version_test.go new file mode 100644 index 0000000000..73e5beceeb --- /dev/null +++ b/pkg/env/builder/ocm_version_test.go @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package builder_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/contexts/ocm/testhelper" + . "github.com/open-component-model/ocm/pkg/env/builder" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compositionmodeattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" + ocmutils "github.com/open-component-model/ocm/pkg/contexts/ocm/utils" +) + +const ARCH = "/tmp/ctf" +const ARCH2 = "/tmp/ctf2" +const PROVIDER = "mandelsoft" +const VERSION = "v1" +const COMPONENT = "github.com/mandelsoft/test" +const OUT = "/tmp/res" + +var _ = Describe("Transfer handler", func() { + var env *Builder + + BeforeEach(func() { + env = NewBuilder() + compositionmodeattr.Set(env.OCMContext(), true) + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + env.Component(COMPONENT, func() { + env.Version(VERSION, func() { + env.Provider(PROVIDER) + TestDataResource(env) + }) + }) + }) + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("add ocm resource", func() { + src := Must(ctf.Open(env.OCMContext(), accessobj.ACC_READONLY, ARCH, 0, env)) + cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) + + Expect(len(cv.GetDescriptor().Resources)).To(Equal(1)) + + r := Must(cv.GetResourceByIndex(0)) + a := Must(r.Access()) + Expect(a.GetType()).To(Equal(localblob.Type)) + + data := Must(ocmutils.GetResourceData(r)) + Expect(string(data)).To(Equal(S_TESTDATA)) + }) +}) diff --git a/pkg/errors/notfound.go b/pkg/errors/notfound.go index 86f9425018..79d3546a0a 100644 --- a/pkg/errors/notfound.go +++ b/pkg/errors/notfound.go @@ -29,3 +29,11 @@ func IsErrNotFoundKind(err error, kind string) bool { } return uerr.kind == kind } + +func IsErrNotFoundElem(err error, kind, elem string) bool { + var uerr *NotFoundError + if err == nil || !As(err, &uerr) { + return false + } + return uerr.kind == kind && uerr.elem != nil && *uerr.elem == elem +} diff --git a/pkg/generics/cast.go b/pkg/generics/cast.go index be72bd3bee..c27e21d236 100644 --- a/pkg/generics/cast.go +++ b/pkg/generics/cast.go @@ -42,7 +42,7 @@ func AsE[T any](o interface{}, err error) (T, error) { return o.(T), err } -// AsI converts a struct pointetr to an appropriat interface type +// AsI converts a struct pointer to an appropriate interface type // given as type parameter. // In Go this cannot be done directly, because returning a nil pinter // for an interface return type, would result is a typed nil value for diff --git a/pkg/helm/chartaccess.go b/pkg/helm/chartaccess.go index e9248f9797..a3d7912996 100644 --- a/pkg/helm/chartaccess.go +++ b/pkg/helm/chartaccess.go @@ -16,7 +16,6 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessio/blobaccess" "github.com/open-component-model/ocm/pkg/common/accessio/refmgmt" "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" - "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/utils" ) @@ -80,21 +79,27 @@ func (c *chartAccess) unref() error { return fmt.Errorf("oops: refcount is already zero") } c.refcnt-- + if c.refcnt == 0 && c.closed { + return c.cleanup() + } return nil } func (c *chartAccess) Close() error { c.lock.Lock() defer c.lock.Unlock() - - if c.refcnt > 0 { - return errors.ErrStillInUse("chart access") - } - defer func() { c.closed = true }() + if c.refcnt == 0 { + return c.cleanup() + } + return nil +} - if c.root != "" && !c.closed { - return os.RemoveAll(c.root) +func (c *chartAccess) cleanup() error { + if c.root != "" { + err := os.RemoveAll(c.root) + c.root = "" + return err } return nil } diff --git a/pkg/helm/downloader.go b/pkg/helm/downloader.go index 746048587f..e47ce5439b 100644 --- a/pkg/helm/downloader.go +++ b/pkg/helm/downloader.go @@ -5,6 +5,7 @@ package helm import ( + "fmt" "strings" "github.com/mandelsoft/filepath/pkg/filepath" @@ -33,6 +34,9 @@ type chartDownloader struct { } func DownloadChart(out common.Printer, ctx oci.ContextProvider, ref, version, repourl string, opts ...Option) (ChartAccess, error) { + if version == "" { + return nil, fmt.Errorf("version required") + } repourl = strings.TrimSuffix(repourl, "/") acc, err := newTempChartAccess(osfs.New()) diff --git a/pkg/helm/identity/identity.go b/pkg/helm/identity/identity.go index e6288895ee..cb6baca730 100644 --- a/pkg/helm/identity/identity.go +++ b/pkg/helm/identity/identity.go @@ -51,7 +51,7 @@ the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) +var identityMatcher = hostpath.IdentityMatcher("") func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { return identityMatcher(pattern, cur, id) diff --git a/pkg/optionutils/nested.go b/pkg/optionutils/nested.go new file mode 100644 index 0000000000..04b28605c4 --- /dev/null +++ b/pkg/optionutils/nested.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package optionutils + +// NestedOptionsProvider is the interface for a +// more specific options object to provide +// access to a nested options object of type T. +// T must be a pointer type. +type NestedOptionsProvider[T any] interface { + NestedOptions() T +} + +// OptionTargetProvider is helper interface +// to declare a pointer type (*O) for an options +// object providing access to a nested +// options object of type N (must be a pointer type). +type OptionTargetProvider[N, O any] interface { + NestedOptionsProvider[N] + *O +} + +// OptionWrapper genericly wraps a nested option of type Option[N] to +// an option of type Option[*O], assuming that the nested option source +// N implements NestedOptionsProvider[N]. +// P is only a helper type parameter for Go and doesn't need to be given. +// It is the pointer type for O (P = *O). +// +// create a wrap option function for all wrappable options with +// +// func WrapXXX[O any, P optionutils.OptionTargetProvider[*Options, O]](args...any) optionutils.Option[P] { +// return optionutils.OptionWrapper[*Options, O, P](WithXXX(args...)) +// } +// +// where *Options is the type of the pointer type to the options object to be nested. +// +// The outer option functions wrapping the nested one can then be defined as +// +// func WithXXX(h string) Option { +// return optionutils.WrapXXX[Options](h) +// } +// +// For an example see package github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes/rpi. +func OptionWrapper[N, O any, P OptionTargetProvider[N, O]](o Option[N]) Option[P] { + return optionWrapper[N, O, P]{o} +} + +type optionWrapper[N, O any, P OptionTargetProvider[N, O]] struct { + opt Option[N] +} + +func (w optionWrapper[N, O, P]) ApplyTo(opts P) { + w.opt.ApplyTo(opts.NestedOptions()) +} + +/////////////////////////////////////////////////////////////////////////////(// + +func OptionWrapperFunc[N, O any](o Option[N], nested func(outer O) N) Option[O] { + return optionWrapperFunc[N, O]{o, nested} +} + +type optionWrapperFunc[N, O any] struct { + opt Option[N] + nested func(O) N +} + +func (w optionWrapperFunc[N, O]) ApplyTo(opts O) { + w.opt.ApplyTo(w.nested(opts)) +} diff --git a/pkg/optionutils/options.go b/pkg/optionutils/options.go new file mode 100644 index 0000000000..80a29e313e --- /dev/null +++ b/pkg/optionutils/options.go @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package optionutils + +type Option[T any] interface { + ApplyTo(T) +} + +func EvalOptions[O any](opts ...Option[*O]) *O { + var eff O + for _, opt := range opts { + if opt != nil { + opt.ApplyTo(&eff) + } + } + return &eff +} diff --git a/pkg/optionutils/pointer.go b/pkg/optionutils/pointer.go new file mode 100644 index 0000000000..a982b1c6b6 --- /dev/null +++ b/pkg/optionutils/pointer.go @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package optionutils + +func PointerTo[T any](v T) *T { + temp := v + return &temp +} + +func AsValue[T any](p *T) T { + var r T + if p != nil { + r = *p + } + return r +} diff --git a/pkg/utils/tarutils/list.go b/pkg/utils/tarutils/list.go new file mode 100644 index 0000000000..e61ff51e27 --- /dev/null +++ b/pkg/utils/tarutils/list.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package tarutils + +import ( + "archive/tar" + "io" + + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common/compression" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/utils" +) + +func ListArchiveContent(path string, fss ...vfs.FileSystem) ([]string, error) { + sfs := utils.OptionalDefaulted(osfs.New(), fss...) + + f, err := sfs.Open(path) + if err != nil { + return nil, errors.Wrapf(err, "cannot open %s", path) + } + defer f.Close() + return ListArchiveContentFromReader(f) +} + +func ListArchiveContentFromReader(r io.Reader) ([]string, error) { + in, _, err := compression.AutoDecompress(r) + if err != nil { + return nil, errors.Wrapf(err, "cannot determine compression") + } + + var result []string + + tr := tar.NewReader(in) + for { + header, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return result, nil + } + return nil, err + } + + switch header.Typeflag { + case tar.TypeDir: + result = append(result, header.Name) + case tar.TypeSymlink, tar.TypeLink: + result = append(result, header.Name) + case tar.TypeReg: + result = append(result, header.Name) + } + } +}