start working on composition mode (#552)

* compose

* align Add/NewVerion for comparch

* prepare composition mode

* various fixes + tests

* more fixes

* Extend AccessProvider

* fix all tests

* picture

* artifact sources

* generalize option support + docerdeamon resource

* fix helm chart handling

* more blob accesses for inputs

* composition repo and version + meta composition

* blob compression + composition version

README.md

@@ -27,7 +27,7 @@ The library currently supports the following [repository mappings](docs/ocm/inte
 - **Component Archive**: Compose the content of a component version on the
   filesystem.
 
-For the usage of the library to access OCM repositories, handle configuratio and credentials see the [examples section](examples/lib/README.md).
+For the usage of the library to access OCM repositories, handle configuration and credentials see the [examples section](examples/lib/README.md).
 
 Additionally, OCM provides a generic solution for how to:
 

cmds/ocm/commands/ocmcmds/common/inputs/cpi/helper.go

@@ -157,7 +157,7 @@ func (s *MediaFileSpec) ValidateFile(fldPath *field.Path, ctx clictx.Context, in
 		pathField := fldPath.Child("path")
 		fileInfo, filePath, err := inputs.FileInfo(ctx, s.Path, inputFilePath)
 		if err != nil {
-			allErrs = append(allErrs, field.Invalid(pathField, filePath, err.Error()))
+			allErrs = append(allErrs, field.Invalid(pathField, s.Path, err.Error()))
 		}
 		return fileInfo, filePath, allErrs
 	}

cmds/ocm/commands/ocmcmds/common/inputs/types/docker/spec.go

@@ -11,9 +11,7 @@ import (
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/cpi"
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/ociimage"
 	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess"
-	"github.com/open-component-model/ocm/pkg/contexts/oci"
-	"github.com/open-component-model/ocm/pkg/contexts/oci/annotations"
-	"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset"
+	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockerdaemon"
 	"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact"
 )
@@ -53,25 +51,7 @@ func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (bloba
 	if err != nil {
 		return nil, "", err
 	}
-	spec := docker.NewRepositorySpec()
-	repo, err := ctx.OCIContext().RepositoryForSpec(spec)
-	if err != nil {
-		return nil, "", err
-	}
-	ns, err := repo.LookupNamespace(locator)
-	if err != nil {
-		return nil, "", err
-	}
-
-	if version == "" || version == "latest" {
-		version = info.ComponentVersion.GetVersion()
-	}
-	blob, err := artifactset.SynthesizeArtifactBlob(ns, version,
-		func(art oci.ArtifactAccess) error {
-			art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String())
-			return nil
-		},
-	)
+	blob, version, err := dockerdaemon.BlobAccessForImageFromDockerDaemon(s.Path, dockerdaemon.WithVersion(info.ComponentVersion.GetVersion()), dockerdaemon.WithOrigin(info.ComponentVersion))
 	if err != nil {
 		return nil, "", err
 	}

cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti/spec.go

@@ -9,21 +9,17 @@ import (
 
 	. "github.com/open-component-model/ocm/pkg/finalizer"
 
-	"github.com/opencontainers/go-digest"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs"
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/ociimage"
 	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess"
+	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/dockermulti"
 	"github.com/open-component-model/ocm/pkg/contexts/clictx"
 	"github.com/open-component-model/ocm/pkg/contexts/oci"
-	"github.com/open-component-model/ocm/pkg/contexts/oci/annotations"
-	"github.com/open-component-model/ocm/pkg/contexts/oci/artdesc"
-	"github.com/open-component-model/ocm/pkg/contexts/oci/cpi"
 	"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset"
 	"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/docker"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact"
-	"github.com/open-component-model/ocm/pkg/errors"
 	"github.com/open-component-model/ocm/pkg/runtime"
 )
 
@@ -100,77 +96,12 @@ func (s *Spec) getVariant(ctx clictx.Context, finalize *Finalizer, variant strin
 }
 
 func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blobaccess.BlobAccess, string, error) {
-	index := artdesc.NewIndexArtifact()
-	i := 0
-
-	index.SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String())
-
-	feedback := func(blob blobaccess.BlobAccess, art cpi.ArtifactAccess) error {
-		desc := artdesc.DefaultBlobDescriptor(blob)
-		if art.IsManifest() {
-			cfgBlob, err := art.ManifestAccess().GetConfigBlob()
-			if err != nil {
-				return errors.Wrapf(err, "cannot get config blob")
-			}
-			cfg, err := artdesc.ParseImageConfig(cfgBlob)
-			if err != nil {
-				return errors.Wrapf(err, "cannot parse config blob")
-			}
-			if cfg.Architecture != "" {
-				desc.Platform = &artdesc.Platform{
-					Architecture: cfg.Architecture,
-					OS:           cfg.OS,
-					Variant:      cfg.Variant,
-				}
-			}
-		}
-		index.Index().AddManifest(desc)
-		return nil
-	}
-
-	blob, err := artifactset.SynthesizeArtifactBlobFor(info.ComponentVersion.GetVersion(), func() (fac artifactset.ArtifactFactory, main bool, err error) {
-		var art cpi.ArtifactAccess
-		var blob blobaccess.BlobAccess
-
-		switch {
-		case i > len(s.Variants):
-			// end loop
-		case i == len(s.Variants):
-			// provide index (main) artifact
-			ctx.Printf("image %d: INDEX\n", i)
-			fac = func(set *artifactset.ArtifactSet) (digest.Digest, string, error) {
-				art, err = set.NewArtifact(index)
-				if err != nil {
-					return "", "", errors.Wrapf(err, "cannot create index artifact")
-				}
-				defer art.Close()
-				blob, err = set.AddArtifact(art)
-				if err != nil {
-					return "", "", errors.Wrapf(err, "cannot add index artifact")
-				}
-				defer blob.Close()
-				return blob.Digest(), blob.MimeType(), nil
-			}
-			main = true
-		default:
-			// provide variant
-			ctx.Printf("image %d: %s\n", i, s.Variants[i])
-			var finalize Finalizer
-
-			art, err = s.getVariant(ctx, &finalize, s.Variants[i])
-
-			if err == nil {
-				art.Artifact().SetAnnotation(annotations.COMPVERS_ANNOTATION, info.ComponentVersion.String())
-				blob, err = art.Blob()
-				if err == nil {
-					finalize.Close(art)
-					fac = artifactset.ArtifactTransferCreator(art, &finalize, feedback)
-				}
-			}
-		}
-		i++
-		return
-	})
+	blob, err := dockermulti.BlobAccessForMultiImageFromDockerDaemon(
+		dockermulti.WithContext(ctx),
+		dockermulti.WithPrinter(ctx.Printer()),
+		dockermulti.WithVariants(s.Variants...),
+		dockermulti.WithOrigin(info.ComponentVersion),
+		dockermulti.WithVersion(info.ComponentVersion.GetVersion()))
 	if err != nil {
 		return nil, "", err
 	}

cmds/ocm/commands/ocmcmds/common/inputs/types/helm/spec.go

@@ -5,19 +5,14 @@
 package helm
 
 import (
-	"github.com/mandelsoft/vfs/pkg/vfs"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs"
 	"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/cpi"
-	"github.com/open-component-model/ocm/pkg/common"
 	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess"
-	ocihelm "github.com/open-component-model/ocm/pkg/contexts/oci/ociutils/helm"
+	"github.com/open-component-model/ocm/pkg/common/accessio/blobaccess/helm"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact"
 	"github.com/open-component-model/ocm/pkg/errors"
-	"github.com/open-component-model/ocm/pkg/helm"
-	"github.com/open-component-model/ocm/pkg/helm/identity"
-	"github.com/open-component-model/ocm/pkg/helm/loader"
 )
 
 type Spec struct {
@@ -93,60 +88,34 @@ func (s *Spec) Validate(fldPath *field.Path, ctx inputs.Context, inputFilePath s
 }
 
 func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blob blobaccess.BlobAccess, hint string, err error) {
-	var chartLoader loader.Loader
+	path := s.Path
 	if s.HelmRepository == "" {
-		_, inputPath, err := inputs.FileInfo(ctx, s.Path, info.InputFilePath)
+		_, inputPath, err := inputs.FileInfo(ctx, path, info.InputFilePath)
 		if err != nil {
 			return nil, "", errors.Wrapf(err, "cannot handle input path %q", s.Path)
 		}
-		chartLoader = loader.VFSLoader(inputPath, ctx.FileSystem())
-
-	} else {
-		cert := []byte(s.CACert)
-		if s.CACertFile != "" {
-			_, certPath, err := inputs.FileInfo(ctx, s.CACertFile, info.InputFilePath)
-			if err != nil {
-				return nil, "", err
-			}
-			cert, err = vfs.ReadFile(ctx.FileSystem(), certPath)
-			if err != nil {
-				return nil, "", errors.Wrapf(err, "cannot read root certificates from %q", s.CACertFile)
-			}
-		}
-
-		acc, err := helm.DownloadChart(common.NewPrinter(ctx.StdOut()), ctx, s.Path, s.Version, s.HelmRepository,
-			helm.WithCredentials(identity.GetCredentials(ctx, s.HelmRepository, s.Path)),
-			helm.WithRootCert([]byte(cert)))
-		if err != nil {
-			return nil, "", errors.Wrapf(err, "cannot download chart %s:%s from %s", s.Path, s.Version, s.HelmRepository)
-		}
-		chartLoader = loader.AccessLoader(acc)
-	}
-
-	defer errors.PropagateError(&err, chartLoader.Close)
-
-	chart, err := chartLoader.Chart()
-	if err != nil {
-		return nil, "", err
-	}
-	vers := chart.Metadata.Version
-	if s.Version != "" {
-		vers = s.Version
+		path = inputPath
 	}
+	vers := s.Version
+	override := true
 	if vers == "" {
 		vers = info.ComponentVersion.GetVersion()
+		override = false
 	}
 
-	hint = ociartifact.Hint(info.ComponentVersion, chart.Name(), s.Repository, vers)
-	blob, err = chartLoader.ChartArtefactSet()
-	if err != nil || blob != nil {
-		return blob, hint, err
-	}
-	blob, err = ocihelm.SynthesizeArtifactBlob(chartLoader)
+	blob, name, vers, err := helm.BlobAccessForHelmChart(path,
+		helm.WithContext(ctx),
+		helm.WithFileSystem(ctx.FileSystem()),
+		helm.WithPrinter(ctx.Printer()),
+		helm.WithVersionOverride(vers, override),
+		helm.WithCACert(s.CACert),
+		helm.WithCACertFile(s.CACertFile),
+		helm.WithHelmRepository(s.HelmRepository),
+	)
 	if err != nil {
-		return nil, "", errors.Wrapf(err, "cannot synthesize artifact blob")
+		return nil, "", err
 	}
-
+	hint = ociartifact.Hint(info.ComponentVersion, name, s.Repository, vers)
 	return blob, hint, err
 }
 

cmds/ocm/commands/ocmcmds/routingslips/add/cmd.go

@@ -223,5 +223,5 @@ func (a *action) Out() error {
 	} else {
 		_, err = routingslip.AddEntry(cv, a.cmd.Name, a.cmd.Algorithm, a.cmd.Entry, links, digest.Digest(a.cmd.Digest))
 	}
-	return err
+	return cv.Update()
 }

cmds/ocm/commands/ocmcmds/routingslips/get/cmd_test.go

@@ -49,6 +49,7 @@ var _ = Describe("Test Environment", func() {
 		defer Close(cv)
 
 		e1a = Must(routingslip.AddEntry(cv, PROVIDER, rsa.Algorithm, comment.New("first entry"), nil))
+		MustBeSuccessful(cv.Update())
 	})
 
 	AfterEach(func() {
@@ -66,6 +67,7 @@ var _ = Describe("Test Environment", func() {
 		slip := Must(routingslip.GetSlip(cv, PROVIDER))
 		slip.Get(0).Signature.Value = slip.Get(0).Signature.Value[1:] + "0"
 		MustBeSuccessful(routingslip.SetSlip(cv, slip))
+		MustBeSuccessful(cv.Update())
 		MustBeSuccessful(finalize.Finalize())
 
 		buf := bytes.NewBuffer(nil)
@@ -110,6 +112,8 @@ test.de/x:v1      acme.org comment ` + e1a.Timestamp.String() + ` Comment: first
 				"status", "passed",
 			))
 			e2c = Must(routingslip.AddEntry(cv, OTHER, rsa.Algorithm, te, nil))
+
+			MustBeSuccessful(cv.Update())
 		})
 
 		It("gets different slips", func() {
@@ -168,6 +172,7 @@ acme.org/test ` + digests(e2c, e2b) + `       name: unit-tests, status: passed
 					Name:   PROVIDER,
 					Digest: e1a.Digest,
 				}}))
+				MustBeSuccessful(cv.Update())
 			})
 			It("gets dedicated wide slip with link", func() {
 				buf := bytes.NewBuffer(nil)

components/ecrplugin/Makefile

@@ -60,6 +60,11 @@ $(GEN)/ca.done: $(GEN)/.exists $(GEN)/build resources.yaml $(CHARTSRCS)
 	$(OCM) add resources --templater=spiff --file $(GEN)/ca NAME="$(NAME)" VERSION="$(VERSION)" COMMIT="$(COMMIT)" GEN="$(GEN)" PLATFORMS="$(PLATFORMS)" resources.yaml
 	@touch $(GEN)/ca.done
 
+.PHONY: plain-ca
+plain-ca: $(GEN)/.exists resources.yaml $(CHARTSRCS)
+	$(OCM) create ca -f $(COMPONENT) "$(VERSION)" --provider $(PROVIDER) --file $(GEN)/ca
+	$(OCM) add resources --templater=spiff --file $(GEN)/ca NAME="$(NAME)" VERSION="$(VERSION)" COMMIT="$(COMMIT)" GEN="$(GEN)" PLATFORMS="$(PLATFORMS)" resources.yaml
+
 .PHONY: push
 push: $(GEN)/ctf $(GEN)/push.$(NAME)
 

components/helminstaller/Makefile

@@ -56,7 +56,7 @@ $(GEN)/ca: $(GEN)/.exists $(GEN)/image.$(NAME)$(FLAGSUF) resources.yaml executor
 
 .PHONY: plain-ca
 plain-ca: $(GEN)/.exists
-	$(OCM) create ca -f $(COMPONENT) "$(VERSION)"--provider  $(PROVIDER) --file $(GEN)/ca
+	$(OCM) create ca -f $(COMPONENT) "$(VERSION)" --provider  $(PROVIDER) --file $(GEN)/ca
 	$(OCM) add resources --templater spiff $(GEN)/ca $(ATTRIBUTES) resources.yaml
 	@touch $(GEN)/ca
 

components/helminstaller/a.yaml

@@ -0,0 +1 @@
+flag: (( flag ? "yes" :"no" ))

components/helminstaller/resources.yaml

@@ -3,10 +3,10 @@ name: toiimage
 type: ociImage
 version: (( values.VERSION ))
 input:
-  type: (( values.MULTI ? "dockermulti" :"docker" ))
+  type: (( bool(values.MULTI) ? "dockermulti" :"docker" ))
   repository:  (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE ))
-  variants: (( values.MULTI ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
-  path: (( !values.MULTI ? values.IMAGE :~~ ))
+  variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
+  path: (( !bool(values.MULTI) ? values.IMAGE :~~ ))
 ---
 name: toiexecutor
 type: toiExecutor

components/ocmcli/resources.yaml

@@ -21,10 +21,10 @@ helper:
     type: ociImage
     version: (( values.VERSION ))
     input:
-      type: (( values.MULTI ? "dockermulti" :"docker" ))
+      type: (( bool(values.MULTI) ? "dockermulti" :"docker" ))
       repository: (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE ))
-      variants: (( values.MULTI ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
-      path: (( !values.MULTI ? values.IMAGE :~~ ))
+      variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
+      path: (( !bool(values.MULTI) ? values.IMAGE :~~ ))
 
 
 resources: (( map[split(" ", values.PLATFORMS)|p|-> *helper.executable] *helper.image ))