From 6bc7f62e2b21c8bc14b756d1b6e4ec4a47a8df71 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 9 Apr 2024 12:16:33 +0200 Subject: [PATCH 001/100] init --- pkg/contexts/ocm/accessmethods/mvn/README.md | 32 +++ pkg/contexts/ocm/accessmethods/mvn/cli.go | 42 ++++ pkg/contexts/ocm/accessmethods/mvn/method.go | 193 ++++++++++++++++++ .../ocm/accessmethods/mvn/method_test.go | 65 ++++++ .../ocm/accessmethods/mvn/suite_test.go | 13 ++ pkg/mime/types.go | 6 +- 6 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/README.md create mode 100644 pkg/contexts/ocm/accessmethods/mvn/cli.go create mode 100644 pkg/contexts/ocm/accessmethods/mvn/method.go create mode 100644 pkg/contexts/ocm/accessmethods/mvn/method_test.go create mode 100644 pkg/contexts/ocm/accessmethods/mvn/suite_test.go diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md new file mode 100644 index 0000000000..e077f3c2ea --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -0,0 +1,32 @@ +# `mvn` - Java packages (jar) in a Maven (mvn) repository (e.g. mvnrepository.com) + +### Synopsis +``` +type: mvn/v1 +``` + +Provided blobs use the following media type: `application/x-tgz` // FIXME: Do we need to define a new media type for this? + +### Description + +This method implements the access of a Java package from a Maven (mvn) repository. + +### Specification Versions + +Supported specification version is `v1` + +#### Version `v1` + +The type specific specification fields are: + +- **`repository`** *string* + + Base URL of the Maven (mvn) repository. + +- **`package`** *string* + + The name of the Maven (mvn) package. + +- **`version`** *string* + + The version of the Maven (mvn) package. diff --git a/pkg/contexts/ocm/accessmethods/mvn/cli.go b/pkg/contexts/ocm/accessmethods/mvn/cli.go new file mode 100644 index 0000000000..391337f99d --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/cli.go @@ -0,0 +1,42 @@ +package mvn + +import ( + "github.com/open-component-model/ocm/pkg/cobrautils/flagsets" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/options" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + Type, AddConfig, + options.RegistryOption, + options.PackageOption, + options.VersionOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository") + flagsets.AddFieldByOptionP(opts, options.PackageOption, config, "package") + flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "version") + return nil +} + +var usage = ` +This method implements the access of a Maven (mvn) package in a Maven repository. +` + +var formatV1 = ` +The type specific specification fields are: + +- **repository** *string* + + Base URL of the Maven (mvn) repository. + +- **package** *string* + + The name of the Maven (mvn) package + +- **version** *string* + + The version name of the Maven (mvn) package +` diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go new file mode 100644 index 0000000000..980c684d11 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -0,0 +1,193 @@ +package mvn + +import ( + "bytes" + "context" + "crypto" + "encoding/json" + "encoding/xml + "fmt" + "io" + "net/http" + "path" + "strings" + + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/datacontext/attrs/vfsattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/runtime" +) + +// Type is the access type of Maven (mvn) repository. +const ( + Type = "mvn" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) + +func init() { + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) +} + +// AccessSpec describes the access for a Maven (mvn) repository. +type AccessSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // Repository is the base URL of the Maven (mvn) repository + Repository string `json:"repository"` + // ArtifactId is the name of Maven (mvn) package + GroupId string `json:"groupId"` + // ArtifactId is the name of Maven (mvn) package + ArtifactId string `json:"artifactId"` + // Version of the Maven (mvn) package. + Version string `json:"version"` +} + +var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) + +// New creates a new Maven (mvn) repository access spec version v1. +func New(repository, groupId, artifactId, version string) *AccessSpec { + return &AccessSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + Repository: repository, + GroupId: groupId, + ArtifactId: artifactId, + Version: version, + } +} + +func (a *AccessSpec) Describe(_ accspeccpi.Context) string { + return fmt.Sprintf("Maven (mvn) package %s:%s:%s in repository %s", a.GroupId, a.ArtifactId, a.Version, a.Repository) +} + +func (_ *AccessSpec) IsLocal(accspeccpi.Context) bool { + return false +} + +func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpec { + return a +} + +func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { + return a.GroupId + ":" + a.ArtifactId + ":" + a.Version +} + +func (_ *AccessSpec) GetType() string { + return Type +} + +func (a *AccessSpec) AccessMethod(c accspeccpi.ComponentVersionAccess) (accspeccpi.AccessMethod, error) { + return accspeccpi.AccessMethodForImplementation(newMethod(c, a)) +} + +func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.ComponentVersionAccess) string { + meta, _ := a.getPackageMeta(access.GetContext()) + if meta != nil { + return meta.Dist.Shasum + } + return "" +} + +func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) { + // cn.afternode.commons + // commons + // 1.6 + // repository: https://repo1.maven.org/maven2/ + // groupId: cn/afternode/commons + // version: 1.6 + // artifactId: commons + url := a.Repository + path.Join("/", strings.Replace(a.GroupId, ".", "/", -1), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".pom") + // https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/commons-1.6.pom + r, err := reader(url, vfsattr.Get(ctx)) + if err != nil { + return nil, err + } + buf := &bytes.Buffer{} + _, err = io.Copy(buf, io.LimitReader(r, 200000)) + if err != nil { + return nil, errors.Wrapf(err, "cannot get version metadata for %s", url) + } + + var metadata meta + + // read xml from buf and fill metadata + xml.Unmarshal(buf.Bytes(), &metadata) + + err = json.Unmarshal(buf.Bytes(), &metadata) + if err != nil { + return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", url) + } + return &metadata, nil +} + +//////////////////////////////////////////////////////////////////////////////// + +func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { + factory := func() (blobaccess.BlobAccess, error) { + meta, err := a.getPackageMeta(c.GetContext()) + if err != nil { + return nil, err + } + + f := func() (io.ReadCloser, error) { + return reader(meta.Dist.Tarball, vfsattr.Get(c.GetContext())) + } + if meta.Dist.Shasum != "" { + tf := f + f = func() (io.ReadCloser, error) { + r, err := tf() + if err != nil { + return nil, err + } + return iotools.VerifyingReaderWithHash(r, crypto.SHA1, meta.Dist.Shasum), nil + } + } + acc := blobaccess.DataAccessForReaderFunction(f, meta.Dist.Tarball) + return accessobj.CachedBlobAccessForWriter(c.GetContext(), mime.MIME_JAR, accessio.NewDataAccessWriter(acc)), nil + } + return accspeccpi.NewDefaultMethodImpl(c, a, "", mime.MIME_JAR, factory), nil +} + +type meta struct { + Type string `json:"type"` // pom, jar, sources, javadoc, module, ... + MD5 string `json:"md5"` + Sha1 string `json:"sha1"` + Sha256 string `json:"sha256"` + Sha512 string `json:"sha512"` + Asc string `json:"asc"` +} + +func reader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { + c := &http.Client{} + + if strings.HasPrefix(url, "file://") { + path := url[7:] + return fs.OpenFile(path, vfs.O_RDONLY, 0o600) + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + buf := &bytes.Buffer{} + _, err = io.Copy(buf, io.LimitReader(resp.Body, 2000)) + if err != nil { + return nil, errors.Newf("version meta data request %s provides %s", url, resp.Status) + } + return nil, errors.Newf("version meta data request %s provides %s: %s", url, resp.Status, buf.String()) + } + return resp.Body, nil +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go new file mode 100644 index 0000000000..504a0654d6 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -0,0 +1,65 @@ +package mvn_test + +import ( + "crypto" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/env" + . "github.com/open-component-model/ocm/pkg/env/builder" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" // FIXME: Update import path + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/mime" +) + +const mvnPATH = "/testdata/registry" +const FAILPATH = "/testdata/failregistry" + +var _ = Describe("Method", func() { + var env *Builder + var cv ocm.ComponentVersionAccess + + BeforeEach(func() { + env = NewBuilder(TestData()) + cv = &cpi.DummyComponentVersionAccess{env.OCMContext()} + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("accesses artifact", func() { + acc := mvn.New("file://"+mvnPATH, "yargs", "17.7.1") + // acc := mvn.New("https://registry.mvnjs.org", "yargs", "17.7.1") + + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + + r := Must(m.Reader()) + defer r.Close() + dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) + for { + var buf [8096]byte + _, err := dr.Read(buf[:]) + if err != nil { + break + } + } + Expect(dr.Size()).To(Equal(int64(65690))) + Expect(dr.Digest().String()).To(Equal("SHA-1:34a77645201d1a8fc5213ace787c220eabbd0967")) + }) + + It("detects digests mismatch", func() { + acc := mvn.New("file://"+FAILPATH, "yargs", "17.7.1") + + m := Must(acc.AccessMethod(cv)) + defer m.Close() + _, err := m.Reader() + Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found 34a77645201d1a8fc5213ace787c220eabbd0967"))) + }) +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/suite_test.go b/pkg/contexts/ocm/accessmethods/mvn/suite_test.go new file mode 100644 index 0000000000..c62cadc7b0 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/suite_test.go @@ -0,0 +1,13 @@ +package mvn_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Maven (mvn) Test Suite") +} diff --git a/pkg/mime/types.go b/pkg/mime/types.go index 9a56cac904..677f5030b2 100644 --- a/pkg/mime/types.go +++ b/pkg/mime/types.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package mime const ( @@ -19,4 +15,6 @@ const ( MIME_TAR = "application/x-tar" MIME_TGZ = "application/x-tgz" MIME_TGZ_ALT = MIME_TAR + "+gzip" + + MIME_JAR = "application/x-jar" ) From 995d87cbbe9466027307fd7afa85c0f82a073221 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 16:30:51 +0200 Subject: [PATCH 002/100] testdata --- .../5.7.0/sdk-modules-bom-5.7.0.pom | 210 ++++++++++++++++++ .../fail/repository/42/repository-42.pom | 210 ++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom create mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom new file mode 100644 index 0000000000..b3baaee32f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom @@ -0,0 +1,210 @@ + + + 4.0.0 + com.sap.cloud.sdk + sdk-modules-bom + 5.7.0 + pom + SAP Cloud SDK - Modules BOM + Bill of Materials (BOM) of the SAP Cloud SDK modules. + https://sap.github.io/cloud-sdk/docs/java/getting-started + + SAP SE + https://www.sap.com + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + SAP + cloudsdk@sap.com + SAP SE + https://www.sap.com + + + + + + + + UTF-8 + Public + Stable + + 5.7.0 + + + + + com.sap.cloud.sdk + sdk-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + dwc-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + caching + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient4 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient5 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity-scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-destination-service + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-oauth + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-dwc + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-ztis + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience-api + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience4j + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + security + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + servlet-jakarta + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + tenant + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-core + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + rfc + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + datamodel-metadata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator-utility + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-client + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + fluent-result + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + + soap + ${sdk.version} + + + + diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom new file mode 100644 index 0000000000..b3baaee32f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom @@ -0,0 +1,210 @@ + + + 4.0.0 + com.sap.cloud.sdk + sdk-modules-bom + 5.7.0 + pom + SAP Cloud SDK - Modules BOM + Bill of Materials (BOM) of the SAP Cloud SDK modules. + https://sap.github.io/cloud-sdk/docs/java/getting-started + + SAP SE + https://www.sap.com + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + SAP + cloudsdk@sap.com + SAP SE + https://www.sap.com + + + + + + + + UTF-8 + Public + Stable + + 5.7.0 + + + + + com.sap.cloud.sdk + sdk-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + dwc-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + caching + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient4 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient5 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity-scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-destination-service + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-oauth + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-dwc + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-ztis + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience-api + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience4j + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + security + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + servlet-jakarta + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + tenant + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-core + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + rfc + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + datamodel-metadata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator-utility + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-client + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + fluent-result + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + + soap + ${sdk.version} + + + + From 3015cabcf7ec67a853dc26ce87fc21fc3b89ba5a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 16:31:17 +0200 Subject: [PATCH 003/100] cli flags/options --- pkg/contexts/ocm/accessmethods/mvn/cli.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/cli.go b/pkg/contexts/ocm/accessmethods/mvn/cli.go index 391337f99d..56445a91f1 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/cli.go +++ b/pkg/contexts/ocm/accessmethods/mvn/cli.go @@ -8,7 +8,8 @@ import ( func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { return flagsets.NewConfigOptionTypeSetHandler( Type, AddConfig, - options.RegistryOption, + options.RepositoryOption, + options.GroupOption, options.PackageOption, options.VersionOption, ) @@ -16,13 +17,14 @@ func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository") - flagsets.AddFieldByOptionP(opts, options.PackageOption, config, "package") + flagsets.AddFieldByOptionP(opts, options.GroupOption, config, "groupId") + flagsets.AddFieldByOptionP(opts, options.PackageOption, config, "artifactId") flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "version") return nil } var usage = ` -This method implements the access of a Maven (mvn) package in a Maven repository. +This method implements the access of a Maven (mvn) artifact in a Maven repository. ` var formatV1 = ` @@ -32,11 +34,15 @@ The type specific specification fields are: Base URL of the Maven (mvn) repository. -- **package** *string* +- **groupId** *string* - The name of the Maven (mvn) package + The groupId of the Maven (mvn) artifact + +- **artifactId** *string* + + The artifactId of the Maven (mvn) artifact - **version** *string* - The version name of the Maven (mvn) package + The version name of the Maven (mvn) artifact ` From 0a0915ee467422dda2e5a7653e9ff03689327157 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 16:31:34 +0200 Subject: [PATCH 004/100] access method --- pkg/contexts/ocm/accessmethods/mvn/method.go | 154 +++++++++++++----- .../ocm/accessmethods/mvn/method_test.go | 44 ++++- 2 files changed, 145 insertions(+), 53 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 980c684d11..c7e92a44da 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "crypto" - "encoding/json" - "encoding/xml + "encoding/xml" "fmt" + npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" + "github.com/open-component-model/ocm/pkg/logging" "io" "net/http" "path" @@ -36,6 +37,8 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } +var REALM = logging.DefineSubRealm("Maven repository", "mvn") + // AccessSpec describes the access for a Maven (mvn) repository. type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` @@ -88,83 +91,146 @@ func (a *AccessSpec) AccessMethod(c accspeccpi.ComponentVersionAccess) (accspecc } func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.ComponentVersionAccess) string { - meta, _ := a.getPackageMeta(access.GetContext()) + meta, _ := a.GetPackageMeta(access.GetContext()) if meta != nil { - return meta.Dist.Shasum + return meta.Hash } return "" } -func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) { - // cn.afternode.commons - // commons - // 1.6 - // repository: https://repo1.maven.org/maven2/ - // groupId: cn/afternode/commons - // version: 1.6 - // artifactId: commons - url := a.Repository + path.Join("/", strings.Replace(a.GroupId, ".", "/", -1), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".pom") - // https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/commons-1.6.pom - r, err := reader(url, vfsattr.Get(ctx)) +type meta struct { + Packaging string `json:"packaging"` + HashType crypto.Hash `json:"hashType"` + Hash string `json:"hash"` + Asc string `json:"asc"` + Bin string `json:"bin"` +} + +func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { + // this is how the usual maven repository structure looks like + urlPrefix := a.Repository + path.Join("/", strings.Replace(a.GroupId, ".", "/", -1), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".") + fs := vfsattr.Get(ctx) + + // first let's read the pom file and check which binary artifact we need to read + log := logging.Context().Logger(npmCredentials.REALM) + log.Debug("Reading %s", urlPrefix+"pom") + pom, err := readPom(urlPrefix+"pom", fs) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "cannot read pom file for %s", urlPrefix) } - buf := &bytes.Buffer{} - _, err = io.Copy(buf, io.LimitReader(r, 200000)) + + hashType, hash, err := getBestHashValue(urlPrefix+pom.Packaging, fs) if err != nil { - return nil, errors.Wrapf(err, "cannot get version metadata for %s", url) + log.Debug("Could not find any hash value for %s", urlPrefix+pom.Packaging) + } + asc, err := getStringData(urlPrefix+pom.Packaging+".asc", fs) + if err != nil { + log.Debug("No signing info found for %s", urlPrefix+pom.Packaging) } - var metadata meta + var metadata meta = meta{ + Packaging: pom.Packaging, + HashType: hashType, + Hash: hash, + Bin: urlPrefix + pom.Packaging, + Asc: asc, + } + + return &metadata, nil +} - // read xml from buf and fill metadata - xml.Unmarshal(buf.Bytes(), &metadata) +//////////////////////////////////////////////////////////////////////////////// - err = json.Unmarshal(buf.Bytes(), &metadata) +type project struct { + GroupId string `xml:"groupId"` + ArtifactId string `xml:"artifactId"` + Version string `xml:"version"` + Packaging string `xml:"packaging"` +} + +func readPom(url string, fs vfs.FileSystem) (*project, error) { + reader, err := getReader(url, fs) + if err != nil { + return nil, err + } + buf := &bytes.Buffer{} + _, err = io.Copy(buf, io.LimitReader(reader, 200000)) + if err != nil { + return nil, errors.Wrapf(err, "cannot get version metadata for %s", url) + } + var pom project + err = xml.Unmarshal(buf.Bytes(), &pom) if err != nil { return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", url) } - return &metadata, nil + if pom.Packaging == "" { + pom.Packaging = "jar" + } + return &pom, nil } -//////////////////////////////////////////////////////////////////////////////// +// getStringData reads all data from the given URL and returns it as a string. +func getStringData(url string, fs vfs.FileSystem) (string, error) { + r, err := getReader(url, fs) + if err != nil { + return "", err + } + b, err := io.ReadAll(r) + if err != nil { + return "", err + } + return string(b), nil +} + +// getBestHashValue returns the best hash value (SHA-512, SHA-256, SHA-1, MD5) for the given artifact (POM, JAR, ...). +func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error) { + arr := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} + log := logging.Context().Logger(npmCredentials.REALM) + for i := 0; i < len(arr); i++ { + v, err := getStringData(url+hashUrlExt(arr[i]), fs) + if v != "" { + return arr[i], v, err + } + if err != nil { + log.Debug("hash value not found: %s - %s", url+hashUrlExt(arr[i]), err) + } + } + return 0, "", errors.New("no hash value found") +} + +// hashUrlExt returns the 'maven' hash extension for the given hash. +// Maven usually uses sha1, sha256, sha512, md5 instead of SHA-1, SHA-256, SHA-512, MD5. +func hashUrlExt(h crypto.Hash) string { + return "." + strings.ReplaceAll(strings.ToLower(h.String()), "-", "") +} func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { factory := func() (blobaccess.BlobAccess, error) { - meta, err := a.getPackageMeta(c.GetContext()) + meta, err := a.GetPackageMeta(c.GetContext()) if err != nil { return nil, err } - f := func() (io.ReadCloser, error) { - return reader(meta.Dist.Tarball, vfsattr.Get(c.GetContext())) + reader := func() (io.ReadCloser, error) { + return getReader(meta.Bin, vfsattr.Get(c.GetContext())) } - if meta.Dist.Shasum != "" { - tf := f - f = func() (io.ReadCloser, error) { - r, err := tf() + if meta.Hash != "" { + getreader := reader + reader = func() (io.ReadCloser, error) { + readCloser, err := getreader() if err != nil { return nil, err } - return iotools.VerifyingReaderWithHash(r, crypto.SHA1, meta.Dist.Shasum), nil + return iotools.VerifyingReaderWithHash(readCloser, meta.HashType, meta.Hash), nil } } - acc := blobaccess.DataAccessForReaderFunction(f, meta.Dist.Tarball) + acc := blobaccess.DataAccessForReaderFunction(reader, meta.Bin) return accessobj.CachedBlobAccessForWriter(c.GetContext(), mime.MIME_JAR, accessio.NewDataAccessWriter(acc)), nil } return accspeccpi.NewDefaultMethodImpl(c, a, "", mime.MIME_JAR, factory), nil } -type meta struct { - Type string `json:"type"` // pom, jar, sources, javadoc, module, ... - MD5 string `json:"md5"` - Sha1 string `json:"sha1"` - Sha256 string `json:"sha256"` - Sha512 string `json:"sha512"` - Asc string `json:"asc"` -} - -func reader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { +func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { c := &http.Client{} if strings.HasPrefix(url, "file://") { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 504a0654d6..f25a938c9c 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -10,14 +10,14 @@ import ( . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/contexts/ocm" - "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" // FIXME: Update import path + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/mime" ) -const mvnPATH = "/testdata/registry" -const FAILPATH = "/testdata/failregistry" +const mvnPATH = "/testdata/.m2/repository" +const FAILPATH = "/testdata" var _ = Describe("Method", func() { var env *Builder @@ -32,13 +32,21 @@ var _ = Describe("Method", func() { env.Cleanup() }) - It("accesses artifact", func() { - acc := mvn.New("file://"+mvnPATH, "yargs", "17.7.1") - // acc := mvn.New("https://registry.mvnjs.org", "yargs", "17.7.1") + It("get packaging", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Packaging).To(Equal("pom")) + Expect(meta.Hash).To(Equal("34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) + Expect(meta.HashType).To(Equal(crypto.SHA1)) + Expect(meta.Asc).To(ContainSubstring("-----BEGIN PGP SIGNATURE-----")) + }) + It("accesses artifact", func() { + acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") m := Must(acc.AccessMethod(cv)) defer m.Close() - Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + Expect(m.MimeType()).To(Equal(mime.MIME_JAR)) // FIXME what about POM? r := Must(m.Reader()) defer r.Close() @@ -51,15 +59,33 @@ var _ = Describe("Method", func() { } } Expect(dr.Size()).To(Equal(int64(65690))) - Expect(dr.Digest().String()).To(Equal("SHA-1:34a77645201d1a8fc5213ace787c220eabbd0967")) + Expect(dr.Digest().String()).To(Equal("SHA-1:34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) }) It("detects digests mismatch", func() { - acc := mvn.New("file://"+FAILPATH, "yargs", "17.7.1") + acc := mvn.New("file://"+FAILPATH, "fail", "repository", "42") m := Must(acc.AccessMethod(cv)) defer m.Close() _, err := m.Reader() Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found 34a77645201d1a8fc5213ace787c220eabbd0967"))) }) + + /* tests are failing with: + + <*fmt.wrapError | 0xc0008f2240>: + unable to read access: unable to create temporary file: mkdir C:\SAPDEV~1\TEMP\user: lstat /C:: CreateFile C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:: The filename, directory name, or volume label syntax is incorrect. + msg: "unable to create temporary file: mkdir C:\\SAPDEV~1\\TEMP\\user: lstat /C:: CreateFile C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:: The filename, directory name, or volume label syntax is incorrect." + err: <*fs.PathError | 0xc0008f47e0>{ + Op: "mkdir", + Path: "C:\\SAPDEV~1\\TEMP\\user", + Err: <*fs.PathError | 0xc0008f47b0>{ + Op: "lstat", + Path: "/C:", + Err: <*fs.PathError | 0xc0008f4780>{ + Op: "CreateFile", + Path: "C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:", + Err: 0x7b, + */ + }) From 403f332e51ba7cb072dca9de9a753d12077456fb Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 16:31:56 +0200 Subject: [PATCH 005/100] options: GroupOption --- pkg/contexts/ocm/accessmethods/options/standard.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/options/standard.go b/pkg/contexts/ocm/accessmethods/options/standard.go index 052f7e73e0..00a0f63e61 100644 --- a/pkg/contexts/ocm/accessmethods/options/standard.go +++ b/pkg/contexts/ocm/accessmethods/options/standard.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package options // HintOption. @@ -22,6 +18,9 @@ var ReferenceOption = RegisterOption(NewStringOptionType("reference", "reference // PackageOption. var PackageOption = RegisterOption(NewStringOptionType("accessPackage", "package or object name")) +// GroupOption. +var GroupOption = RegisterOption(NewStringOptionType("accessGroup", "GroupID or namespace")) + // RepositoryOption. var RepositoryOption = RegisterOption(NewStringOptionType("accessRepository", "repository URL")) @@ -57,7 +56,5 @@ var HTTPBodyOption = RegisterOption(NewStringOptionType("body", "body of a http var HTTPRedirectOption = RegisterOption(NewBoolOptionType("noredirect", "http redirect behavior")) -//////////////////////////////////////////////////////////////////////////////// - // CommentOption. var CommentOption = RegisterOption(NewStringOptionType("comment", "comment field value")) From 6e2020ca681bc6a1f179df3b1ddd29599d31ca22 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 16:45:07 +0200 Subject: [PATCH 006/100] tiny fail pom --- .../fail/repository/42/repository-42.pom | 208 +----------------- 1 file changed, 6 insertions(+), 202 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom index b3baaee32f..218894d775 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom @@ -1,210 +1,14 @@ - + 4.0.0 - com.sap.cloud.sdk - sdk-modules-bom - 5.7.0 + fail + repository + 42 pom - SAP Cloud SDK - Modules BOM - Bill of Materials (BOM) of the SAP Cloud SDK modules. - https://sap.github.io/cloud-sdk/docs/java/getting-started + ocm + test SAP SE https://www.sap.com - - - The Apache Software License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - SAP - cloudsdk@sap.com - SAP SE - https://www.sap.com - - - - - - - - UTF-8 - Public - Stable - - 5.7.0 - - - - - com.sap.cloud.sdk - sdk-core - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - scp-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - dwc-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-core - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - caching - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-connectivity - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-apache-httpclient4 - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-apache-httpclient5 - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-connectivity-scp-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-destination-service - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-oauth - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-dwc - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-ztis - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience-api - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience4j - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - security - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - servlet-jakarta - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - tenant - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - s4hana-core - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - s4hana-connectivity - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - rfc - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - datamodel-metadata-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-generator-utility - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-client - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-v4-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-v4-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - openapi-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - openapi-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - fluent-result - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - - soap - ${sdk.version} - - - From c6f5b720b06a543936750d2ebd4ceb87c2cd6acf Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 17:15:19 +0200 Subject: [PATCH 007/100] ReplaceAll --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index c7e92a44da..9b04ac75be 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -108,7 +108,7 @@ type meta struct { func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { // this is how the usual maven repository structure looks like - urlPrefix := a.Repository + path.Join("/", strings.Replace(a.GroupId, ".", "/", -1), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".") + urlPrefix := a.Repository + path.Join("/", strings.ReplaceAll(a.GroupId, ".", "/"), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".") fs := vfsattr.Get(ctx) // first let's read the pom file and check which binary artifact we need to read From f5cfcffd152a8d2af7e671ea0f2453bf953dfa5b Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 17:15:36 +0200 Subject: [PATCH 008/100] media type --- pkg/contexts/ocm/accessmethods/mvn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md index e077f3c2ea..778e68db78 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/README.md +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -5,7 +5,7 @@ type: mvn/v1 ``` -Provided blobs use the following media type: `application/x-tgz` // FIXME: Do we need to define a new media type for this? +Provided blobs use the following media type: `application/x-jar` ### Description From 6326df38fd944c1b0566822aec55e255763f5a32 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 17:16:20 +0200 Subject: [PATCH 009/100] MVN_ARTIFACT --- pkg/contexts/ocm/resourcetypes/const.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/contexts/ocm/resourcetypes/const.go b/pkg/contexts/ocm/resourcetypes/const.go index 287d3a3c5e..e71bab1bb1 100644 --- a/pkg/contexts/ocm/resourcetypes/const.go +++ b/pkg/contexts/ocm/resourcetypes/const.go @@ -20,6 +20,8 @@ const ( HELM_CHART = "helmChart" // NPM_PACKAGE describes an NPM package. NPM_PACKAGE = "npmPackage" + // MVN_ARTIFACT describes a MVN artifact. + MVN_ARTIFACT = "mvnArtifact" // BLUEPRINT describes a Gardener Landscaper blueprint which is an artifact used in its installations describing // how to deploy a software component. BLUEPRINT = "landscaper.gardener.cloud/blueprint" From 923cd1cb664d183947ade0f8cb494479bf0d93ef Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 17:20:14 +0200 Subject: [PATCH 010/100] docs --- pkg/contexts/ocm/accessmethods/mvn/README.md | 12 ++++++++---- pkg/contexts/ocm/accessmethods/mvn/cli.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md index 778e68db78..3bd388e8ab 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/README.md +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -21,12 +21,16 @@ The type specific specification fields are: - **`repository`** *string* - Base URL of the Maven (mvn) repository. + Base URL of the Maven (mvn) repository -- **`package`** *string* +- **`groupId`** *string* - The name of the Maven (mvn) package. + The groupId of the Maven (mvn) artifact +- +- **`artifactId`** *string* + + The artifactId of the Maven (mvn) artifact - **`version`** *string* - The version of the Maven (mvn) package. + The version name of the Maven (mvn) artifact diff --git a/pkg/contexts/ocm/accessmethods/mvn/cli.go b/pkg/contexts/ocm/accessmethods/mvn/cli.go index 56445a91f1..df859a541a 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/cli.go +++ b/pkg/contexts/ocm/accessmethods/mvn/cli.go @@ -32,7 +32,7 @@ The type specific specification fields are: - **repository** *string* - Base URL of the Maven (mvn) repository. + Base URL of the Maven (mvn) repository - **groupId** *string* From acd3c247651d94ed9acadb248af6523fe83d657e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 10 Apr 2024 17:21:48 +0200 Subject: [PATCH 011/100] fix docs list --- pkg/contexts/ocm/accessmethods/mvn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md index 3bd388e8ab..8b166fda6e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/README.md +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -26,7 +26,7 @@ The type specific specification fields are: - **`groupId`** *string* The groupId of the Maven (mvn) artifact -- + - **`artifactId`** *string* The artifactId of the Maven (mvn) artifact From 54844d3f82e269ca0b1486370d5c60f4c3a21986 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 11 Apr 2024 15:34:32 +0200 Subject: [PATCH 012/100] init mvn access --- pkg/contexts/ocm/accessmethods/init.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/init.go b/pkg/contexts/ocm/accessmethods/init.go index 11f5e759e2..f0a70ce7dd 100644 --- a/pkg/contexts/ocm/accessmethods/init.go +++ b/pkg/contexts/ocm/accessmethods/init.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package accessmethods import ( @@ -10,6 +6,7 @@ import ( _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localfsblob" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localociblob" + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/none" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" From 73c4f164bb0484d0c36c9f59a9a449e67c18b7b9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 11 Apr 2024 15:35:01 +0200 Subject: [PATCH 013/100] mvnaccess --- .../artifactaccess/mvnaccess/resource.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go diff --git a/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go b/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go new file mode 100644 index 0000000000..6c90b396a9 --- /dev/null +++ b/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go @@ -0,0 +1,30 @@ +package mvnaccess + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm" + access "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/elements/artifactaccess/genericaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" +) + +const TYPE = resourcetypes.MVN_ARTIFACT + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, repository, groupId, artifactId, version string) cpi.ArtifactAccess[M] { + if meta.GetType() == "" { + meta.SetType(TYPE) + } + + spec := access.New(repository, groupId, artifactId, version) + // is global access, must work, otherwise there is an error in the lib. + return genericaccess.MustAccess(ctx, meta, spec) +} + +func ResourceAccess(ctx ocm.Context, meta *cpi.ResourceMeta, repository, groupId, artifactId, version string) cpi.ResourceAccess { + return Access(ctx, meta, repository, groupId, artifactId, version) +} + +func SourceAccess(ctx ocm.Context, meta *cpi.SourceMeta, repository, groupId, artifactId, version string) cpi.SourceAccess { + return Access(ctx, meta, repository, groupId, artifactId, version) +} From 94fb5d7d71e861726c9e5e1ea594b64321b73eb5 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 11 Apr 2024 15:35:41 +0200 Subject: [PATCH 014/100] logging & gofumpt --- pkg/contexts/ocm/accessmethods/mvn/method.go | 16 ++++++++-------- .../ocm/accessmethods/mvn/method_test.go | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 9b04ac75be..0ed5692781 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -6,8 +6,6 @@ import ( "crypto" "encoding/xml" "fmt" - npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" - "github.com/open-component-model/ocm/pkg/logging" "io" "net/http" "path" @@ -18,10 +16,12 @@ import ( "github.com/open-component-model/ocm/pkg/blobaccess" "github.com/open-component-model/ocm/pkg/common/accessio" "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" "github.com/open-component-model/ocm/pkg/contexts/datacontext/attrs/vfsattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/runtime" ) @@ -112,8 +112,8 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { fs := vfsattr.Get(ctx) // first let's read the pom file and check which binary artifact we need to read - log := logging.Context().Logger(npmCredentials.REALM) - log.Debug("Reading %s", urlPrefix+"pom") + log := logging.Context().Logger(identity.REALM) + log.Debug("Reading ", "pom", urlPrefix+"pom") pom, err := readPom(urlPrefix+"pom", fs) if err != nil { return nil, errors.Wrapf(err, "cannot read pom file for %s", urlPrefix) @@ -121,11 +121,11 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { hashType, hash, err := getBestHashValue(urlPrefix+pom.Packaging, fs) if err != nil { - log.Debug("Could not find any hash value for %s", urlPrefix+pom.Packaging) + log.Debug("Could not find any hash value for ", "file", urlPrefix+pom.Packaging) } asc, err := getStringData(urlPrefix+pom.Packaging+".asc", fs) if err != nil { - log.Debug("No signing info found for %s", urlPrefix+pom.Packaging) + log.Debug("No signing info found for ", "file", urlPrefix+pom.Packaging) } var metadata meta = meta{ @@ -185,14 +185,14 @@ func getStringData(url string, fs vfs.FileSystem) (string, error) { // getBestHashValue returns the best hash value (SHA-512, SHA-256, SHA-1, MD5) for the given artifact (POM, JAR, ...). func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error) { arr := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} - log := logging.Context().Logger(npmCredentials.REALM) + log := logging.Context().Logger(identity.REALM) for i := 0; i < len(arr); i++ { v, err := getStringData(url+hashUrlExt(arr[i]), fs) if v != "" { return arr[i], v, err } if err != nil { - log.Debug("hash value not found: %s - %s", url+hashUrlExt(arr[i]), err) + log.Debug("hash file not found", "url", url+hashUrlExt(arr[i])) } } return 0, "", errors.New("no hash value found") diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index f25a938c9c..83aaf53454 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -16,8 +16,10 @@ import ( "github.com/open-component-model/ocm/pkg/mime" ) -const mvnPATH = "/testdata/.m2/repository" -const FAILPATH = "/testdata" +const ( + mvnPATH = "/testdata/.m2/repository" + FAILPATH = "/testdata" +) var _ = Describe("Method", func() { var env *Builder @@ -87,5 +89,4 @@ var _ = Describe("Method", func() { Path: "C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:", Err: 0x7b, */ - }) From 57fea18fb84572c9e3e674df0fb5f92a24191da7 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 11 Apr 2024 15:36:54 +0200 Subject: [PATCH 015/100] copy & paste npm-identity --- .../builtin/mvn/identity/identity.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 pkg/contexts/credentials/builtin/mvn/identity/identity.go diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go new file mode 100644 index 0000000000..1c67d3a0da --- /dev/null +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -0,0 +1,69 @@ +package identity + +import ( + "path" + + . "net/url" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/logging" +) + +const ( + // CONSUMER_TYPE is the mvn repository type. + CONSUMER_TYPE = "Repository.maven.apache.org" + + // FIXME use correct settings for maven repo authentication + + // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. + ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. + ATTR_PASSWORD = cpi.ATTR_PASSWORD + // ATTR_EMAIL is the email attribute. Required for login at any mvn registry. + ATTR_EMAIL = cpi.ATTR_EMAIL + // ATTR_TOKEN is the token attribute. May exist after login at any mvn registry. + ATTR_TOKEN = cpi.ATTR_TOKEN +) + +// Logging Realm. +var REALM = logging.DefineSubRealm("MVN repository", "MVN") + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_EMAIL, "MVN registry, require an email address", + ATTR_TOKEN, "the token attribute. May exist after login at any mvn registry. Check your .m2/settings.xml file!", + }) + + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `MVN repository + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { + url, err := Parse(rawURL) + if err != nil { + return nil + } + + url.Path = path.Join(url.Path, pkgName) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) +} + +func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { + id := GetConsumerId(repoUrl, pkgName) + if id == nil { + return nil + } + credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) + if credentials == nil || err != nil { + return nil + } + return credentials.Properties() +} From 1f4554494cf17699b2c9b35af03031d5b54e07eb Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 12:25:01 +0200 Subject: [PATCH 016/100] DRAFT mvn identity --- .../builtin/mvn/identity/identity.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 1c67d3a0da..1e8d1cb606 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -16,16 +16,12 @@ const ( // CONSUMER_TYPE is the mvn repository type. CONSUMER_TYPE = "Repository.maven.apache.org" - // FIXME use correct settings for maven repo authentication - + // FIXME use correct settings for maven repo authentication + // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. ATTR_USERNAME = cpi.ATTR_USERNAME // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. ATTR_PASSWORD = cpi.ATTR_PASSWORD - // ATTR_EMAIL is the email attribute. Required for login at any mvn registry. - ATTR_EMAIL = cpi.ATTR_EMAIL - // ATTR_TOKEN is the token attribute. May exist after login at any mvn registry. - ATTR_TOKEN = cpi.ATTR_TOKEN ) // Logging Realm. @@ -35,8 +31,6 @@ func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ ATTR_USERNAME, "the basic auth user name", ATTR_PASSWORD, "the basic auth password", - ATTR_EMAIL, "MVN registry, require an email address", - ATTR_TOKEN, "the token attribute. May exist after login at any mvn registry. Check your .m2/settings.xml file!", }) cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `MVN repository @@ -46,18 +40,18 @@ the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { +func GetConsumerId(rawURL, groupId string) cpi.ConsumerIdentity { url, err := Parse(rawURL) if err != nil { return nil } - url.Path = path.Join(url.Path, pkgName) + url.Path = path.Join(url.Path, groupId) return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) } -func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { - id := GetConsumerId(repoUrl, pkgName) +func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Properties { + id := GetConsumerId(repoUrl, groupId) if id == nil { return nil } From b1f4ddc222292360c264b05ae2b10555ca71b180 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 12:25:31 +0200 Subject: [PATCH 017/100] log success --- pkg/contexts/ocm/accessmethods/mvn/method.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 0ed5692781..d59ff4cd1f 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -189,6 +189,7 @@ func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error for i := 0; i < len(arr); i++ { v, err := getStringData(url+hashUrlExt(arr[i]), fs) if v != "" { + log.Debug("found hash ", "url", url+hashUrlExt(arr[i])) return arr[i], v, err } if err != nil { From 43c680400bdfcc03dd5392f5a632ac314a3d3f32 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 12:25:59 +0200 Subject: [PATCH 018/100] typos --- pkg/contexts/ocm/internal/blobhandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/internal/blobhandler.go b/pkg/contexts/ocm/internal/blobhandler.go index d655f6743a..4e1ad25f84 100644 --- a/pkg/contexts/ocm/internal/blobhandler.go +++ b/pkg/contexts/ocm/internal/blobhandler.go @@ -37,9 +37,9 @@ type StorageContext interface { GetImplementationRepositoryType() ImplementationRepositoryType } -// BlobHandler s the interface for a dedicated handling of storing blobs +// BlobHandler is the interface for a dedicated handling of storing blobs // for the LocalBlob access method in a dedicated kind of repository. -// with the possibility of access by an external distribution spec. +// With the possibility of access by an external distribution spec // (besides of the blob storage as part of a component version). // The technical repository to use should be derivable from the chosen // component directory or passed together with the storage context. From 3f49aeb5032bb62820a92f6a5dd223836285deea Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 12:29:09 +0200 Subject: [PATCH 019/100] FIXME: hardcoded coordinates!!! --- .../handlers/generic/mvn/artifact.go | 47 ++++++++ .../handlers/generic/mvn/artifact_test.go | 53 ++++++++ .../handlers/generic/mvn/blobhandler.go | 114 ++++++++++++++++++ .../handlers/generic/mvn/registration.go | 74 ++++++++++++ .../handlers/generic/mvn/registration_test.go | 23 ++++ .../handlers/generic/mvn/suite_test.go | 13 ++ 6 files changed, 324 insertions(+) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go new file mode 100644 index 0000000000..939114bcf3 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go @@ -0,0 +1,47 @@ +package mvn + +import "strings" + +// Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. +type Artifact struct { + GroupId string `xml:"groupId"` + ArtifactId string `xml:"artifactId"` + Version string `xml:"version"` + Packaging string `xml:"packaging"` +} + +// GAV returns the GAV coordinates of the Maven Artifact. +func (a *Artifact) GAV() string { + return a.GroupId + ":" + a.ArtifactId + ":" + a.Version +} + +// Path returns the Maven Artifact's path within a repository. +func (a *Artifact) Path() string { + return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version + "/" + a.ArtifactId + "-" + a.Version + "." + a.Packaging +} + +// GroupPath returns GroupId with `/` instead of `.`. +func (a *Artifact) GroupPath() string { + return strings.ReplaceAll(a.GroupId, ".", "/") +} + +// Purl returns the Package URL of the Maven Artifact. +func (a *Artifact) Purl() string { + return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version +} + +// Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). +type Body struct { + Repo string `json:"repo"` + Path string `json:"path"` + DownloadUri string `json:"downloadUri"` + Uri string `json:"uri"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + Checksums struct { + Md5 string `json:"md5"` + Sha1 string `json:"sha1"` + Sha256 string `json:"sha256"` + Sha512 string `json:"sha512"` + } `json:"checksums"` +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go new file mode 100644 index 0000000000..646f785bbc --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go @@ -0,0 +1,53 @@ +package mvn + +import ( + "encoding/json" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Maven Test Environment", func() { + + It("GAV, GroupPath, Path", func() { + artifact := &Artifact{ + GroupId: "ocm.software", + ArtifactId: "hello-ocm", + Version: "0.0.1", + Packaging: "jar", + } + Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) + Expect(artifact.GroupPath()).To(Equal("ocm/software")) + Expect(artifact.Path()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + }) + + It("Body", func() { + resp := `{ "repo" : "ocm-mvn-test", + "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "created" : "2024-04-11T15:09:28.920Z", + "createdBy" : "d057539", + "downloadUri" : "https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "mimeType" : "application/java-archive", + "size" : "1792", + "checksums" : { + "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", + "md5" : "6cb7520b65d820b3b35773a8daa8368e", + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "originalChecksums" : { + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "uri" : "https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` + var body Body + err := json.Unmarshal([]byte(resp), &body) + Expect(err).To(BeNil()) + Expect(body.Repo).To(Equal("ocm-mvn-test")) + Expect(body.Path).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.DownloadUri).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.Uri).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.MimeType).To(Equal("application/java-archive")) + Expect(body.Size).To(Equal("1792")) + Expect(body.Checksums.Md5).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) + Expect(body.Checksums.Sha1).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) + Expect(body.Checksums.Sha256).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) + Expect(body.Checksums.Sha512).To(Equal("")) + }) + +}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go new file mode 100644 index 0000000000..61be17791f --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -0,0 +1,114 @@ +package mvn + +import ( + "encoding/json" + "fmt" + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/logging" + "github.com/open-component-model/ocm/pkg/mime" + "io" + "net/http" +) + +const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.MVN_ARTIFACT + +type artifactHandler struct { + spec *Config +} + +func NewArtifactHandler(repospec *Config) cpi.BlobHandler { + return &artifactHandler{repospec} +} + +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { + // check conditions + if b.spec == nil { + return nil, nil + } + mimeType := blob.MimeType() + if mime.MIME_JAR != mimeType { + return nil, nil + } + if b.spec.Url == "" { + return nil, fmt.Errorf("MVN repository url not provided") + } + + // setup logger + log := logging.Context().Logger(mvn.REALM) + log = log.WithValues("repository", b.spec.Url) + + blobReader, err := blob.Reader() + if err != nil { + return nil, err + } + defer blobReader.Close() + + /*/ FIXME: do we need to read the whole file into memory? Maybe to guess the groupId, artifactId and version? + data, err := io.ReadAll(blobReader) + if err != nil { + return nil, err + } */ + var artifact *Artifact + // FIXME: how do I get the groupId, artifactId and version from accessSpec.AccessMethod().??? + log.Debug("reading jar file - but, where is the groupId, artifactId and version?") + artifact = &Artifact{ + GroupId: "ocm.software", // FIXME: hardcoded + ArtifactId: "hello-ocm", // FIXME: hardcoded + Version: "0.0.1", // FIXME: hardcoded + Packaging: "jar", // FIXME: hardcoded + } + log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) + log.Debug("identified") + + // get credentials + cred := identity.GetCredentials(ctx.GetContext(), b.spec.Url, artifact.GroupPath()) + if cred == nil { + return nil, fmt.Errorf("no credentials found for %s. Couldn't upload '%s'", b.spec.Url, artifact.GAV()) + } + log.Debug("found credentials") + + username := cred[identity.ATTR_USERNAME] + password := cred[identity.ATTR_PASSWORD] + if username == "" || password == "" { + return nil, fmt.Errorf("credentials for %s are invalid. Username or password missing! Couldn't upload '%s'", b.spec.Url, artifact.GAV()) + } + log = log.WithValues("user", username) + + // Create a new request + req, err := http.NewRequest("PUT", b.spec.Url+"/"+artifact.Path(), blobReader) + if err != nil { + panic(err) + } + + // Add authentication + req.SetBasicAuth(username, password) + + // Execute the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + // Check the response + if resp.StatusCode != http.StatusCreated { + all, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) + } + + respBody, err := io.ReadAll(resp.Body) + err = json.Unmarshal(respBody, &artifact) + if err != nil { + return nil, err + } + + log.Debug("successfully uploaded") + return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version), nil +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go new file mode 100644 index 0000000000..e0b7ebfe9b --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go @@ -0,0 +1,74 @@ +package mvn + +import ( + "encoding/json" + "fmt" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/registrations" +) + +type Config struct { + Url string `json:"url"` +} + +type rawConfig Config + +func (c *Config) UnmarshalJSON(data []byte) error { + err := json.Unmarshal(data, &c.Url) + if err == nil { + return nil + } + var raw rawConfig + err = json.Unmarshal(data, &raw) + if err != nil { + return err + } + *c = Config(raw) + + return nil +} + +func init() { + cpi.RegisterBlobHandlerRegistrationHandler(BLOB_HANDLER_NAME, &RegistrationHandler{}) +} + +type RegistrationHandler struct{} + +var _ cpi.BlobHandlerRegistrationHandler = (*RegistrationHandler)(nil) + +func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, config cpi.BlobHandlerConfig, olist ...cpi.BlobHandlerOption) (bool, error) { + if handler != "" { + return true, fmt.Errorf("invalid %s handler %q", resourcetypes.MVN_ARTIFACT, handler) + } + if config == nil { + return true, fmt.Errorf("mvn target specification required") + } + cfg, err := registrations.DecodeConfig[Config](config) + if err != nil { + return true, errors.Wrapf(err, "blob handler configuration") + } + + ctx.BlobHandlers().Register(NewArtifactHandler(cfg), + cpi.ForArtifactType(resourcetypes.MVN_ARTIFACT), + cpi.ForMimeType(mime.MIME_JAR), + cpi.NewBlobHandlerOptions(olist...), + ) + + return true, nil +} + +func (r *RegistrationHandler) GetHandlers(_ cpi.Context) registrations.HandlerInfos { + return registrations.NewLeafHandlerInfo("uploading mvn artifacts", ` +The `+BLOB_HANDLER_NAME+` uploader is able to upload mvn artifacts +as artifact archive according to the mvn artifact spec. +If registered the default mime type is: `+mime.MIME_JAR+` + +It accepts a plain string for the URL or a config with the following field: +'url': the URL of the mvn repository. +`, + ) +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go new file mode 100644 index 0000000000..c64b3e13bb --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go @@ -0,0 +1,23 @@ +package mvn_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" + "github.com/open-component-model/ocm/pkg/registrations" +) + +var _ = Describe("Config deserialization Test Environment", func() { + + It("deserializes string", func() { + cfg := Must(registrations.DecodeConfig[mvn.Config]("test")) + Expect(cfg).To(Equal(&mvn.Config{Url: "test"})) + }) + + It("deserializes struct", func() { + cfg := Must(registrations.DecodeConfig[mvn.Config](`{"Url":"test"}`)) + Expect(cfg).To(Equal(&mvn.Config{Url: "test"})) + }) +}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go new file mode 100644 index 0000000000..c62cadc7b0 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go @@ -0,0 +1,13 @@ +package mvn_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Maven (mvn) Test Suite") +} From 65d53c12ed5f335cd96eee5c8d571741a0cea289 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 12:29:32 +0200 Subject: [PATCH 020/100] init mvn handler --- pkg/contexts/ocm/blobhandler/handlers/init.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/init.go b/pkg/contexts/ocm/blobhandler/handlers/init.go index 433e3338be..b318fe2c9e 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/init.go +++ b/pkg/contexts/ocm/blobhandler/handlers/init.go @@ -1,10 +1,7 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package handlers import ( + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo" From 74ad2ac85b26770b2bc767ccb93f1ee3af167903 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 15:04:27 +0200 Subject: [PATCH 021/100] wrong REALM --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index d59ff4cd1f..aa2cc65542 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -37,8 +37,6 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } -var REALM = logging.DefineSubRealm("Maven repository", "mvn") - // AccessSpec describes the access for a Maven (mvn) repository. type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` From e2762ec30a6ca0c7ccec719fe49c829505bf5d34 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:03:31 +0200 Subject: [PATCH 022/100] verify upload response digest --- .../handlers/generic/mvn/artifact.go | 33 +++++++---- .../handlers/generic/mvn/artifact_test.go | 19 ++++--- .../handlers/generic/mvn/blobhandler.go | 57 +++++++++---------- 3 files changed, 57 insertions(+), 52 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go index 939114bcf3..c82b70d706 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go @@ -30,18 +30,27 @@ func (a *Artifact) Purl() string { return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version } +// FromGAV creates new Artifact from GAV coordinates. +func FromGAV(gav string) *Artifact { + parts := strings.Split(gav, ":") + if len(parts) != 3 { + return nil + } + return &Artifact{ + GroupId: parts[0], + ArtifactId: parts[1], + Version: parts[2], + Packaging: "jar", + } +} + // Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). type Body struct { - Repo string `json:"repo"` - Path string `json:"path"` - DownloadUri string `json:"downloadUri"` - Uri string `json:"uri"` - MimeType string `json:"mimeType"` - Size string `json:"size"` - Checksums struct { - Md5 string `json:"md5"` - Sha1 string `json:"sha1"` - Sha256 string `json:"sha256"` - Sha512 string `json:"sha512"` - } `json:"checksums"` + Repo string `json:"repo"` + Path string `json:"path"` + DownloadUri string `json:"downloadUri"` + Uri string `json:"uri"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + Checksums map[string]string `json:"checksums"` } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go index 646f785bbc..2e6e233a10 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go @@ -2,6 +2,7 @@ package mvn import ( "encoding/json" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -24,8 +25,8 @@ var _ = Describe("Maven Test Environment", func() { resp := `{ "repo" : "ocm-mvn-test", "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", "created" : "2024-04-11T15:09:28.920Z", - "createdBy" : "d057539", - "downloadUri" : "https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "createdBy" : "john.doe", + "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", "mimeType" : "application/java-archive", "size" : "1792", "checksums" : { @@ -34,20 +35,20 @@ var _ = Describe("Maven Test Environment", func() { "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, "originalChecksums" : { "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, - "uri" : "https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` + "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` var body Body err := json.Unmarshal([]byte(resp), &body) Expect(err).To(BeNil()) Expect(body.Repo).To(Equal("ocm-mvn-test")) Expect(body.Path).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.DownloadUri).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.Uri).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.MimeType).To(Equal("application/java-archive")) Expect(body.Size).To(Equal("1792")) - Expect(body.Checksums.Md5).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) - Expect(body.Checksums.Sha1).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) - Expect(body.Checksums.Sha256).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) - Expect(body.Checksums.Sha512).To(Equal("")) + Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) + Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) + Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) + Expect(body.Checksums["sha512"]).To(Equal("")) }) }) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 61be17791f..bc789ba1e6 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -3,14 +3,15 @@ package mvn import ( "encoding/json" "fmt" + "io" + "net/http" + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" - "io" - "net/http" ) const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.MVN_ARTIFACT @@ -23,7 +24,7 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { return &artifactHandler{repospec} } -func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { // check conditions if b.spec == nil { return nil, nil @@ -37,29 +38,11 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c } // setup logger - log := logging.Context().Logger(mvn.REALM) + log := logging.Context().Logger(identity.REALM) log = log.WithValues("repository", b.spec.Url) - blobReader, err := blob.Reader() - if err != nil { - return nil, err - } - defer blobReader.Close() - - /*/ FIXME: do we need to read the whole file into memory? Maybe to guess the groupId, artifactId and version? - data, err := io.ReadAll(blobReader) - if err != nil { - return nil, err - } */ - var artifact *Artifact - // FIXME: how do I get the groupId, artifactId and version from accessSpec.AccessMethod().??? - log.Debug("reading jar file - but, where is the groupId, artifactId and version?") - artifact = &Artifact{ - GroupId: "ocm.software", // FIXME: hardcoded - ArtifactId: "hello-ocm", // FIXME: hardcoded - Version: "0.0.1", // FIXME: hardcoded - Packaging: "jar", // FIXME: hardcoded - } + // identify artifact + artifact := FromGAV(hint) log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) log.Debug("identified") @@ -68,29 +51,31 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c if cred == nil { return nil, fmt.Errorf("no credentials found for %s. Couldn't upload '%s'", b.spec.Url, artifact.GAV()) } - log.Debug("found credentials") - username := cred[identity.ATTR_USERNAME] password := cred[identity.ATTR_PASSWORD] if username == "" || password == "" { return nil, fmt.Errorf("credentials for %s are invalid. Username or password missing! Couldn't upload '%s'", b.spec.Url, artifact.GAV()) } log = log.WithValues("user", username) + log.Debug("found credentials") // Create a new request + blobReader, err := blob.Reader() + if err != nil { + return nil, err + } + defer blobReader.Close() req, err := http.NewRequest("PUT", b.spec.Url+"/"+artifact.Path(), blobReader) if err != nil { - panic(err) + return nil, err } - - // Add authentication req.SetBasicAuth(username, password) // Execute the request client := &http.Client{} resp, err := client.Do(req) if err != nil { - panic(err) + return nil, err } defer resp.Body.Close() @@ -103,12 +88,22 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c return nil, fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) } + // Validate the response - especially the hash values with the ones we've tried to send respBody, err := io.ReadAll(resp.Body) - err = json.Unmarshal(respBody, &artifact) + var artifactBody Body + err = json.Unmarshal(respBody, &artifactBody) if err != nil { return nil, err } + blobDigest := blob.Digest() + remoteDigest := artifactBody.Checksums[string(blobDigest.Algorithm())] + if remoteDigest == "" { + log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", blobDigest.Algorithm()) + } else if remoteDigest != blobDigest.Encoded() { + return nil, fmt.Errorf("failed to upload artifact: checksums do not match") + } + log.Debug("successfully uploaded") return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version), nil } From 1e313d0e793849ade1956f3a749bb5f5ed75915e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:04:36 +0200 Subject: [PATCH 023/100] use resourcetypes const --- .../ocm/blobhandler/handlers/generic/npm/blobhandler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 709141fd4d..fc1c78168e 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -12,11 +12,12 @@ import ( npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" ) -const BLOB_HANDLER_NAME = "ocm/npmPackage" +const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.NPM_PACKAGE type artifactHandler struct { spec *Config From 4d56ee3ce8093532f46f48593a9cc90f0ea07372 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:05:14 +0200 Subject: [PATCH 024/100] fix doc --- pkg/contexts/ocm/resourcetypes/const.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/resourcetypes/const.go b/pkg/contexts/ocm/resourcetypes/const.go index e71bab1bb1..eac1162aa3 100644 --- a/pkg/contexts/ocm/resourcetypes/const.go +++ b/pkg/contexts/ocm/resourcetypes/const.go @@ -28,8 +28,9 @@ const ( BLUEPRINT_LEGACY = "blueprint" // BLOB describes any anonymous untyped blob data. BLOB = "blob" + // DIRECTORY_TREE describes a directory structure. + DIRECTORY_TREE = "directoryTree" // FILESYSTEM describes a directory structure stored as archive (tar, tgz). - DIRECTORY_TREE = "directoryTree" FILESYSTEM = DIRECTORY_TREE FILESYSTEM_LEGACY = "filesystem" // EXECUTABLE describes an OS executable. From 9f20db210db5c274e2d4ead88c49e909432b53b9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:05:56 +0200 Subject: [PATCH 025/100] lower case REALM --- pkg/contexts/credentials/builtin/mvn/identity/identity.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 1e8d1cb606..a3bffeeaf4 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -16,8 +16,6 @@ const ( // CONSUMER_TYPE is the mvn repository type. CONSUMER_TYPE = "Repository.maven.apache.org" - // FIXME use correct settings for maven repo authentication - // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. ATTR_USERNAME = cpi.ATTR_USERNAME // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. @@ -25,7 +23,7 @@ const ( ) // Logging Realm. -var REALM = logging.DefineSubRealm("MVN repository", "MVN") +var REALM = logging.DefineSubRealm("Maven repository", "mvn") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ From 25ea322e1e83bfb2c33c3bf8e30c104ff1ee7364 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:36:37 +0200 Subject: [PATCH 026/100] fix test, doc --- pkg/contexts/ocm/accessmethods/mvn/method_test.go | 8 ++++---- .../fail/repository/42/repository-42.pom.sha1 | 14 ++++++++++++++ .../handlers/generic/mvn/blobhandler.go | 1 - pkg/contexts/ocm/resourcetypes/const.go | 4 ++-- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 83aaf53454..a761509917 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -5,15 +5,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/open-component-model/ocm/pkg/env" - . "github.com/open-component-model/ocm/pkg/env/builder" - . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + . "github.com/open-component-model/ocm/pkg/env" + . "github.com/open-component-model/ocm/pkg/env/builder" "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/mime" + . "github.com/open-component-model/ocm/pkg/testutils" ) const ( @@ -60,7 +60,7 @@ var _ = Describe("Method", func() { break } } - Expect(dr.Size()).To(Equal(int64(65690))) + Expect(dr.Size()).To(Equal(int64(7153))) Expect(dr.Digest().String()).To(Equal("SHA-1:34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) }) diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 new file mode 100644 index 0000000000..218894d775 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 @@ -0,0 +1,14 @@ + + + 4.0.0 + fail + repository + 42 + pom + ocm + test + + SAP SE + https://www.sap.com + + diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index bc789ba1e6..4de8358572 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -95,7 +95,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, if err != nil { return nil, err } - blobDigest := blob.Digest() remoteDigest := artifactBody.Checksums[string(blobDigest.Algorithm())] if remoteDigest == "" { diff --git a/pkg/contexts/ocm/resourcetypes/const.go b/pkg/contexts/ocm/resourcetypes/const.go index eac1162aa3..f7fdcae3a2 100644 --- a/pkg/contexts/ocm/resourcetypes/const.go +++ b/pkg/contexts/ocm/resourcetypes/const.go @@ -18,9 +18,9 @@ const ( // HELM_CHART describes a helm chart, either stored as OCI artifact or as tar // blob (tar media type). HELM_CHART = "helmChart" - // NPM_PACKAGE describes an NPM package. + // NPM_PACKAGE describes a Node.js (npm) package. NPM_PACKAGE = "npmPackage" - // MVN_ARTIFACT describes a MVN artifact. + // MVN_ARTIFACT describes a Maven artifact (jar). MVN_ARTIFACT = "mvnArtifact" // BLUEPRINT describes a Gardener Landscaper blueprint which is an artifact used in its installations describing // how to deploy a software component. From 5c9ef9e42a4fe0d6b0d9096a24ec7c0165bcb437 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 12 Apr 2024 16:51:34 +0200 Subject: [PATCH 027/100] fix linter issues --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 4de8358572..3a0f787ca5 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -1,6 +1,7 @@ package mvn import ( + "context" "encoding/json" "fmt" "io" @@ -65,7 +66,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, return nil, err } defer blobReader.Close() - req, err := http.NewRequest("PUT", b.spec.Url+"/"+artifact.Path(), blobReader) + req, err := http.NewRequestWithContext(context.Background(), "PUT", b.spec.Url+"/"+artifact.Path(), blobReader) if err != nil { return nil, err } @@ -90,6 +91,9 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, // Validate the response - especially the hash values with the ones we've tried to send respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } var artifactBody Body err = json.Unmarshal(respBody, &artifactBody) if err != nil { From 78bd2da8d800307e8deedfc4944587912df2dc6d Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 15 Apr 2024 15:16:25 +0200 Subject: [PATCH 028/100] fix linter issue --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 3a0f787ca5..0bfc93429b 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -66,7 +66,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, return nil, err } defer blobReader.Close() - req, err := http.NewRequestWithContext(context.Background(), "PUT", b.spec.Url+"/"+artifact.Path(), blobReader) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, b.spec.Url+"/"+artifact.Path(), blobReader) if err != nil { return nil, err } From c4bbe90f87951b2b52ef08a681b51cf88e573f68 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 15 Apr 2024 15:19:56 +0200 Subject: [PATCH 029/100] fix test --- .../ocm/accessmethods/mvn/method_test.go | 19 +------------------ .../fail/repository/42/repository-42.pom.sha1 | 15 +-------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index a761509917..9a1a0dea61 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -70,23 +70,6 @@ var _ = Describe("Method", func() { m := Must(acc.AccessMethod(cv)) defer m.Close() _, err := m.Reader() - Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found 34a77645201d1a8fc5213ace787c220eabbd0967"))) + Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found b3242b8c31f8ce14f729b8fd132ac77bc4bc5bf7"))) }) - - /* tests are failing with: - - <*fmt.wrapError | 0xc0008f2240>: - unable to read access: unable to create temporary file: mkdir C:\SAPDEV~1\TEMP\user: lstat /C:: CreateFile C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:: The filename, directory name, or volume label syntax is incorrect. - msg: "unable to create temporary file: mkdir C:\\SAPDEV~1\\TEMP\\user: lstat /C:: CreateFile C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:: The filename, directory name, or volume label syntax is incorrect." - err: <*fs.PathError | 0xc0008f47e0>{ - Op: "mkdir", - Path: "C:\\SAPDEV~1\\TEMP\\user", - Err: <*fs.PathError | 0xc0008f47b0>{ - Op: "lstat", - Path: "/C:", - Err: <*fs.PathError | 0xc0008f4780>{ - Op: "CreateFile", - Path: "C:/SAPDEV~1/TEMP/user/VFS-1551776656/C:", - Err: 0x7b, - */ }) diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 index 218894d775..3d6c52fe9e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 @@ -1,14 +1 @@ - - - 4.0.0 - fail - repository - 42 - pom - ocm - test - - SAP SE - https://www.sap.com - - +44a77645201d1a8fc5213ace787c220eabbd0967 \ No newline at end of file From 88c40ae70e8656ee6eb9a133b0f0adf8f5c5171c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 18 Apr 2024 15:09:50 +0200 Subject: [PATCH 030/100] intermediate state --- .../ocm/accessmethods/mvn/artifact.go | 110 +++++++++++ .../ocm/accessmethods/mvn/artifact_test.go | 85 +++++++++ pkg/contexts/ocm/accessmethods/mvn/method.go | 177 +++++++++++++++--- .../ocm/accessmethods/mvn/method_test.go | 36 ++++ .../handlers/generic/mvn/artifact.go | 56 ------ .../handlers/generic/mvn/artifact_test.go | 54 ------ .../handlers/generic/mvn/blobhandler.go | 42 ++++- pkg/utils/tarutils/pack.go | 79 +++++++- 8 files changed, 490 insertions(+), 149 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/artifact.go create mode 100644 pkg/contexts/ocm/accessmethods/mvn/artifact_test.go delete mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go delete mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go new file mode 100644 index 0000000000..575b23fc13 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -0,0 +1,110 @@ +package mvn + +import ( + "strings" +) + +// Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. +// https://maven.apache.org/ref/3.9.6/maven-core/artifact-handlers.html +type Artifact struct { + GroupId string + ArtifactId string + Version string + Classifier string + Extension string + // Type string + // Packaging string +} + +// GAV returns the GAV coordinates of the Maven Artifact. +func (a *Artifact) GAV() string { + return a.GroupId + ":" + a.ArtifactId + ":" + a.Version +} + +// String returns the GAV coordinates of the Maven Artifact. +func (a *Artifact) String() string { + return a.GAV() +} + +// GavPath returns the Maven repository path. +func (a *Artifact) GavPath() string { + return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version +} + +// Path returns the Maven Artifact's path with classifier and extension. +func (a *Artifact) Path() string { + path := a.GavPath() + "/" + a.FilePrefix() + if a.Classifier != "" { + path += "-" + a.Classifier + } + if a.Extension != "" { + path += "." + a.Extension + } else { + path += ".jar" + } + return path +} + +// GroupPath returns GroupId with `/` instead of `.`. +func (a *Artifact) GroupPath() string { + return strings.ReplaceAll(a.GroupId, ".", "/") +} + +func (a *Artifact) FilePrefix() string { + return a.ArtifactId + "-" + a.Version +} + +// Purl returns the Package URL of the Maven Artifact. +func (a *Artifact) Purl() string { + return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version +} + +func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { + s := strings.TrimPrefix(filename, a.FilePrefix()) + if strings.HasPrefix(s, "-") { + s = strings.TrimPrefix(s, "-") + a.Classifier = s[:strings.Index(s, ".")] + s = strings.TrimPrefix(s, a.Classifier) + } + a.Extension = strings.TrimPrefix(s, ".") + return a +} + +// ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. +func ArtifactFromHint(gav string) *Artifact { + parts := strings.Split(gav, ":") + if len(parts) < 3 { + return nil + } + artifact := &Artifact{ + GroupId: parts[0], + ArtifactId: parts[1], + Version: parts[2], + } + if len(parts) >= 4 { + artifact.Classifier = parts[3] + } + if len(parts) >= 5 { + artifact.Extension = parts[4] + } + return artifact +} + +func IsArtifact(fileName string) bool { + if strings.HasSuffix(fileName, ".asc") { + return false + } + if strings.HasSuffix(fileName, ".md5") { + return false + } + if strings.HasSuffix(fileName, ".sha1") { + return false + } + if strings.HasSuffix(fileName, ".sha256") { + return false + } + if strings.HasSuffix(fileName, ".sha512") { + return false + } + return true +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go new file mode 100644 index 0000000000..3787c1f7d1 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go @@ -0,0 +1,85 @@ +package mvn + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Maven Test Environment", func() { + + It("GAV, GroupPath, Path", func() { + artifact := &Artifact{ + GroupId: "ocm.software", + ArtifactId: "hello-ocm", + Version: "0.0.1", + Extension: "jar", + } + Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) + Expect(artifact.GroupPath()).To(Equal("ocm/software")) + Expect(artifact.Path()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + }) + + It("ClassifierExtensionFrom", func() { + artifact := &Artifact{ + GroupId: "ocm.software", + ArtifactId: "hello-ocm", + Version: "0.0.1", + } + artifact.ClassifierExtensionFrom("hello-ocm-0.0.1.pom") + Expect(artifact.Classifier).To(Equal("")) + Expect(artifact.Extension).To(Equal("pom")) + + artifact.ClassifierExtensionFrom("hello-ocm-0.0.1-tests.jar") + Expect(artifact.Classifier).To(Equal("tests")) + Expect(artifact.Extension).To(Equal("jar")) + + artifact.ArtifactId = "apache-maven" + artifact.Version = "3.9.6" + artifact.ClassifierExtensionFrom("apache-maven-3.9.6-bin.tar.gz") + Expect(artifact.Classifier).To(Equal("bin")) + Expect(artifact.Extension).To(Equal("tar.gz")) + }) + + It("parse GAV", func() { + gav := "org.apache.commons:commons-compress:1.26.1:cyclonedx:xml" + artifact := ArtifactFromHint(gav) + Expect(artifact.GroupId).To(Equal("org.apache.commons")) + Expect(artifact.ArtifactId).To(Equal("commons-compress")) + Expect(artifact.Version).To(Equal("1.26.1")) + Expect(artifact.Classifier).To(Equal("cyclonedx")) + Expect(artifact.Extension).To(Equal("xml")) + Expect(artifact.Path()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) + }) + + /* + It("Body", func() { + resp := `{ "repo" : "ocm-mvn-test", + "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "created" : "2024-04-11T15:09:28.920Z", + "createdBy" : "john.doe", + "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "mimeType" : "application/java-archive", + "size" : "1792", + "checksums" : { + "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", + "md5" : "6cb7520b65d820b3b35773a8daa8368e", + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "originalChecksums" : { + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` + var body Body + err := json.Unmarshal([]byte(resp), &body) + Expect(err).To(BeNil()) + Expect(body.Repo).To(Equal("ocm-mvn-test")) + Expect(body.Path).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.MimeType).To(Equal("application/java-archive")) + Expect(body.Size).To(Equal("1792")) + Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) + Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) + Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) + Expect(body.Checksums["sha512"]).To(Equal("")) + }) + */ +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index aa2cc65542..d8539217a4 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -4,14 +4,16 @@ import ( "bytes" "context" "crypto" - "encoding/xml" "fmt" "io" "net/http" "path" + "sort" "strings" "github.com/mandelsoft/vfs/pkg/vfs" + "golang.org/x/exp/slices" + "golang.org/x/net/html" "github.com/open-component-model/ocm/pkg/blobaccess" "github.com/open-component-model/ocm/pkg/common/accessio" @@ -37,30 +39,54 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } -// AccessSpec describes the access for a Maven (mvn) repository. +// AccessSpec describes the access for a Maven (mvn) artifact. type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` - // Repository is the base URL of the Maven (mvn) repository + // Repository is the base URL of the Maven (mvn) repository. Repository string `json:"repository"` - // ArtifactId is the name of Maven (mvn) package + // ArtifactId is the name of Maven (mvn) artifact. GroupId string `json:"groupId"` - // ArtifactId is the name of Maven (mvn) package + // ArtifactId is the name of Maven (mvn) artifact. ArtifactId string `json:"artifactId"` - // Version of the Maven (mvn) package. + // Version of the Maven (mvn) artifact. Version string `json:"version"` + // Classifier of the Maven (mvn) artifact. + Classifier string `json:"classifier"` + // Extension of the Maven (mvn) artifact. + Extension string `json:"extension"` } var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) // New creates a new Maven (mvn) repository access spec version v1. -func New(repository, groupId, artifactId, version string) *AccessSpec { - return &AccessSpec{ +func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { + accessSpec := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), Repository: repository, GroupId: groupId, ArtifactId: artifactId, Version: version, + Classifier: "", + Extension: "jar", + } + for _, option := range options { + option(accessSpec) + } + return accessSpec +} + +// WithClassifier sets the classifier of the Maven (mvn) artifact. +func WithClassifier(classifier string) func(*AccessSpec) { + return func(a *AccessSpec) { + a.Classifier = classifier + } +} + +// WithExtension sets the extension of the Maven (mvn) artifact. +func WithExtension(extension string) func(*AccessSpec) { + return func(a *AccessSpec) { + a.Extension = extension } } @@ -76,8 +102,10 @@ func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpe return a } +// GetReferenceHint returns the reference hint for the Maven (mvn) artifact. In the following form: +// groupId:artifactId:version:classifier:extension func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { - return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + ":" + a.Classifier + ":" + a.Extension } func (_ *AccessSpec) GetType() string { @@ -96,6 +124,24 @@ func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.Comp return "" } +func (a *AccessSpec) BaseUrl() string { + return a.Repository + "/" + a.AsArtifact().GavPath() +} + +func (a *AccessSpec) ArtifactUrl() string { + return a.Repository + a.AsArtifact().Path() +} + +func (a *AccessSpec) AsArtifact() *Artifact { + return &Artifact{ + GroupId: a.GroupId, + ArtifactId: a.ArtifactId, + Version: a.Version, + Classifier: a.Classifier, + Extension: a.Extension, + } +} + type meta struct { Packaging string `json:"packaging"` HashType crypto.Hash `json:"hashType"` @@ -106,44 +152,116 @@ type meta struct { func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { // this is how the usual maven repository structure looks like - urlPrefix := a.Repository + path.Join("/", strings.ReplaceAll(a.GroupId, ".", "/"), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version+".") - fs := vfsattr.Get(ctx) + baseUrl := a.Repository + path.Join("/", strings.ReplaceAll(a.GroupId, ".", "/"), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version) + //fs := vfsattr.Get(ctx) // first let's read the pom file and check which binary artifact we need to read log := logging.Context().Logger(identity.REALM) - log.Debug("Reading ", "pom", urlPrefix+"pom") - pom, err := readPom(urlPrefix+"pom", fs) - if err != nil { - return nil, errors.Wrapf(err, "cannot read pom file for %s", urlPrefix) + log.Debug("Reading ", "repository", baseUrl) + + /* + pom, err := readPom(urlPrefix+"pom", fs) + if err != nil { + return nil, errors.Wrapf(err, "cannot read GAV: %s", baseUrl) + } + + hashType, hash, err := getBestHashValue(urlPrefix+pom.Packaging, fs) + if err != nil { + log.Debug("Could not find any hash value for ", "file", urlPrefix+pom.Packaging) + } + asc, err := getStringData(urlPrefix+pom.Packaging+".asc", fs) + if err != nil { + log.Debug("No signing info found for ", "file", urlPrefix+pom.Packaging) + } + + + */ + var metadata meta = meta{ + Packaging: "", + HashType: crypto.MD5, + Hash: "hash", + Bin: "urlPrefix + pom.Packaging", + Asc: "asc", } - hashType, hash, err := getBestHashValue(urlPrefix+pom.Packaging, fs) + return &metadata, nil +} + +func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { + // Create a new request + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, a.BaseUrl(), nil) + if err != nil { + return nil, err + } + // Execute the request + client := &http.Client{} + resp, err := client.Do(req) if err != nil { - log.Debug("Could not find any hash value for ", "file", urlPrefix+pom.Packaging) + return nil, err + } + defer resp.Body.Close() + // Check the response + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s - %s", a.BaseUrl(), resp.Status) } - asc, err := getStringData(urlPrefix+pom.Packaging+".asc", fs) + htmlDoc, err := html.Parse(resp.Body) if err != nil { - log.Debug("No signing info found for ", "file", urlPrefix+pom.Packaging) + return nil, err } - var metadata meta = meta{ - Packaging: pom.Packaging, - HashType: hashType, - Hash: hash, - Bin: urlPrefix + pom.Packaging, - Asc: asc, + // Which files are listed in the repository? + var fileList []string + var process func(*html.Node) + prefix := a.AsArtifact().FilePrefix() + process = func(node *html.Node) { + // check if the node is an element node and the tag is "" + if node.Type == html.ElementNode && node.Data == "a" { + // iterate over the attr and read the href attribute + for _, attribute := range node.Attr { + if attribute.Key == "href" { + // check if the href starts with artifactId-version + if strings.HasPrefix(attribute.Val, prefix) { + fileList = append(fileList, attribute.Val) + } + } + } + } + for nextChild := node.FirstChild; nextChild != nil; nextChild = nextChild.NextSibling { + process(nextChild) // recursive call! + } } + process(htmlDoc) - return &metadata, nil + var filesAndHashes map[string]crypto.Hash + filesAndHashes = make(map[string]crypto.Hash) + for _, file := range fileList { + if IsArtifact(file) { + filesAndHashes[file] = bestAvailableHash(fileList, file) + } + } + + sort.Strings(fileList) + return filesAndHashes, nil +} + +func bestAvailableHash(list []string, filename string) crypto.Hash { + hashes := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} + for _, hash := range hashes { + if slices.Contains(list, filename+hashUrlExt(hash)) { + return hash + } + } + return 0 } //////////////////////////////////////////////////////////////////////////////// +/* type project struct { GroupId string `xml:"groupId"` ArtifactId string `xml:"artifactId"` Version string `xml:"version"` - Packaging string `xml:"packaging"` + Classifier string `xml:"packaging"` } func readPom(url string, fs vfs.FileSystem) (*project, error) { @@ -161,11 +279,12 @@ func readPom(url string, fs vfs.FileSystem) (*project, error) { if err != nil { return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", url) } - if pom.Packaging == "" { - pom.Packaging = "jar" + if pom.Classifier == "" { + pom.Classifier = "jar" } return &pom, nil } +*/ // getStringData reads all data from the given URL and returns it as a string. func getStringData(url string, fs vfs.FileSystem) (string, error) { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 9a1a0dea61..2818bd797e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -2,6 +2,7 @@ package mvn_test import ( "crypto" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,6 +37,41 @@ var _ = Describe("Method", func() { It("get packaging", func() { acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + files, err := acc.GetGAVFiles() + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(1)) + Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) + }) + + It("get packaging", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") + Expect(acc).ToNot(BeNil()) + Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) + files, err := acc.GetGAVFiles() + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(8)) + + for _, f := range files { + fmt.Println(f) + } + + //Expect(files[0]).To(Equal("sdk-modules-bom-5.7.0.pom")) + Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) + Expect(files["apache-maven-3.9.6.pom"]).To(Equal(crypto.SHA1)) + }) + + It("get packaging", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + + /* + repos to test with: + - https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6/ // bin + tar.gz etc. + - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx + - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! + - https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/ // one single pom only! + - https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/ // jar only! + */ + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) Expect(err).ToNot(HaveOccurred()) Expect(meta.Packaging).To(Equal("pom")) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go deleted file mode 100644 index c82b70d706..0000000000 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact.go +++ /dev/null @@ -1,56 +0,0 @@ -package mvn - -import "strings" - -// Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. -type Artifact struct { - GroupId string `xml:"groupId"` - ArtifactId string `xml:"artifactId"` - Version string `xml:"version"` - Packaging string `xml:"packaging"` -} - -// GAV returns the GAV coordinates of the Maven Artifact. -func (a *Artifact) GAV() string { - return a.GroupId + ":" + a.ArtifactId + ":" + a.Version -} - -// Path returns the Maven Artifact's path within a repository. -func (a *Artifact) Path() string { - return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version + "/" + a.ArtifactId + "-" + a.Version + "." + a.Packaging -} - -// GroupPath returns GroupId with `/` instead of `.`. -func (a *Artifact) GroupPath() string { - return strings.ReplaceAll(a.GroupId, ".", "/") -} - -// Purl returns the Package URL of the Maven Artifact. -func (a *Artifact) Purl() string { - return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version -} - -// FromGAV creates new Artifact from GAV coordinates. -func FromGAV(gav string) *Artifact { - parts := strings.Split(gav, ":") - if len(parts) != 3 { - return nil - } - return &Artifact{ - GroupId: parts[0], - ArtifactId: parts[1], - Version: parts[2], - Packaging: "jar", - } -} - -// Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). -type Body struct { - Repo string `json:"repo"` - Path string `json:"path"` - DownloadUri string `json:"downloadUri"` - Uri string `json:"uri"` - MimeType string `json:"mimeType"` - Size string `json:"size"` - Checksums map[string]string `json:"checksums"` -} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go deleted file mode 100644 index 2e6e233a10..0000000000 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/artifact_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package mvn - -import ( - "encoding/json" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Maven Test Environment", func() { - - It("GAV, GroupPath, Path", func() { - artifact := &Artifact{ - GroupId: "ocm.software", - ArtifactId: "hello-ocm", - Version: "0.0.1", - Packaging: "jar", - } - Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) - Expect(artifact.GroupPath()).To(Equal("ocm/software")) - Expect(artifact.Path()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) - }) - - It("Body", func() { - resp := `{ "repo" : "ocm-mvn-test", - "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", - "created" : "2024-04-11T15:09:28.920Z", - "createdBy" : "john.doe", - "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", - "mimeType" : "application/java-archive", - "size" : "1792", - "checksums" : { - "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", - "md5" : "6cb7520b65d820b3b35773a8daa8368e", - "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, - "originalChecksums" : { - "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, - "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` - var body Body - err := json.Unmarshal([]byte(resp), &body) - Expect(err).To(BeNil()) - Expect(body.Repo).To(Equal("ocm-mvn-test")) - Expect(body.Path).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.MimeType).To(Equal("application/java-archive")) - Expect(body.Size).To(Equal("1792")) - Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) - Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) - Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) - Expect(body.Checksums["sha512"]).To(Equal("")) - }) - -}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 0bfc93429b..8b8a99b69d 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -25,13 +25,16 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { return &artifactHandler{repospec} } -func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { // check conditions if b.spec == nil { return nil, nil } mimeType := blob.MimeType() - if mime.MIME_JAR != mimeType { + if resourcetypes.MVN_ARTIFACT != resourceType { + return nil, nil + } + if mime.MIME_JAR != mimeType && mime.MIME_TGZ != mimeType { return nil, nil } if b.spec.Url == "" { @@ -43,19 +46,19 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, log = log.WithValues("repository", b.spec.Url) // identify artifact - artifact := FromGAV(hint) + artifact := mvn.ArtifactFromHint(hint) log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) log.Debug("identified") // get credentials cred := identity.GetCredentials(ctx.GetContext(), b.spec.Url, artifact.GroupPath()) if cred == nil { - return nil, fmt.Errorf("no credentials found for %s. Couldn't upload '%s'", b.spec.Url, artifact.GAV()) + return nil, fmt.Errorf("no credentials found for %s. Couldn't upload '%s'", b.spec.Url, artifact) } username := cred[identity.ATTR_USERNAME] password := cred[identity.ATTR_PASSWORD] if username == "" || password == "" { - return nil, fmt.Errorf("credentials for %s are invalid. Username or password missing! Couldn't upload '%s'", b.spec.Url, artifact.GAV()) + return nil, fmt.Errorf("credentials for %s are invalid. Username or password missing! Couldn't upload '%s'", b.spec.Url, artifact) } log = log.WithValues("user", username) log.Debug("found credentials") @@ -66,11 +69,27 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, return nil, err } defer blobReader.Close() + + switch mimeType { + case mime.MIME_JAR: + case mime.MIME_TGZ: + + } + + // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis + // vs. https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifacts-from-archive + // Headers: X-Explode-Archive: true + // Map headers = ['X-Checksum-Deploy': "true", 'X-Checksum-Sha1': sha1] + // -H "X-Checksum-Md5: $md5Value" \ + // -H "X-Checksum-Sha1: $sha1Value" \ + // -H "X-Checksum-Sha256: $sha256Value" \ + // Headers: X-Checksum-Deploy: true, X-Checksum-Sha1: sha1Value, X-Checksum-Sha256: sha256Value, X-Checksum: checksum value (type is resolved by length) req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, b.spec.Url+"/"+artifact.Path(), blobReader) if err != nil { return nil, err } req.SetBasicAuth(username, password) + //req.Header.Set("X-Checksum", ) // Execute the request client := &http.Client{} @@ -108,5 +127,16 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, hint string, } log.Debug("successfully uploaded") - return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version), nil + return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil +} + +// Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). +type Body struct { + Repo string `json:"repo"` + Path string `json:"path"` + DownloadUri string `json:"downloadUri"` + Uri string `json:"uri"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + Checksums map[string]string `json:"checksums"` } diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index 1464838319..2a7c09e2a7 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -1,15 +1,14 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package tarutils import ( "archive/tar" "fmt" "io" + "io/fs" pathutil "path" + "sort" "strings" + "time" "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/vfs/pkg/osfs" @@ -189,3 +188,75 @@ func addFileToTar(fs vfs.FileSystem, tw *tar.Writer, path string, realPath strin return fmt.Errorf("unsupported file type %s in %s", info.Mode().String(), path) } } + +func Epoch() time.Time { + return time.Unix(0, 0) +} + +func SimpleTarHeader(fs vfs.FileSystem, filepath string) (*tar.Header, error) { + info, err := fs.Lstat(filepath) + if err != nil { + return nil, err + } + return RegularFileInfoHeader(info), nil +} + +func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { + h := &tar.Header{ + Typeflag: tar.TypeReg, + Name: fi.Name(), + Size: fi.Size(), + Mode: int64(fs.ModePerm), + Uid: 0, + Gid: 0, + Uname: "", + Gname: "", + ModTime: Epoch(), + AccessTime: Epoch(), + ChangeTime: Epoch(), + } + return h +} + +func ListSortedFilesInDir(fs vfs.FileSystem, root string) ([]string, error) { + var files []string + err := vfs.Walk(fs, root, func(path string, info vfs.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + sort.Strings(files) + return files, err +} + +func TarFs(fs vfs.FileSystem, writer io.Writer) error { + tw := tar.NewWriter(writer) + files, err := ListSortedFilesInDir(fs, "") + if err != nil { + return err + } + + for _, fileName := range files { + header, err := SimpleTarHeader(fs, fileName) + if err != nil { + return err + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("unable to write header for %q: %w", fileName, err) + } + file, err := fs.OpenFile(fileName, vfs.O_RDONLY, vfs.ModePerm) + if err != nil { + return fmt.Errorf("unable to open file %q: %w", fileName, err) + } + if _, err := io.Copy(tw, file); err != nil { + _ = file.Close() + return fmt.Errorf("unable to add file to tar %q: %w", fileName, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("unable to close file %q: %w", fileName, err) + } + } + + return tw.Close() +} From dd6375c402c6c5ab3772e5eaf329524f224a9eda Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 19 Apr 2024 12:59:40 +0200 Subject: [PATCH 031/100] intermediate state --- .../ocm/accessmethods/mvn/artifact.go | 29 +++++- .../ocm/accessmethods/mvn/artifact_test.go | 8 +- pkg/contexts/ocm/accessmethods/mvn/method.go | 88 +++++++++++-------- pkg/mime/types.go | 1 + pkg/utils/tarutils/pack.go | 2 +- 5 files changed, 82 insertions(+), 46 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 575b23fc13..c6b902744b 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -2,6 +2,8 @@ package mvn import ( "strings" + + "github.com/open-component-model/ocm/pkg/mime" ) // Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. @@ -31,8 +33,9 @@ func (a *Artifact) GavPath() string { return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version } -// Path returns the Maven Artifact's path with classifier and extension. -func (a *Artifact) Path() string { +// FileName returns the Maven Artifact's name with classifier and extension. +// Default extension is jar. +func (a *Artifact) FileName() string { path := a.GavPath() + "/" + a.FilePrefix() if a.Classifier != "" { path += "-" + a.Classifier @@ -45,6 +48,10 @@ func (a *Artifact) Path() string { return path } +func (a *Artifact) DownloadUrl(baseUrl string) string { + return baseUrl + "/" + a.FileName() +} + // GroupPath returns GroupId with `/` instead of `.`. func (a *Artifact) GroupPath() string { return strings.ReplaceAll(a.GroupId, ".", "/") @@ -70,6 +77,24 @@ func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { return a } +// MimeType returns the MIME type of the Maven Artifact based on the file extension. +// Default is application/x-tgz. +func (a *Artifact) MimeType() string { + switch a.Extension { + case "jar": + return mime.MIME_JAR + case "json", "module": + return mime.MIME_JSON + case "pom", "xml": + return mime.MIME_XML + case "tar.gz": + return mime.MIME_TGZ + case "zip": + return mime.MIME_GZIP + } + return mime.MIME_TGZ +} + // ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. func ArtifactFromHint(gav string) *Artifact { parts := strings.Split(gav, ":") diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go index 3787c1f7d1..5b63738b58 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go @@ -7,7 +7,7 @@ import ( var _ = Describe("Maven Test Environment", func() { - It("GAV, GroupPath, Path", func() { + It("GAV, GroupPath, FileName", func() { artifact := &Artifact{ GroupId: "ocm.software", ArtifactId: "hello-ocm", @@ -16,7 +16,7 @@ var _ = Describe("Maven Test Environment", func() { } Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) Expect(artifact.GroupPath()).To(Equal("ocm/software")) - Expect(artifact.Path()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + Expect(artifact.FileName()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) }) It("ClassifierExtensionFrom", func() { @@ -48,7 +48,7 @@ var _ = Describe("Maven Test Environment", func() { Expect(artifact.Version).To(Equal("1.26.1")) Expect(artifact.Classifier).To(Equal("cyclonedx")) Expect(artifact.Extension).To(Equal("xml")) - Expect(artifact.Path()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) + Expect(artifact.FileName()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) }) /* @@ -71,7 +71,7 @@ var _ = Describe("Maven Test Environment", func() { err := json.Unmarshal([]byte(resp), &body) Expect(err).To(BeNil()) Expect(body.Repo).To(Equal("ocm-mvn-test")) - Expect(body.Path).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.FileName).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.MimeType).To(Equal("application/java-archive")) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index d8539217a4..54caf2fc3d 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "path" "sort" "strings" @@ -59,6 +58,8 @@ type AccessSpec struct { var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) +var log = logging.Context().Logger(identity.REALM) + // New creates a new Maven (mvn) repository access spec version v1. func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { accessSpec := &AccessSpec{ @@ -129,7 +130,7 @@ func (a *AccessSpec) BaseUrl() string { } func (a *AccessSpec) ArtifactUrl() string { - return a.Repository + a.AsArtifact().Path() + return a.AsArtifact().DownloadUrl(a.BaseUrl()) } func (a *AccessSpec) AsArtifact() *Artifact { @@ -143,80 +144,84 @@ func (a *AccessSpec) AsArtifact() *Artifact { } type meta struct { - Packaging string `json:"packaging"` - HashType crypto.Hash `json:"hashType"` - Hash string `json:"hash"` - Asc string `json:"asc"` - Bin string `json:"bin"` + MimeType string `json:"packaging"` + HashType crypto.Hash `json:"hashType"` + Hash string `json:"hash"` + Bin string `json:"bin"` } func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { - // this is how the usual maven repository structure looks like - baseUrl := a.Repository + path.Join("/", strings.ReplaceAll(a.GroupId, ".", "/"), a.ArtifactId, a.Version, a.ArtifactId+"-"+a.Version) - //fs := vfsattr.Get(ctx) - // first let's read the pom file and check which binary artifact we need to read - log := logging.Context().Logger(identity.REALM) - log.Debug("Reading ", "repository", baseUrl) + fs := vfsattr.Get(ctx) - /* - pom, err := readPom(urlPrefix+"pom", fs) - if err != nil { - return nil, errors.Wrapf(err, "cannot read GAV: %s", baseUrl) - } + fileMap, err := a.GetGAVFiles() + if err != nil { + return nil, errors.Wrapf(err, "cannot read GAV: %s", a.BaseUrl()) + } - hashType, hash, err := getBestHashValue(urlPrefix+pom.Packaging, fs) - if err != nil { - log.Debug("Could not find any hash value for ", "file", urlPrefix+pom.Packaging) - } - asc, err := getStringData(urlPrefix+pom.Packaging+".asc", fs) - if err != nil { - log.Debug("No signing info found for ", "file", urlPrefix+pom.Packaging) + singleArtifact := len(fileMap) == 1 + var metadata = meta{} + + for file, hash := range fileMap { + artifact := a.AsArtifact() + artifact.ClassifierExtensionFrom(file) + metadata.Bin = artifact.DownloadUrl(a.BaseUrl()) + log.WithValues("file", metadata.Bin) + log.Debug("processing") + metadata.MimeType = artifact.MimeType() + if hash > 0 { + metadata.HashType = hash + metadata.Hash, err = getStringData(metadata.Bin+hashUrlExt(hash), fs) + if err != nil { + return nil, errors.Wrapf(err, "cannot read %s digest of: %s", hash, metadata.Bin) + } + } else { + log.Warn("no digest available") } - - */ - var metadata meta = meta{ - Packaging: "", - HashType: crypto.MD5, - Hash: "hash", - Bin: "urlPrefix + pom.Packaging", - Asc: "asc", + if singleArtifact { + return &metadata, nil + } } + // if we have multiple artifacts, we pack them into a tar.gz file + metadata.MimeType = mime.MIME_TGZ + + // create a tar.gz file + return &metadata, nil } +// GetGAVFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { - // Create a new request + log.WithValues("repository", a.BaseUrl()) + log.Debug("reading") req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, a.BaseUrl(), nil) if err != nil { return nil, err } - // Execute the request client := &http.Client{} resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() - // Check the response if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("%s - %s", a.BaseUrl(), resp.Status) } + + // Which files are listed in the repository? + log.Debug("parsing html") htmlDoc, err := html.Parse(resp.Body) if err != nil { return nil, err } - - // Which files are listed in the repository? var fileList []string var process func(*html.Node) prefix := a.AsArtifact().FilePrefix() process = func(node *html.Node) { // check if the node is an element node and the tag is "" if node.Type == html.ElementNode && node.Data == "a" { - // iterate over the attr and read the href attribute for _, attribute := range node.Attr { if attribute.Key == "href" { // check if the href starts with artifactId-version @@ -232,18 +237,23 @@ func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { } process(htmlDoc) + // Which hash files are available? var filesAndHashes map[string]crypto.Hash filesAndHashes = make(map[string]crypto.Hash) for _, file := range fileList { if IsArtifact(file) { filesAndHashes[file] = bestAvailableHash(fileList, file) + log.Debug("found", "file", file) } } + // Sort the list of files, to ensure always the same results for e.g. identical tar.gz files. sort.Strings(fileList) return filesAndHashes, nil } +// bestAvailableHash returns the best available hash for the given file. +// It first checks for SHA-512, then SHA-256, SHA-1, and finally MD5. If nothing is found, it returns 0. func bestAvailableHash(list []string, filename string) crypto.Hash { hashes := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} for _, hash := range hashes { diff --git a/pkg/mime/types.go b/pkg/mime/types.go index 677f5030b2..783ed12ab2 100644 --- a/pkg/mime/types.go +++ b/pkg/mime/types.go @@ -7,6 +7,7 @@ const ( MIME_JSON = "application/x-json" MIME_JSON_ALT = "text/json" // no utf8 MIME_JSON_OFFICIAL = "application/json" + MIME_XML = "application/xml" MIME_YAML = "application/x-yaml" MIME_YAML_ALT = "text/yaml" // no utf8 MIME_YAML_OFFICIAL = "application/yaml" diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index 2a7c09e2a7..f15daf354d 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -257,6 +257,6 @@ func TarFs(fs vfs.FileSystem, writer io.Writer) error { return fmt.Errorf("unable to close file %q: %w", fileName, err) } } - + return tw.Close() } From eb79406a8dd248a7ed4223093a141f29c7776a22 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 19 Apr 2024 16:39:45 +0200 Subject: [PATCH 032/100] handle single artifacts --- .../ocm/accessmethods/mvn/artifact.go | 10 ++- pkg/contexts/ocm/accessmethods/mvn/method.go | 75 ++++++++++++++++--- .../ocm/accessmethods/mvn/method_test.go | 25 ++++--- .../handlers/generic/mvn/blobhandler.go | 55 +++++++++----- 4 files changed, 123 insertions(+), 42 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index c6b902744b..819ba56f16 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -48,7 +48,7 @@ func (a *Artifact) FileName() string { return path } -func (a *Artifact) DownloadUrl(baseUrl string) string { +func (a *Artifact) Url(baseUrl string) string { return baseUrl + "/" + a.FileName() } @@ -95,6 +95,14 @@ func (a *Artifact) MimeType() string { return mime.MIME_TGZ } +func IsMimeTypeSupported(mimeType string) bool { + switch mimeType { + case mime.MIME_JAR, mime.MIME_JSON, mime.MIME_XML, mime.MIME_TGZ, mime.MIME_GZIP: + return true + } + return false +} + // ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. func ArtifactFromHint(gav string) *Artifact { parts := strings.Split(gav, ":") diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 54caf2fc3d..a4cd441ecd 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -2,14 +2,17 @@ package mvn import ( "bytes" + "compress/gzip" "context" "crypto" + "crypto/sha256" "fmt" "io" "net/http" "sort" "strings" + "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" "golang.org/x/exp/slices" "golang.org/x/net/html" @@ -25,6 +28,7 @@ import ( "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils/tarutils" ) // Type is the access type of Maven (mvn) repository. @@ -69,7 +73,7 @@ func New(repository, groupId, artifactId, version string, options ...func(*Acces ArtifactId: artifactId, Version: version, Classifier: "", - Extension: "jar", + Extension: "", } for _, option := range options { option(accessSpec) @@ -130,7 +134,7 @@ func (a *AccessSpec) BaseUrl() string { } func (a *AccessSpec) ArtifactUrl() string { - return a.AsArtifact().DownloadUrl(a.BaseUrl()) + return a.AsArtifact().Url(a.Repository) } func (a *AccessSpec) AsArtifact() *Artifact { @@ -151,6 +155,14 @@ type meta struct { } func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { + artifact := a.AsArtifact() + var metadata = meta{} + + if artifact.Extension != "" { + metadata.Bin = artifact.Url(a.Repository) + metadata.MimeType = artifact.MimeType() + return &metadata, nil + } fs := vfsattr.Get(ctx) @@ -160,12 +172,16 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } singleArtifact := len(fileMap) == 1 - var metadata = meta{} + + tempFs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + defer vfs.Cleanup(tempFs) for file, hash := range fileMap { - artifact := a.AsArtifact() artifact.ClassifierExtensionFrom(file) - metadata.Bin = artifact.DownloadUrl(a.BaseUrl()) + metadata.Bin = artifact.Url(a.Repository) log.WithValues("file", metadata.Bin) log.Debug("processing") metadata.MimeType = artifact.MimeType() @@ -182,13 +198,51 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { if singleArtifact { return &metadata, nil } + + // download the artifact into the temporary file system + out, err := tempFs.Create(file) + if err != nil { + return nil, err + } + resp, err := http.Get(metadata.Bin) + defer resp.Body.Close() + if err != nil { + return nil, err + } + _, err = io.Copy(out, resp.Body) + if err != nil { + return nil, err + } + } + + // pack all downloaded files into a tar.gz file + tgz, err := blobaccess.NewTempFile("", Type+"-"+artifact.FilePrefix()+"-*.tar.gz", fs) + defer tgz.Close() + if err != nil { + return nil, err } + err = tarutils.TarFs(tempFs, gzip.NewWriter(tgz.Writer())) + if err != nil { + return nil, err + } + metadata.Bin = "file://" + tgz.Name() + log.Debug("created", "file", metadata.Bin) - // if we have multiple artifacts, we pack them into a tar.gz file + // calculate digest for the tar.gz file + file, err := fs.OpenFile(tgz.Name(), vfs.O_RDONLY, vfs.ModePerm) + defer file.Close() + if err != nil { + return nil, err + } + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return nil, err + } + metadata.Hash = fmt.Sprintf("%x", hash.Sum(nil)) + log.Debug("hash", "sum", metadata.Hash) + metadata.HashType = crypto.SHA256 metadata.MimeType = mime.MIME_TGZ - // create a tar.gz file - return &metadata, nil } @@ -353,9 +407,10 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A } } acc := blobaccess.DataAccessForReaderFunction(reader, meta.Bin) - return accessobj.CachedBlobAccessForWriter(c.GetContext(), mime.MIME_JAR, accessio.NewDataAccessWriter(acc)), nil + return accessobj.CachedBlobAccessForWriter(c.GetContext(), a.AsArtifact().MimeType(), accessio.NewDataAccessWriter(acc)), nil } - return accspeccpi.NewDefaultMethodImpl(c, a, "", mime.MIME_JAR, factory), nil + // FIXME add Digest! + return accspeccpi.NewDefaultMethodImpl(c, a, "", a.AsArtifact().MimeType(), factory), nil } func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 2818bd797e..cfb3102a37 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -2,7 +2,6 @@ package mvn_test import ( "crypto" - "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -51,16 +50,12 @@ var _ = Describe("Method", func() { Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(8)) - for _, f := range files { - fmt.Println(f) - } - //Expect(files[0]).To(Equal("sdk-modules-bom-5.7.0.pom")) Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) Expect(files["apache-maven-3.9.6.pom"]).To(Equal(crypto.SHA1)) }) - It("get packaging", func() { + It("GetPackageMeta - com.sap.cloud.sdk", func() { acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") /* @@ -74,17 +69,25 @@ var _ = Describe("Method", func() { meta, err := acc.GetPackageMeta(ocm.DefaultContext()) Expect(err).ToNot(HaveOccurred()) - Expect(meta.Packaging).To(Equal("pom")) + Expect(meta.Bin).To(Equal("https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom")) Expect(meta.Hash).To(Equal("34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) Expect(meta.HashType).To(Equal(crypto.SHA1)) - Expect(meta.Asc).To(ContainSubstring("-----BEGIN PGP SIGNATURE-----")) + }) + + It("GetPackageMeta - int.repositories.cloud.sap: hello-ocm", func() { + acc := mvn.New("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test", "open-component-model", "hello-ocm", "0.0.1") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + Expect(meta.Hash).To(Equal("")) + Expect(meta.HashType).To(Equal(crypto.Hash(0))) }) It("accesses artifact", func() { - acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0", mvn.WithExtension("pom")) m := Must(acc.AccessMethod(cv)) defer m.Close() - Expect(m.MimeType()).To(Equal(mime.MIME_JAR)) // FIXME what about POM? + Expect(m.MimeType()).To(Equal(mime.MIME_XML)) r := Must(m.Reader()) defer r.Close() @@ -101,7 +104,7 @@ var _ = Describe("Method", func() { }) It("detects digests mismatch", func() { - acc := mvn.New("file://"+FAILPATH, "fail", "repository", "42") + acc := mvn.New("file://"+FAILPATH, "fail", "repository", "42", mvn.WithExtension("pom")) m := Must(acc.AccessMethod(cv)) defer m.Close() diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 8b8a99b69d..5ed438883a 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -6,6 +6,9 @@ import ( "fmt" "io" "net/http" + "strings" + + "github.com/opencontainers/go-digest" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" @@ -25,6 +28,8 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { return &artifactHandler{repospec} } +var log = logging.Context().Logger(identity.REALM) + func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { // check conditions if b.spec == nil { @@ -34,7 +39,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi if resourcetypes.MVN_ARTIFACT != resourceType { return nil, nil } - if mime.MIME_JAR != mimeType && mime.MIME_TGZ != mimeType { + if !mvn.IsMimeTypeSupported(mimeType) { return nil, nil } if b.spec.Url == "" { @@ -42,7 +47,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } // setup logger - log := logging.Context().Logger(identity.REALM) log = log.WithValues("repository", b.spec.Url) // identify artifact @@ -71,9 +75,10 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi defer blobReader.Close() switch mimeType { - case mime.MIME_JAR: case mime.MIME_TGZ: - + // TODO extract the archive and upload the content + default: + err = deploy(artifact, b.spec.Url, blobReader, username, password, blob.Digest()) } // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis @@ -83,19 +88,31 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi // -H "X-Checksum-Md5: $md5Value" \ // -H "X-Checksum-Sha1: $sha1Value" \ // -H "X-Checksum-Sha256: $sha256Value" \ + + log.Debug("successfully uploaded") + return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil +} + +func ChecksumHeader(digest digest.Digest) string { + a := digest.Algorithm().String() + return "X-Checksum-" + strings.ToUpper(a[:1]) + a[1:] +} + +func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, digest digest.Digest) error { // Headers: X-Checksum-Deploy: true, X-Checksum-Sha1: sha1Value, X-Checksum-Sha256: sha256Value, X-Checksum: checksum value (type is resolved by length) - req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, b.spec.Url+"/"+artifact.Path(), blobReader) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) if err != nil { - return nil, err + return err } req.SetBasicAuth(username, password) - //req.Header.Set("X-Checksum", ) + req.Header.Set("X-Checksum", digest.Encoded()) + req.Header.Set(ChecksumHeader(digest), digest.Encoded()) // Execute the request client := &http.Client{} resp, err := client.Do(req) if err != nil { - return nil, err + return err } defer resp.Body.Close() @@ -103,31 +120,29 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi if resp.StatusCode != http.StatusCreated { all, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return err } - return nil, fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) + return fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) } // Validate the response - especially the hash values with the ones we've tried to send respBody, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return err } var artifactBody Body err = json.Unmarshal(respBody, &artifactBody) if err != nil { - return nil, err + return err } - blobDigest := blob.Digest() - remoteDigest := artifactBody.Checksums[string(blobDigest.Algorithm())] + + remoteDigest := artifactBody.Checksums[string(digest.Algorithm())] if remoteDigest == "" { - log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", blobDigest.Algorithm()) - } else if remoteDigest != blobDigest.Encoded() { - return nil, fmt.Errorf("failed to upload artifact: checksums do not match") + log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", digest.Algorithm()) + } else if remoteDigest != digest.Encoded() { + return fmt.Errorf("failed to upload artifact: checksums do not match") } - - log.Debug("successfully uploaded") - return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil + return nil } // Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). From 1a8eb0d7b3b2f3c37884dac7b70bfd4b58dcd13a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 22 Apr 2024 11:58:31 +0200 Subject: [PATCH 033/100] next intermediate test state --- pkg/contexts/ocm/accessmethods/mvn/method.go | 8 +-- .../ocm/accessmethods/mvn/method_test.go | 9 ---- .../handlers/generic/mvn/blobhandler.go | 53 +++++++++++++++---- .../handlers/generic/mvn/registration.go | 5 ++ 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index a4cd441ecd..10bb0a267d 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -108,7 +108,7 @@ func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpe } // GetReferenceHint returns the reference hint for the Maven (mvn) artifact. In the following form: -// groupId:artifactId:version:classifier:extension +// groupId:artifactId:version:classifier:extension. func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + ":" + a.Classifier + ":" + a.Extension } @@ -204,11 +204,12 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { if err != nil { return nil, err } + defer out.Close() resp, err := http.Get(metadata.Bin) - defer resp.Body.Close() if err != nil { return nil, err } + defer resp.Body.Close() _, err = io.Copy(out, resp.Body) if err != nil { return nil, err @@ -292,8 +293,7 @@ func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { process(htmlDoc) // Which hash files are available? - var filesAndHashes map[string]crypto.Hash - filesAndHashes = make(map[string]crypto.Hash) + var filesAndHashes = make(map[string]crypto.Hash) for _, file := range fileList { if IsArtifact(file) { filesAndHashes[file] = bestAvailableHash(fileList, file) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index cfb3102a37..5b6e65537e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -74,15 +74,6 @@ var _ = Describe("Method", func() { Expect(meta.HashType).To(Equal(crypto.SHA1)) }) - It("GetPackageMeta - int.repositories.cloud.sap: hello-ocm", func() { - acc := mvn.New("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test", "open-component-model", "hello-ocm", "0.0.1") - meta, err := acc.GetPackageMeta(ocm.DefaultContext()) - Expect(err).ToNot(HaveOccurred()) - Expect(meta.Bin).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) - Expect(meta.Hash).To(Equal("")) - Expect(meta.HashType).To(Equal(crypto.Hash(0))) - }) - It("accesses artifact", func() { acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0", mvn.WithExtension("pom")) m := Must(acc.AccessMethod(cv)) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 5ed438883a..2b81df758f 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -2,12 +2,15 @@ package mvn import ( "context" + "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "strings" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" "github.com/opencontainers/go-digest" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" @@ -16,6 +19,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/utils/tarutils" ) const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.MVN_ARTIFACT @@ -76,20 +80,45 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi switch mimeType { case mime.MIME_TGZ: - // TODO extract the archive and upload the content + tempFs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + defer vfs.Cleanup(tempFs) + err = tarutils.ExtractTarToFs(tempFs, blobReader) + // err = tarutils.ExtractTarToFs(tempFs, compression.AutoDecompress(blobReader)) // ??? + if err != nil { + return nil, err + } + files, err := tarutils.ListSortedFilesInDir(tempFs, "") + if err != nil { + return nil, err + } + for _, file := range files { + log.Debug("uploading", "file", file) + artifact = artifact.ClassifierExtensionFrom(file) + reader, err := tempFs.Open(file) + if err != nil { + return nil, err + } + defer reader.Close() + hash := sha256.New() + if _, err := io.Copy(hash, reader); err != nil { + return nil, err + } + err = deploy(artifact, b.spec.Url, reader, username, password, digest.NewDigest(digest.SHA256, hash)) + if err != nil { + return nil, err + } + } default: err = deploy(artifact, b.spec.Url, blobReader, username, password, blob.Digest()) + if err != nil { + return nil, err + } } - // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis - // vs. https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifacts-from-archive - // Headers: X-Explode-Archive: true - // Map headers = ['X-Checksum-Deploy': "true", 'X-Checksum-Sha1': sha1] - // -H "X-Checksum-Md5: $md5Value" \ - // -H "X-Checksum-Sha1: $sha1Value" \ - // -H "X-Checksum-Sha256: $sha256Value" \ - - log.Debug("successfully uploaded") + log.Debug("done", "artifact", artifact) return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil } @@ -99,6 +128,8 @@ func ChecksumHeader(digest digest.Digest) string { } func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, digest digest.Digest) error { + // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis + // vs. https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifacts-from-archive // Headers: X-Checksum-Deploy: true, X-Checksum-Sha1: sha1Value, X-Checksum-Sha256: sha256Value, X-Checksum: checksum value (type is resolved by length) req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) if err != nil { @@ -124,6 +155,7 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username s } return fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) } + log.Debug("uploaded", "artifact", artifact, "extension", artifact.Extension, "classifier", artifact.Classifier) // Validate the response - especially the hash values with the ones we've tried to send respBody, err := io.ReadAll(resp.Body) @@ -142,6 +174,7 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username s } else if remoteDigest != digest.Encoded() { return fmt.Errorf("failed to upload artifact: checksums do not match") } + log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest.Encoded()) return nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go index e0b7ebfe9b..9784c38f39 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go @@ -54,7 +54,12 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co ctx.BlobHandlers().Register(NewArtifactHandler(cfg), cpi.ForArtifactType(resourcetypes.MVN_ARTIFACT), + // see artifact.go#IsMimeTypeSupported() cpi.ForMimeType(mime.MIME_JAR), + cpi.ForMimeType(mime.MIME_JSON), + cpi.ForMimeType(mime.MIME_XML), + cpi.ForMimeType(mime.MIME_GZIP), + cpi.ForMimeType(mime.MIME_TGZ), cpi.NewBlobHandlerOptions(olist...), ) From 20af227b3221fac16171215f067cca645d1bc56d Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 22 Apr 2024 11:58:39 +0200 Subject: [PATCH 034/100] typo --- pkg/utils/tarutils/extract.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/utils/tarutils/extract.go b/pkg/utils/tarutils/extract.go index 280ef9cd43..b6769ff39d 100644 --- a/pkg/utils/tarutils/extract.go +++ b/pkg/utils/tarutils/extract.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package tarutils import ( @@ -33,7 +29,7 @@ func ExtractArchiveToFs(fs vfs.FileSystem, path string, fss ...vfs.FileSystem) e return ExtractTarToFs(fs, r) } -// ExtractArchiveToFsWithInfo wunpacks an archive to a filesystem. +// ExtractArchiveToFsWithInfo unpacks an archive to a filesystem. func ExtractArchiveToFsWithInfo(fs vfs.FileSystem, path string, fss ...vfs.FileSystem) (int64, int64, error) { sfs := utils.OptionalDefaulted(osfs.New(), fss...) From b16c20fad0851a4643a24f0af1f91864a9514707 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 22 Apr 2024 12:00:12 +0200 Subject: [PATCH 035/100] "print-semver" --- pkg/version/generate/release_generate.go | 6 ++---- pkg/version/version.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/version/generate/release_generate.go b/pkg/version/generate/release_generate.go index cf193fd7b5..d93f2c6a6c 100644 --- a/pkg/version/generate/release_generate.go +++ b/pkg/version/generate/release_generate.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package main import ( @@ -86,6 +82,8 @@ func main() { //nolint:forbidigo // Logger not needed for this command. switch cmd { + case "print-semver": + fmt.Print(nonpre) case "print-version": fmt.Print(v) case "print-rc-version": diff --git a/pkg/version/version.go b/pkg/version/version.go index ef3506e228..ac487f68a1 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package version import ( @@ -40,6 +36,16 @@ func (info Info) String() string { return info.GitVersion } +// String returns info as a short semantic version string (0.8.15). +func (info Info) SemVer() string { + return info.Major + "." + info.Minor + "." + info.Patch +} + +// String returns current Release version. +func Current() string { + return Get().SemVer() +} + // GetInterface returns the overall codebase version. It's for detecting // what code a binary was built from. // These variables typically come from -ldflags settings and in From 7217f2601c69dcec62885e7e7f0cad87c9089033 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 22 Apr 2024 12:01:14 +0200 Subject: [PATCH 036/100] rm --- pkg/contexts/ocm/accessmethods/mvn/method_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 5b6e65537e..2303af17b4 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -64,7 +64,6 @@ var _ = Describe("Method", func() { - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! - https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/ // one single pom only! - - https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/ // jar only! */ meta, err := acc.GetPackageMeta(ocm.DefaultContext()) From 8264307278f2e8644244af8b046d0cfd3ca0f0f1 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 12:49:53 +0200 Subject: [PATCH 037/100] mvn access works! --- .../ocm/accessmethods/mvn/artifact.go | 2 + pkg/contexts/ocm/accessmethods/mvn/method.go | 160 ++++++++----- .../ocm/accessmethods/mvn/method_test.go | 54 ++++- .../5.7.0/sdk-modules-bom-5.7.0.pom | 210 ++++++++++++++++++ .../handlers/generic/mvn/blobhandler.go | 60 ++--- .../handlers/generic/mvn/registration.go | 9 +- pkg/utils/tarutils/pack.go | 20 +- 7 files changed, 416 insertions(+), 99 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 819ba56f16..51297d1139 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -95,6 +95,7 @@ func (a *Artifact) MimeType() string { return mime.MIME_TGZ } +/* func IsMimeTypeSupported(mimeType string) bool { switch mimeType { case mime.MIME_JAR, mime.MIME_JSON, mime.MIME_XML, mime.MIME_TGZ, mime.MIME_GZIP: @@ -102,6 +103,7 @@ func IsMimeTypeSupported(mimeType string) bool { } return false } +*/ // ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. func ArtifactFromHint(gav string) *Artifact { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 10bb0a267d..1c42a25014 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -2,10 +2,8 @@ package mvn import ( "bytes" - "compress/gzip" "context" "crypto" - "crypto/sha256" "fmt" "io" "net/http" @@ -14,6 +12,7 @@ import ( "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/opencontainers/go-digest" "golang.org/x/exp/slices" "golang.org/x/net/html" @@ -62,7 +61,7 @@ type AccessSpec struct { var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) -var log = logging.Context().Logger(identity.REALM) +var logger = logging.Context().Logger(identity.REALM) // New creates a new Maven (mvn) repository access spec version v1. func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { @@ -155,23 +154,18 @@ type meta struct { } func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { - artifact := a.AsArtifact() - var metadata = meta{} - - if artifact.Extension != "" { - metadata.Bin = artifact.Url(a.Repository) - metadata.MimeType = artifact.MimeType() - return &metadata, nil - } - fs := vfsattr.Get(ctx) - fileMap, err := a.GetGAVFiles() + log := logger.WithValues("BaseUrl", a.BaseUrl()) + fileMap, err := a.GavFiles(fs) if err != nil { - return nil, errors.Wrapf(err, "cannot read GAV: %s", a.BaseUrl()) + return nil, err } - singleArtifact := len(fileMap) == 1 + if a.Classifier != "" { + fileMap = filterByClassifier(fileMap, a.Classifier) + } + singleBinary := len(fileMap) == 1 tempFs, err := osfs.NewTempFileSystem() if err != nil { @@ -179,10 +173,12 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } defer vfs.Cleanup(tempFs) + artifact := a.AsArtifact() + var metadata = meta{} for file, hash := range fileMap { artifact.ClassifierExtensionFrom(file) metadata.Bin = artifact.Url(a.Repository) - log.WithValues("file", metadata.Bin) + log = logger.WithValues("file", metadata.Bin) log.Debug("processing") metadata.MimeType = artifact.MimeType() if hash > 0 { @@ -195,7 +191,9 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { log.Warn("no digest available") } - if singleArtifact { + // single binary dependency, this will never be a complete GAV - no maven uploader support! + if a.Extension != "" || singleBinary && a.Classifier != "" { + // in case you want to transport pom, then you should NOT set the extension return &metadata, nil } @@ -205,36 +203,56 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { return nil, err } defer out.Close() - resp, err := http.Get(metadata.Bin) + reader, err := getReader(metadata.Bin, fs) if err != nil { return nil, err } - defer resp.Body.Close() - _, err = io.Copy(out, resp.Body) - if err != nil { - return nil, err + defer reader.Close() + + if hash > 0 { + dreader := iotools.NewDigestReaderWithHash(hash, reader) + _, err = io.Copy(out, dreader) + if err != nil { + return nil, err + } + sum := dreader.Digest().Encoded() + if metadata.Hash != sum { + return nil, errors.Newf("checksum mismatch for %s", metadata.Bin) + } + } else { + _, err = io.Copy(out, reader) + if err != nil { + return nil, err + } } } // pack all downloaded files into a tar.gz file - tgz, err := blobaccess.NewTempFile("", Type+"-"+artifact.FilePrefix()+"-*.tar.gz", fs) - defer tgz.Close() + // tgz, err := blobaccess.NewTempFile("", Type+"-"+artifact.FilePrefix()+"-*.tar.gz", fs) + tgz, err := vfs.TempFile(fs, "", Type+"-"+artifact.FilePrefix()+"-*.tar.gz") if err != nil { return nil, err } - err = tarutils.TarFs(tempFs, gzip.NewWriter(tgz.Writer())) + defer tgz.Close() + + dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) + defer dw.Close() + err = tarutils.TgzFs(tempFs, dw) if err != nil { return nil, err } + metadata.Bin = "file://" + tgz.Name() + metadata.Hash = dw.Digest().Encoded() + metadata.HashType = crypto.SHA256 log.Debug("created", "file", metadata.Bin) - // calculate digest for the tar.gz file + /*/ calculate digest for the tar.gz file file, err := fs.OpenFile(tgz.Name(), vfs.O_RDONLY, vfs.ModePerm) - defer file.Close() if err != nil { return nil, err } + defer file.Close() hash := sha256.New() if _, err := io.Copy(hash, file); err != nil { return nil, err @@ -242,32 +260,64 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { metadata.Hash = fmt.Sprintf("%x", hash.Sum(nil)) log.Debug("hash", "sum", metadata.Hash) metadata.HashType = crypto.SHA256 + */ metadata.MimeType = mime.MIME_TGZ return &metadata, nil } -// GetGAVFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. -func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { - log.WithValues("repository", a.BaseUrl()) - log.Debug("reading") - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, a.BaseUrl(), nil) +func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[string]crypto.Hash { + var filtered = make(map[string]crypto.Hash) + for file, hash := range fileMap { + if strings.Contains(file, "-"+classifier+".") { + filtered[file] = hash + } + } + return filtered +} + +/* +func (a *AccessSpec) fileListFactory(fs vfs.FileSystem) func() (map[string]crypto.Hash, error) { + return func() (map[string]crypto.Hash, error) { + if strings.HasPrefix(a.BaseUrl(), "file://") { + dir := a.BaseUrl()[7:] + return gavFilesFromDisk(fs, dir) + } + return a.gavOnlineFiles() + } +} +*/ + +func (a *AccessSpec) GavFiles(fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { + if strings.HasPrefix(a.Repository, "file://") && len(fs) > 0 { + dir := a.Repository[7:] + return gavFilesFromDisk(fs[0], dir) + } + return a.gavOnlineFiles() +} + +func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, error) { + files, err := tarutils.FlatListSortedFilesInDir(fs, dir) if err != nil { return nil, err } - client := &http.Client{} - resp, err := client.Do(req) + return filesAndHashes(files), nil +} + +// gavOnlineFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. +func (a *AccessSpec) gavOnlineFiles() (map[string]crypto.Hash, error) { + log := logger.WithValues("BaseUrl", a.BaseUrl()) + log.Debug("gavOnlineFiles") + + reader, err := getReader(a.BaseUrl(), nil) if err != nil { return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%s - %s", a.BaseUrl(), resp.Status) - } + defer reader.Close() // Which files are listed in the repository? - log.Debug("parsing html") - htmlDoc, err := html.Parse(resp.Body) + log.Debug("parse-html") + htmlDoc, err := html.Parse(reader) if err != nil { return nil, err } @@ -292,18 +342,22 @@ func (a *AccessSpec) GetGAVFiles() (map[string]crypto.Hash, error) { } process(htmlDoc) + return filesAndHashes(fileList), nil +} + +func filesAndHashes(fileList []string) map[string]crypto.Hash { + // Sort the list of files, to ensure always the same results for e.g. identical tar.gz files. + sort.Strings(fileList) + // Which hash files are available? - var filesAndHashes = make(map[string]crypto.Hash) + var result = make(map[string]crypto.Hash) for _, file := range fileList { if IsArtifact(file) { - filesAndHashes[file] = bestAvailableHash(fileList, file) - log.Debug("found", "file", file) + result[file] = bestAvailableHash(fileList, file) + logger.Debug("found", "file", file) } } - - // Sort the list of files, to ensure always the same results for e.g. identical tar.gz files. - sort.Strings(fileList) - return filesAndHashes, nil + return result } // bestAvailableHash returns the best available hash for the given file. @@ -363,7 +417,7 @@ func getStringData(url string, fs vfs.FileSystem) (string, error) { return string(b), nil } -// getBestHashValue returns the best hash value (SHA-512, SHA-256, SHA-1, MD5) for the given artifact (POM, JAR, ...). +/*/ getBestHashValue returns the best hash value (SHA-512, SHA-256, SHA-1, MD5) for the given artifact (POM, JAR, ...). func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error) { arr := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} log := logging.Context().Logger(identity.REALM) @@ -379,6 +433,7 @@ func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error } return 0, "", errors.New("no hash value found") } +*/ // hashUrlExt returns the 'maven' hash extension for the given hash. // Maven usually uses sha1, sha256, sha512, md5 instead of SHA-1, SHA-256, SHA-512, MD5. @@ -414,8 +469,6 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A } func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { - c := &http.Client{} - if strings.HasPrefix(url, "file://") { path := url[7:] return fs.OpenFile(path, vfs.O_RDONLY, 0o600) @@ -425,7 +478,8 @@ func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { if err != nil { return nil, err } - resp, err := c.Do(req) + httpClient := &http.Client{} + resp, err := httpClient.Do(req) if err != nil { return nil, err } @@ -434,9 +488,9 @@ func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { buf := &bytes.Buffer{} _, err = io.Copy(buf, io.LimitReader(resp.Body, 2000)) if err != nil { - return nil, errors.Newf("version meta data request %s provides %s", url, resp.Status) + return nil, errors.Newf("http %s error - %s", resp.Status, url) } - return nil, errors.Newf("version meta data request %s provides %s: %s", url, resp.Status, buf.String()) + return nil, errors.Newf("http %s error - %s returned: %s", resp.Status, url, buf.String()) } return resp.Body, nil } diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 2303af17b4..b0f3ed6de6 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -17,7 +17,7 @@ import ( ) const ( - mvnPATH = "/testdata/.m2/repository" + mvnPATH = "/testdata/success" FAILPATH = "/testdata" ) @@ -36,7 +36,7 @@ var _ = Describe("Method", func() { It("get packaging", func() { acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") - files, err := acc.GetGAVFiles() + files, err := acc.GavFiles() Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(1)) Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) @@ -46,7 +46,7 @@ var _ = Describe("Method", func() { acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") Expect(acc).ToNot(BeNil()) Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) - files, err := acc.GetGAVFiles() + files, err := acc.GavFiles() Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(8)) @@ -64,16 +64,51 @@ var _ = Describe("Method", func() { - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! - https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/ // one single pom only! + - https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/ // jar only! */ meta, err := acc.GetPackageMeta(ocm.DefaultContext()) Expect(err).ToNot(HaveOccurred()) - Expect(meta.Bin).To(Equal("https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom")) - Expect(meta.Hash).To(Equal("34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) - Expect(meta.HashType).To(Equal(crypto.SHA1)) + Expect(meta.Bin).To(HavePrefix("file://")) + Expect(meta.Bin).To(ContainSubstring("mvn-sdk-modules-bom-5.7.0-")) + Expect(meta.Bin).To(HaveSuffix(".tar.gz")) + Expect(meta.Hash).To(Equal("217feb1e7490015dd0a2b231b9cea45804df3d2a9b37287ac861bb45b8c0de55")) + Expect(meta.HashType).To(Equal(crypto.SHA256)) }) - It("accesses artifact", func() { + /*/ works only internal + It("GetPackageMeta - int.repositories.cloud.sap: hello-ocm", func() { + acc := mvn.New("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test", "open-component-model", "hello-ocm", "0.0.1") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + Expect(meta.Hash).To(Equal("")) + Expect(meta.HashType).To(Equal(crypto.Hash(0))) + }) + + */ + + It("accesses local artifact", func() { + acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + + r := Must(m.Reader()) + defer r.Close() + dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) + for { + var buf [8096]byte + _, err := dr.Read(buf[:]) + if err != nil { + break + } + } + Expect(dr.Size()).To(Equal(int64(10))) + Expect(dr.Digest().String()).To(Equal("SHA-1:e727ef4792a349c485d893e60874475a54f24b97")) + }) + + It("accesses local artifact with extension", func() { acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0", mvn.WithExtension("pom")) m := Must(acc.AccessMethod(cv)) defer m.Close() @@ -101,4 +136,9 @@ var _ = Describe("Method", func() { _, err := m.Reader() Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found b3242b8c31f8ce14f729b8fd132ac77bc4bc5bf7"))) }) + + FIt("NewDigestWriterWithHash", func() { + w := iotools.NewDigestWriterWithHash(crypto.SHA1, nil) + Expect(w).ToNot(BeNil()) + }) }) diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom new file mode 100644 index 0000000000..b3baaee32f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom @@ -0,0 +1,210 @@ + + + 4.0.0 + com.sap.cloud.sdk + sdk-modules-bom + 5.7.0 + pom + SAP Cloud SDK - Modules BOM + Bill of Materials (BOM) of the SAP Cloud SDK modules. + https://sap.github.io/cloud-sdk/docs/java/getting-started + + SAP SE + https://www.sap.com + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + SAP + cloudsdk@sap.com + SAP SE + https://www.sap.com + + + + + + + + UTF-8 + Public + Stable + + 5.7.0 + + + + + com.sap.cloud.sdk + sdk-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + dwc-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + caching + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient4 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient5 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity-scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-destination-service + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-oauth + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-dwc + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-ztis + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience-api + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience4j + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + security + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + servlet-jakarta + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + tenant + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-core + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + rfc + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + datamodel-metadata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator-utility + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-client + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + fluent-result + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + + soap + ${sdk.version} + + + + diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 2b81df758f..d40b1a7756 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -41,9 +41,11 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } mimeType := blob.MimeType() if resourcetypes.MVN_ARTIFACT != resourceType { + log.Debug("not a MVN artifact", "resourceType", resourceType) return nil, nil } - if !mvn.IsMimeTypeSupported(mimeType) { + if mime.MIME_TGZ != mimeType { + log.Debug("not a tarball, can't be a complete mvn GAV", "mimeType", mimeType) return nil, nil } if b.spec.Url == "" { @@ -78,45 +80,47 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } defer blobReader.Close() - switch mimeType { - case mime.MIME_TGZ: - tempFs, err := osfs.NewTempFileSystem() + //switch mimeType { + //case mime.MIME_TGZ: + tempFs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + defer vfs.Cleanup(tempFs) + err = tarutils.ExtractTarToFs(tempFs, blobReader) + // err = tarutils.ExtractTarToFs(tempFs, compression.AutoDecompress(blobReader)) // ??? + if err != nil { + return nil, err + } + files, err := tarutils.FlatListSortedFilesInDir(tempFs, "") + if err != nil { + return nil, err + } + for _, file := range files { + log.Debug("uploading", "file", file) + artifact = artifact.ClassifierExtensionFrom(file) + reader, err := tempFs.Open(file) if err != nil { return nil, err } - defer vfs.Cleanup(tempFs) - err = tarutils.ExtractTarToFs(tempFs, blobReader) - // err = tarutils.ExtractTarToFs(tempFs, compression.AutoDecompress(blobReader)) // ??? - if err != nil { + defer reader.Close() + hash := sha256.New() + if _, err := io.Copy(hash, reader); err != nil { return nil, err } - files, err := tarutils.ListSortedFilesInDir(tempFs, "") + err = deploy(artifact, b.spec.Url, reader, username, password, digest.NewDigest(digest.SHA256, hash)) if err != nil { return nil, err } - for _, file := range files { - log.Debug("uploading", "file", file) - artifact = artifact.ClassifierExtensionFrom(file) - reader, err := tempFs.Open(file) - if err != nil { - return nil, err - } - defer reader.Close() - hash := sha256.New() - if _, err := io.Copy(hash, reader); err != nil { - return nil, err - } - err = deploy(artifact, b.spec.Url, reader, username, password, digest.NewDigest(digest.SHA256, hash)) + } + /* + default: + err = deploy(artifact, b.spec.Url, blobReader, username, password, blob.Digest()) if err != nil { return nil, err } } - default: - err = deploy(artifact, b.spec.Url, blobReader, username, password, blob.Digest()) - if err != nil { - return nil, err - } - } + */ log.Debug("done", "artifact", artifact) return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go index 9784c38f39..7bd4ec73e1 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go @@ -54,11 +54,6 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co ctx.BlobHandlers().Register(NewArtifactHandler(cfg), cpi.ForArtifactType(resourcetypes.MVN_ARTIFACT), - // see artifact.go#IsMimeTypeSupported() - cpi.ForMimeType(mime.MIME_JAR), - cpi.ForMimeType(mime.MIME_JSON), - cpi.ForMimeType(mime.MIME_XML), - cpi.ForMimeType(mime.MIME_GZIP), cpi.ForMimeType(mime.MIME_TGZ), cpi.NewBlobHandlerOptions(olist...), ) @@ -68,9 +63,9 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co func (r *RegistrationHandler) GetHandlers(_ cpi.Context) registrations.HandlerInfos { return registrations.NewLeafHandlerInfo("uploading mvn artifacts", ` -The `+BLOB_HANDLER_NAME+` uploader is able to upload mvn artifacts +The `+BLOB_HANDLER_NAME+` uploader is able to upload mvn artifacts (whole GAV only!) as artifact archive according to the mvn artifact spec. -If registered the default mime type is: `+mime.MIME_JAR+` +If registered the default mime type is: `+mime.MIME_TGZ+` It accepts a plain string for the URL or a config with the following field: 'url': the URL of the mvn repository. diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index f15daf354d..ba839cf9a3 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -2,6 +2,7 @@ package tarutils import ( "archive/tar" + "compress/gzip" "fmt" "io" "io/fs" @@ -218,11 +219,13 @@ func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { return h } -func ListSortedFilesInDir(fs vfs.FileSystem, root string) ([]string, error) { +// FlatListSortedFilesInDir returns a flat list of files in a directory sorted by name. +// Attention: Files with same name but in different sub-paths, will be listed only once!!! +func FlatListSortedFilesInDir(fs vfs.FileSystem, root string) ([]string, error) { var files []string err := vfs.Walk(fs, root, func(path string, info vfs.FileInfo, err error) error { if !info.IsDir() { - files = append(files, path) + files = append(files, info.Name()) } return nil }) @@ -230,9 +233,18 @@ func ListSortedFilesInDir(fs vfs.FileSystem, root string) ([]string, error) { return files, err } -func TarFs(fs vfs.FileSystem, writer io.Writer) error { +func TgzFs(fs vfs.FileSystem, writer io.Writer) error { + zip := gzip.NewWriter(writer) + err := TarFlatFs(fs, zip) + if err != nil { + return err + } + return zip.Close() +} + +func TarFlatFs(fs vfs.FileSystem, writer io.Writer) error { tw := tar.NewWriter(writer) - files, err := ListSortedFilesInDir(fs, "") + files, err := FlatListSortedFilesInDir(fs, "") if err != nil { return err } From 715208da3788918b28c5863fbc2bbd82b4bc7e93 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 13:23:25 +0200 Subject: [PATCH 038/100] lint & test --- .../ocm/accessmethods/mvn/artifact.go | 16 ++-- pkg/contexts/ocm/accessmethods/mvn/method.go | 84 +------------------ .../ocm/accessmethods/mvn/method_test.go | 22 ----- .../handlers/generic/mvn/blobhandler.go | 2 - 4 files changed, 9 insertions(+), 115 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 51297d1139..55013a2840 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -70,7 +70,11 @@ func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { s := strings.TrimPrefix(filename, a.FilePrefix()) if strings.HasPrefix(s, "-") { s = strings.TrimPrefix(s, "-") - a.Classifier = s[:strings.Index(s, ".")] + i := strings.Index(s, ".") + if i < 0 { + panic("no extension after classifier found in filename: " + filename) + } + a.Classifier = s[:i] s = strings.TrimPrefix(s, a.Classifier) } a.Extension = strings.TrimPrefix(s, ".") @@ -95,16 +99,6 @@ func (a *Artifact) MimeType() string { return mime.MIME_TGZ } -/* -func IsMimeTypeSupported(mimeType string) bool { - switch mimeType { - case mime.MIME_JAR, mime.MIME_JSON, mime.MIME_XML, mime.MIME_TGZ, mime.MIME_GZIP: - return true - } - return false -} -*/ - // ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. func ArtifactFromHint(gav string) *Artifact { parts := strings.Split(gav, ":") diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 1c42a25014..0381be8d97 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -174,7 +174,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { defer vfs.Cleanup(tempFs) artifact := a.AsArtifact() - var metadata = meta{} + metadata := meta{} for file, hash := range fileMap { artifact.ClassifierExtensionFrom(file) metadata.Bin = artifact.Url(a.Repository) @@ -228,7 +228,6 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } // pack all downloaded files into a tar.gz file - // tgz, err := blobaccess.NewTempFile("", Type+"-"+artifact.FilePrefix()+"-*.tar.gz", fs) tgz, err := vfs.TempFile(fs, "", Type+"-"+artifact.FilePrefix()+"-*.tar.gz") if err != nil { return nil, err @@ -243,31 +242,16 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } metadata.Bin = "file://" + tgz.Name() + metadata.MimeType = mime.MIME_TGZ metadata.Hash = dw.Digest().Encoded() metadata.HashType = crypto.SHA256 log.Debug("created", "file", metadata.Bin) - /*/ calculate digest for the tar.gz file - file, err := fs.OpenFile(tgz.Name(), vfs.O_RDONLY, vfs.ModePerm) - if err != nil { - return nil, err - } - defer file.Close() - hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { - return nil, err - } - metadata.Hash = fmt.Sprintf("%x", hash.Sum(nil)) - log.Debug("hash", "sum", metadata.Hash) - metadata.HashType = crypto.SHA256 - */ - metadata.MimeType = mime.MIME_TGZ - return &metadata, nil } func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[string]crypto.Hash { - var filtered = make(map[string]crypto.Hash) + filtered := make(map[string]crypto.Hash) for file, hash := range fileMap { if strings.Contains(file, "-"+classifier+".") { filtered[file] = hash @@ -276,18 +260,6 @@ func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[s return filtered } -/* -func (a *AccessSpec) fileListFactory(fs vfs.FileSystem) func() (map[string]crypto.Hash, error) { - return func() (map[string]crypto.Hash, error) { - if strings.HasPrefix(a.BaseUrl(), "file://") { - dir := a.BaseUrl()[7:] - return gavFilesFromDisk(fs, dir) - } - return a.gavOnlineFiles() - } -} -*/ - func (a *AccessSpec) GavFiles(fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { if strings.HasPrefix(a.Repository, "file://") && len(fs) > 0 { dir := a.Repository[7:] @@ -350,7 +322,7 @@ func filesAndHashes(fileList []string) map[string]crypto.Hash { sort.Strings(fileList) // Which hash files are available? - var result = make(map[string]crypto.Hash) + result := make(map[string]crypto.Hash) for _, file := range fileList { if IsArtifact(file) { result[file] = bestAvailableHash(fileList, file) @@ -374,36 +346,6 @@ func bestAvailableHash(list []string, filename string) crypto.Hash { //////////////////////////////////////////////////////////////////////////////// -/* -type project struct { - GroupId string `xml:"groupId"` - ArtifactId string `xml:"artifactId"` - Version string `xml:"version"` - Classifier string `xml:"packaging"` -} - -func readPom(url string, fs vfs.FileSystem) (*project, error) { - reader, err := getReader(url, fs) - if err != nil { - return nil, err - } - buf := &bytes.Buffer{} - _, err = io.Copy(buf, io.LimitReader(reader, 200000)) - if err != nil { - return nil, errors.Wrapf(err, "cannot get version metadata for %s", url) - } - var pom project - err = xml.Unmarshal(buf.Bytes(), &pom) - if err != nil { - return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", url) - } - if pom.Classifier == "" { - pom.Classifier = "jar" - } - return &pom, nil -} -*/ - // getStringData reads all data from the given URL and returns it as a string. func getStringData(url string, fs vfs.FileSystem) (string, error) { r, err := getReader(url, fs) @@ -417,24 +359,6 @@ func getStringData(url string, fs vfs.FileSystem) (string, error) { return string(b), nil } -/*/ getBestHashValue returns the best hash value (SHA-512, SHA-256, SHA-1, MD5) for the given artifact (POM, JAR, ...). -func getBestHashValue(url string, fs vfs.FileSystem) (crypto.Hash, string, error) { - arr := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} - log := logging.Context().Logger(identity.REALM) - for i := 0; i < len(arr); i++ { - v, err := getStringData(url+hashUrlExt(arr[i]), fs) - if v != "" { - log.Debug("found hash ", "url", url+hashUrlExt(arr[i])) - return arr[i], v, err - } - if err != nil { - log.Debug("hash file not found", "url", url+hashUrlExt(arr[i])) - } - } - return 0, "", errors.New("no hash value found") -} -*/ - // hashUrlExt returns the 'maven' hash extension for the given hash. // Maven usually uses sha1, sha256, sha512, md5 instead of SHA-1, SHA-256, SHA-512, MD5. func hashUrlExt(h crypto.Hash) string { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index b0f3ed6de6..390e967925 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -57,16 +57,6 @@ var _ = Describe("Method", func() { It("GetPackageMeta - com.sap.cloud.sdk", func() { acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") - - /* - repos to test with: - - https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6/ // bin + tar.gz etc. - - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx - - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! - - https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/ // one single pom only! - - https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/ // jar only! - */ - meta, err := acc.GetPackageMeta(ocm.DefaultContext()) Expect(err).ToNot(HaveOccurred()) Expect(meta.Bin).To(HavePrefix("file://")) @@ -76,18 +66,6 @@ var _ = Describe("Method", func() { Expect(meta.HashType).To(Equal(crypto.SHA256)) }) - /*/ works only internal - It("GetPackageMeta - int.repositories.cloud.sap: hello-ocm", func() { - acc := mvn.New("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test", "open-component-model", "hello-ocm", "0.0.1") - meta, err := acc.GetPackageMeta(ocm.DefaultContext()) - Expect(err).ToNot(HaveOccurred()) - Expect(meta.Bin).To(Equal("https://int.repositories.cloud.sap/artifactory/ocm-mvn-test/open-component-model/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) - Expect(meta.Hash).To(Equal("")) - Expect(meta.HashType).To(Equal(crypto.Hash(0))) - }) - - */ - It("accesses local artifact", func() { acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") m := Must(acc.AccessMethod(cv)) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index d40b1a7756..a1884c4dc0 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -80,8 +80,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } defer blobReader.Close() - //switch mimeType { - //case mime.MIME_TGZ: tempFs, err := osfs.NewTempFileSystem() if err != nil { return nil, err From 6a9eb19307ebe6962ee92e07a3c4a7bfd4146d22 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 14:31:15 +0200 Subject: [PATCH 039/100] fix tests --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- .../ocm/accessmethods/mvn/method_test.go | 13 +- .../5.7.0/sdk-modules-bom-5.7.0.pom.sha1 | 1 + .../5.7.0/sdk-modules-bom-5.7.0.pom | 210 ------------------ .../handlers/generic/mvn/blobhandler.go | 10 +- pkg/utils/tarutils/pack.go | 14 +- 6 files changed, 16 insertions(+), 234 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 delete mode 100644 pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 0381be8d97..10a58191c1 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -269,7 +269,7 @@ func (a *AccessSpec) GavFiles(fs ...vfs.FileSystem) (map[string]crypto.Hash, err } func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, error) { - files, err := tarutils.FlatListSortedFilesInDir(fs, dir) + files, err := tarutils.ListSortedFilesInDir(fs, dir, true) if err != nil { return nil, err } diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 390e967925..3a5bfc58cc 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -17,7 +17,7 @@ import ( ) const ( - mvnPATH = "/testdata/success" + mvnPATH = "/testdata/.m2/repository" FAILPATH = "/testdata" ) @@ -62,7 +62,7 @@ var _ = Describe("Method", func() { Expect(meta.Bin).To(HavePrefix("file://")) Expect(meta.Bin).To(ContainSubstring("mvn-sdk-modules-bom-5.7.0-")) Expect(meta.Bin).To(HaveSuffix(".tar.gz")) - Expect(meta.Hash).To(Equal("217feb1e7490015dd0a2b231b9cea45804df3d2a9b37287ac861bb45b8c0de55")) + Expect(meta.Hash).To(Equal("345fe2e640663c3cd6ac87b7afb92e1c934f665f75ddcb9555bc33e1813ef00b")) Expect(meta.HashType).To(Equal(crypto.SHA256)) }) @@ -82,8 +82,8 @@ var _ = Describe("Method", func() { break } } - Expect(dr.Size()).To(Equal(int64(10))) - Expect(dr.Digest().String()).To(Equal("SHA-1:e727ef4792a349c485d893e60874475a54f24b97")) + Expect(dr.Size()).To(Equal(int64(1109))) + Expect(dr.Digest().String()).To(Equal("SHA-1:4ee125ffe4f7690588833f1217a13cc741e4df5f")) }) It("accesses local artifact with extension", func() { @@ -114,9 +114,4 @@ var _ = Describe("Method", func() { _, err := m.Reader() Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found b3242b8c31f8ce14f729b8fd132ac77bc4bc5bf7"))) }) - - FIt("NewDigestWriterWithHash", func() { - w := iotools.NewDigestWriterWithHash(crypto.SHA1, nil) - Expect(w).ToNot(BeNil()) - }) }) diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 new file mode 100644 index 0000000000..35f63a2e1f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 @@ -0,0 +1 @@ +34ccdeb9c008f8aaef90873fc636b09d3ae5c709 \ No newline at end of file diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom deleted file mode 100644 index b3baaee32f..0000000000 --- a/pkg/contexts/ocm/accessmethods/mvn/testdata/success/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom +++ /dev/null @@ -1,210 +0,0 @@ - - - 4.0.0 - com.sap.cloud.sdk - sdk-modules-bom - 5.7.0 - pom - SAP Cloud SDK - Modules BOM - Bill of Materials (BOM) of the SAP Cloud SDK modules. - https://sap.github.io/cloud-sdk/docs/java/getting-started - - SAP SE - https://www.sap.com - - - - The Apache Software License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - SAP - cloudsdk@sap.com - SAP SE - https://www.sap.com - - - - - - - - UTF-8 - Public - Stable - - 5.7.0 - - - - - com.sap.cloud.sdk - sdk-core - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - scp-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - dwc-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-core - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - caching - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-connectivity - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-apache-httpclient4 - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-apache-httpclient5 - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - cloudplatform-connectivity-scp-cf - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-destination-service - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-oauth - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-dwc - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - connectivity-ztis - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience-api - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - resilience4j - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - security - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - servlet-jakarta - ${sdk.version} - - - com.sap.cloud.sdk.cloudplatform - tenant - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - s4hana-core - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - s4hana-connectivity - ${sdk.version} - - - com.sap.cloud.sdk.s4hana - rfc - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - datamodel-metadata-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-generator-utility - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-client - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-v4-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - odata-v4-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - openapi-core - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - openapi-generator - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - fluent-result - ${sdk.version} - - - com.sap.cloud.sdk.datamodel - - soap - ${sdk.version} - - - - diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index a1884c4dc0..4de8794d36 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -90,7 +90,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi if err != nil { return nil, err } - files, err := tarutils.FlatListSortedFilesInDir(tempFs, "") + files, err := tarutils.ListSortedFilesInDir(tempFs, "", false) if err != nil { return nil, err } @@ -111,14 +111,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return nil, err } } - /* - default: - err = deploy(artifact, b.spec.Url, blobReader, username, password, blob.Digest()) - if err != nil { - return nil, err - } - } - */ log.Debug("done", "artifact", artifact) return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index ba839cf9a3..faf2ddd016 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -219,13 +219,17 @@ func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { return h } -// FlatListSortedFilesInDir returns a flat list of files in a directory sorted by name. -// Attention: Files with same name but in different sub-paths, will be listed only once!!! -func FlatListSortedFilesInDir(fs vfs.FileSystem, root string) ([]string, error) { +// ListSortedFilesInDir returns a list of files in a directory sorted by name. +// Attention: If 'flat == true', files with same name but in different sub-paths, will be listed only once!!! +func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string, error) { var files []string err := vfs.Walk(fs, root, func(path string, info vfs.FileInfo, err error) error { if !info.IsDir() { - files = append(files, info.Name()) + pathOrName := path + if flat { + pathOrName = info.Name() + } + files = append(files, pathOrName) } return nil }) @@ -244,7 +248,7 @@ func TgzFs(fs vfs.FileSystem, writer io.Writer) error { func TarFlatFs(fs vfs.FileSystem, writer io.Writer) error { tw := tar.NewWriter(writer) - files, err := FlatListSortedFilesInDir(fs, "") + files, err := ListSortedFilesInDir(fs, "", true) if err != nil { return err } From 57ec467ab6c608c2f42166fb2822085904ad8f7c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 16:18:09 +0200 Subject: [PATCH 040/100] reset Classifier --- pkg/contexts/ocm/accessmethods/mvn/artifact.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 55013a2840..3fcd346476 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -76,6 +76,8 @@ func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { } a.Classifier = s[:i] s = strings.TrimPrefix(s, a.Classifier) + } else { + a.Classifier = "" } a.Extension = strings.TrimPrefix(s, ".") return a From 689ceb224bd974e0bdc15ba3867f1b3d972141f9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 16:19:02 +0200 Subject: [PATCH 041/100] ExtractTgzToTempFs --- .../ocm/accessmethods/mvn/method_test.go | 2 +- .../handlers/generic/mvn/blobhandler.go | 8 +----- pkg/utils/tarutils/extract.go | 25 +++++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index 3a5bfc58cc..ceafc56da4 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -21,7 +21,7 @@ const ( FAILPATH = "/testdata" ) -var _ = Describe("Method", func() { +var _ = Describe("accessmethods.mvn.AccessSpec tests", func() { var env *Builder var cv ocm.ComponentVersionAccess diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 4de8794d36..1fef7c1a6b 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -9,7 +9,6 @@ import ( "net/http" "strings" - "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/opencontainers/go-digest" @@ -80,16 +79,11 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } defer blobReader.Close() - tempFs, err := osfs.NewTempFileSystem() + tempFs, err := tarutils.ExtractTgzToTempFs(blobReader) if err != nil { return nil, err } defer vfs.Cleanup(tempFs) - err = tarutils.ExtractTarToFs(tempFs, blobReader) - // err = tarutils.ExtractTarToFs(tempFs, compression.AutoDecompress(blobReader)) // ??? - if err != nil { - return nil, err - } files, err := tarutils.ListSortedFilesInDir(tempFs, "", false) if err != nil { return nil, err diff --git a/pkg/utils/tarutils/extract.go b/pkg/utils/tarutils/extract.go index b6769ff39d..a917196cb0 100644 --- a/pkg/utils/tarutils/extract.go +++ b/pkg/utils/tarutils/extract.go @@ -51,6 +51,31 @@ func ExtractTarToFs(fs vfs.FileSystem, in io.Reader) error { return err } +// UnzipTarToFs tries to decompress the input stream and then writes the tar stream to a filesystem. +func UnzipTarToFs(fs vfs.FileSystem, in io.Reader) error { + r, _, err := compression.AutoDecompress(in) + if err != nil { + return err + } + defer r.Close() + err = ExtractTarToFs(fs, r) + if err != nil { + return err + } + return err +} + +// ExtractTgzToTempFs extracts a tar.gz archive to a temporary filesystem. +// You should call vfs.Cleanup on the returned filesystem to clean up the temporary files. +func ExtractTgzToTempFs(in io.Reader) (vfs.FileSystem, error) { + fs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + + return fs, UnzipTarToFs(fs, in) +} + func ExtractTarToFsWithInfo(fs vfs.FileSystem, in io.Reader) (fcnt int64, bcnt int64, err error) { tr := tar.NewReader(in) for { From 28090442a7608a2d9ebc48d242a7dff719528db4 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 23 Apr 2024 17:18:46 +0200 Subject: [PATCH 042/100] uploader --- .../ocm/accessmethods/mvn/artifact_test.go | 32 -------------- .../handlers/generic/mvn/blobhandler.go | 42 +++++++++++------- .../handlers/generic/mvn/blobhandler_test.go | 44 +++++++++++++++++++ 3 files changed, 71 insertions(+), 47 deletions(-) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go index 5b63738b58..734f750add 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go @@ -50,36 +50,4 @@ var _ = Describe("Maven Test Environment", func() { Expect(artifact.Extension).To(Equal("xml")) Expect(artifact.FileName()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) }) - - /* - It("Body", func() { - resp := `{ "repo" : "ocm-mvn-test", - "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", - "created" : "2024-04-11T15:09:28.920Z", - "createdBy" : "john.doe", - "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", - "mimeType" : "application/java-archive", - "size" : "1792", - "checksums" : { - "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", - "md5" : "6cb7520b65d820b3b35773a8daa8368e", - "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, - "originalChecksums" : { - "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, - "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` - var body Body - err := json.Unmarshal([]byte(resp), &body) - Expect(err).To(BeNil()) - Expect(body.Repo).To(Equal("ocm-mvn-test")) - Expect(body.FileName).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) - Expect(body.MimeType).To(Equal("application/java-archive")) - Expect(body.Size).To(Equal("1792")) - Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) - Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) - Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) - Expect(body.Checksums["sha512"]).To(Equal("")) - }) - */ }) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 1fef7c1a6b..96d8959d06 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -2,7 +2,7 @@ package mvn import ( "context" - "crypto/sha256" + "crypto" "encoding/json" "fmt" "io" @@ -10,7 +10,6 @@ import ( "strings" "github.com/mandelsoft/vfs/pkg/vfs" - "github.com/opencontainers/go-digest" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" @@ -96,11 +95,11 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return nil, err } defer reader.Close() - hash := sha256.New() - if _, err := io.Copy(hash, reader); err != nil { + hash, err := GetHash(crypto.SHA256, tempFs, file) + if err != nil { return nil, err } - err = deploy(artifact, b.spec.Url, reader, username, password, digest.NewDigest(digest.SHA256, hash)) + err = deploy(artifact, b.spec.Url, reader, username, password, crypto.SHA256, hash) if err != nil { return nil, err } @@ -110,12 +109,25 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil } -func ChecksumHeader(digest digest.Digest) string { - a := digest.Algorithm().String() - return "X-Checksum-" + strings.ToUpper(a[:1]) + a[1:] +func GetHash(hash crypto.Hash, fs vfs.FileSystem, path string) (string, error) { + reader, err := fs.Open(path) + if err != nil { + return "", err + } + defer reader.Close() + digest := hash.New() + if _, err := io.Copy(digest, reader); err != nil { + return "", err + } + return fmt.Sprintf("%x", digest.Sum(nil)), nil +} + +func ChecksumHeader(hash crypto.Hash) string { + a := strings.ReplaceAll(hash.String(), "-", "") + return "X-Checksum-" + a[:1] + strings.ToLower(a[1:]) } -func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, digest digest.Digest) error { +func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, hash crypto.Hash, digest string) error { // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis // vs. https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifacts-from-archive // Headers: X-Checksum-Deploy: true, X-Checksum-Sha1: sha1Value, X-Checksum-Sha256: sha256Value, X-Checksum: checksum value (type is resolved by length) @@ -124,8 +136,8 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username s return err } req.SetBasicAuth(username, password) - req.Header.Set("X-Checksum", digest.Encoded()) - req.Header.Set(ChecksumHeader(digest), digest.Encoded()) + req.Header.Set("X-Checksum", digest) + req.Header.Set(ChecksumHeader(hash), digest) // Execute the request client := &http.Client{} @@ -156,13 +168,13 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username s return err } - remoteDigest := artifactBody.Checksums[string(digest.Algorithm())] + remoteDigest := artifactBody.Checksums[strings.ReplaceAll(strings.ToLower(hash.String()), "-", "")] if remoteDigest == "" { - log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", digest.Algorithm()) - } else if remoteDigest != digest.Encoded() { + log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", hash) + } else if remoteDigest != digest { return fmt.Errorf("failed to upload artifact: checksums do not match") } - log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest.Encoded()) + log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest) return nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go new file mode 100644 index 0000000000..dbd3ba2c06 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go @@ -0,0 +1,44 @@ +package mvn_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" +) + +var _ = Describe("blobhandler generic mvn tests", func() { + + It("Unmarshal deploy response Body", func() { + resp := `{ "repo" : "ocm-mvn-test", + "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "created" : "2024-04-11T15:09:28.920Z", + "createdBy" : "john.doe", + "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "mimeType" : "application/java-archive", + "size" : "1792", + "checksums" : { + "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", + "md5" : "6cb7520b65d820b3b35773a8daa8368e", + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "originalChecksums" : { + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` + var body mvn.Body + err := json.Unmarshal([]byte(resp), &body) + Expect(err).To(BeNil()) + Expect(body.Repo).To(Equal("ocm-mvn-test")) + //Expect(body.FileName).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.MimeType).To(Equal("application/java-archive")) + Expect(body.Size).To(Equal("1792")) + Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) + Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) + Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) + Expect(body.Checksums["sha512"]).To(Equal("")) + }) + +}) From a8aa938e19a055136032dae2a0c3a8b349d73689 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 25 Apr 2024 14:46:00 +0200 Subject: [PATCH 043/100] use mimeutils --- pkg/contexts/ocm/accessmethods/mvn/artifact.go | 15 ++++----------- pkg/mimeutils/type.go | 15 ++++++++++----- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 3fcd346476..72c0fbfc6b 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/mimeutils" ) // Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. @@ -86,17 +87,9 @@ func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { // MimeType returns the MIME type of the Maven Artifact based on the file extension. // Default is application/x-tgz. func (a *Artifact) MimeType() string { - switch a.Extension { - case "jar": - return mime.MIME_JAR - case "json", "module": - return mime.MIME_JSON - case "pom", "xml": - return mime.MIME_XML - case "tar.gz": - return mime.MIME_TGZ - case "zip": - return mime.MIME_GZIP + m := mimeutils.TypeByExtension("." + a.Extension) + if m != "" { + return m } return mime.MIME_TGZ } diff --git a/pkg/mimeutils/type.go b/pkg/mimeutils/type.go index cbd4c8bad1..8854545ae9 100644 --- a/pkg/mimeutils/type.go +++ b/pkg/mimeutils/type.go @@ -83,11 +83,16 @@ var builtinTypesLower = map[string]string{ ".webp": "image/webp", ".xml": "text/xml; charset=utf-8", // added entries - ".txt": ocmmime.MIME_TEXT, - ".yaml": ocmmime.MIME_YAML_OFFICIAL, - ".gzip": ocmmime.MIME_GZIP, - ".tar": ocmmime.MIME_TAR, - ".tgz": ocmmime.MIME_TGZ, + ".txt": ocmmime.MIME_TEXT, + ".yaml": ocmmime.MIME_YAML_OFFICIAL, + ".gzip": ocmmime.MIME_GZIP, + ".tar": ocmmime.MIME_TAR, + ".tgz": ocmmime.MIME_TGZ, + ".tar.gz": ocmmime.MIME_TGZ, + ".pom": ocmmime.MIME_XML, + ".zip": ocmmime.MIME_GZIP, + ".jar": ocmmime.MIME_JAR, + ".module": ocmmime.MIME_JSON, // gradle module metadata } var once sync.Once // guards initMime From 7cafd3aeb8ad39686ba7eee27336493f4b90fb56 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 12:44:31 +0200 Subject: [PATCH 044/100] adress review comments --- pkg/contexts/ocm/accessmethods/mvn/README.md | 89 +++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md index 8b166fda6e..79cc8522b0 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/README.md +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -1,15 +1,16 @@ -# `mvn` - Java packages (jar) in a Maven (mvn) repository (e.g. mvnrepository.com) +# `mvn` - Maven artifacts (Java packages, jars) in a Maven (mvn) repository (e.g. mvnrepository.com) ### Synopsis ``` type: mvn/v1 ``` -Provided blobs use the following media type: `application/x-jar` +Provided blobs use the following media type: `application/x-tgz` ### Description -This method implements the access of a Java package from a Maven (mvn) repository. +This method implements the access of a resource hosted by a maven repository or a +complete resource set denoted by a GAV. ### Specification Versions @@ -34,3 +35,85 @@ The type specific specification fields are: - **`version`** *string* The version name of the Maven (mvn) artifact + +- **`classifier`** *string* + + The optional classifier of the Maven (mvn) artifact + +- **`extension`** *string* + + The optional extension of the Maven (mvn) artifact + +If classifier/extension is given a dedicated resource is described, +otherwise the complete resource set described by a GAV. +Only complete resource sets can be uploaded again to a Maven repository. + +#### Examples + +##### Complete resource set denoted by a GAV + +```yaml +name: acme.org/complete/gav +version: 0.0.1 +provider: + name: acme.org +resources: + - name: java-sap-vcap-services + type: mvnArtifact + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: com.sap.cloud.environment.servicebinding + artifactId: java-sap-vcap-services + version: 0.10.4 +``` + +##### Single pom.xml file + +This can't be uploaded again into a Maven repository, but it can be used to describe the dependencies of a project. +The mime type will be `application/xml`. + +```yaml +name: acme.org/single/pom +version: 0.0.1 +provider: + name: acme.org +resources: + - name: sap-cloud-sdk + type: pom + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: com.sap.cloud.sdk + artifactId: sdk-modules-bom + version: 5.7.0 + classifier: '' + extension: pom +``` + +##### Single binary file + +In case you want to download and install maven itself, you can use the following example. +This can't be uploaded again into a Maven repository. +The mime type will be `application/gzip`. + +```yaml +name: acme.org/bin/zip +version: 0.0.1 +provider: + name: acme.org +resources: + - name: maven + type: bin + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: org.apache.maven + artifactId: apache-maven + version: 3.9.6 + classifier: bin + extension: zip +``` From 6a69014cb45c9876d91c492f2490d34f9fd89514 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:29:26 +0200 Subject: [PATCH 045/100] hash Reader + Writer --- pkg/iotools/hashReaderWriter.go | 137 +++++++++++++++++++++++++++ pkg/iotools/hashReaderWriter_test.go | 74 +++++++++++++++ pkg/iotools/suite_test.go | 13 +++ 3 files changed, 224 insertions(+) create mode 100644 pkg/iotools/hashReaderWriter.go create mode 100644 pkg/iotools/hashReaderWriter_test.go create mode 100644 pkg/iotools/suite_test.go diff --git a/pkg/iotools/hashReaderWriter.go b/pkg/iotools/hashReaderWriter.go new file mode 100644 index 0000000000..375c75376b --- /dev/null +++ b/pkg/iotools/hashReaderWriter.go @@ -0,0 +1,137 @@ +package iotools + +import ( + "crypto" + "fmt" + "hash" + "io" + "strings" +) + +type HashReader struct { + reader io.Reader + hashMap map[crypto.Hash]hash.Hash +} + +func NewHashReader(delegate io.Reader, algorithms ...crypto.Hash) *HashReader { + newInstance := HashReader{ + reader: delegate, + hashMap: initMap(algorithms), + } + return &newInstance +} + +func (h *HashReader) Read(buf []byte) (int, error) { + c, err := h.reader.Read(buf) + return write(h, c, buf, err) +} + +func (h *HashReader) GetString(algorithm crypto.Hash) string { + return getString(h, algorithm) +} + +func (h *HashReader) GetBytes(algorithm crypto.Hash) []byte { + return getBytes(h, algorithm) +} + +func (h *HashReader) HttpHeader() map[string]string { + return httpHeader(h) +} + +func (h *HashReader) hashes() map[crypto.Hash]hash.Hash { + return h.hashMap +} + +func (h *HashReader) ReadAll() ([]byte, error) { + return io.ReadAll(h.reader) +} + +func (h *HashReader) CalcHashes() (int64, error) { + return io.Copy(io.Discard, h.reader) +} + +//////////////////////////////////////////////////////////////////////////////// + +type HashWriter struct { + writer io.Writer + hashMap map[crypto.Hash]hash.Hash +} + +func NewHashWriter(w io.Writer, algorithms ...crypto.Hash) *HashWriter { + newInstance := HashWriter{ + writer: w, + hashMap: initMap(algorithms), + } + return &newInstance +} + +func (h *HashWriter) Write(buf []byte) (int, error) { + c, err := h.writer.Write(buf) + return write(h, c, buf, err) +} + +func (h *HashWriter) GetString(algorithm crypto.Hash) string { + return getString(h, algorithm) +} + +func (h *HashWriter) GetBytes(algorithm crypto.Hash) []byte { + return getBytes(h, algorithm) +} + +func (h *HashWriter) HttpHeader() map[string]string { + return httpHeader(h) +} + +func (h *HashWriter) hashes() map[crypto.Hash]hash.Hash { + return h.hashMap +} + +//////////////////////////////////////////////////////////////////////////////// + +type hashes interface { + hashes() map[crypto.Hash]hash.Hash +} + +func getString(h hashes, algorithm crypto.Hash) string { + return fmt.Sprintf("%x", getBytes(h, algorithm)) +} + +func getBytes(h hashes, algorithm crypto.Hash) []byte { + hash := h.hashes()[algorithm] + if hash != nil { + return hash.Sum(nil) + } + return nil +} + +func httpHeader(h hashes) map[string]string { + headers := make(map[string]string, len(h.hashes())) + for algorithm, _ := range h.hashes() { + headers[headerName(algorithm)] = getString(h, algorithm) + } + return headers +} + +func initMap(algorithms []crypto.Hash) map[crypto.Hash]hash.Hash { + hashMap := make(map[crypto.Hash]hash.Hash, len(algorithms)) + for _, algorithm := range algorithms { + hashMap[algorithm] = algorithm.New() + } + return hashMap +} + +func write(h hashes, c int, buf []byte, err error) (int, error) { + if err == nil && c > 0 { + for _, hash := range h.hashes() { + hash.Write(buf[:c]) + } + } + return c, err +} + +//////////////////////////////////////////////////////////////////////////////// + +func headerName(hash crypto.Hash) string { + a := strings.ReplaceAll(hash.String(), "-", "") + return "X-Checksum-" + a[:1] + strings.ToLower(a[1:]) +} diff --git a/pkg/iotools/hashReaderWriter_test.go b/pkg/iotools/hashReaderWriter_test.go new file mode 100644 index 0000000000..cc535f72e5 --- /dev/null +++ b/pkg/iotools/hashReaderWriter_test.go @@ -0,0 +1,74 @@ +package iotools_test + +import ( + "bytes" + "crypto" + "io" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/iotools" +) + +var _ = Describe("Hash Reader Writer tests", func() { + + It("Ensure interface implementation", func() { + var _ io.Reader = &iotools.HashReader{} + var _ io.Reader = (*iotools.HashReader)(nil) + var _ io.Reader = new(iotools.HashReader) + + var _ io.Writer = &iotools.HashWriter{} + var _ io.Writer = (*iotools.HashWriter)(nil) + var _ io.Writer = new(iotools.HashWriter) + }) + + It("test HashWriter", func() { + s := "Hello Hash!" + var b bytes.Buffer + hr := iotools.NewHashWriter(io.Writer(&b)) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + b.Reset() + + w := io.Writer(&b) + hr = iotools.NewHashWriter(w, crypto.SHA1) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + b.Reset() + + hr = iotools.NewHashWriter(io.Writer(&b), crypto.SHA1, crypto.MD5) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + }) + + It("test HashReader", func() { + s := "Hello Hash!" + hr := iotools.NewHashReader(strings.NewReader(s)) + buf := make([]byte, len(s)) + hr.Read(buf) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(string(buf)).To(Equal(s)) + + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1) + hr.Read(buf) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1, crypto.MD5) + hr.Read(buf) + Expect(hr.GetBytes(crypto.SHA256)).To(BeNil()) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + }) + +}) diff --git a/pkg/iotools/suite_test.go b/pkg/iotools/suite_test.go new file mode 100644 index 0000000000..f2c3e11709 --- /dev/null +++ b/pkg/iotools/suite_test.go @@ -0,0 +1,13 @@ +package iotools_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "iotools Test Suite") +} From bf0dfa87fdc0b855e463a9bada9a9adc2a5aa98a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:29:52 +0200 Subject: [PATCH 046/100] documentation --- pkg/utils/tarutils/pack.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index faf2ddd016..7f5e137d06 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -202,6 +202,8 @@ func SimpleTarHeader(fs vfs.FileSystem, filepath string) (*tar.Header, error) { return RegularFileInfoHeader(info), nil } +// RegularFileInfoHeader creates a tar header for a regular file (`tar.TypeReg`). +// Besides name and size, the other header fields are set to default values (`fs.ModePerm`, 0, "", `time.Unix(0,0)`). func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { h := &tar.Header{ Typeflag: tar.TypeReg, @@ -237,6 +239,8 @@ func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string, return files, err } +// TgzFs creates a tar.gz archive from a filesystem with all files being in the root of the zipped archive. +// The writer is closed after the archive is written. The TAR-headers are normalized, see RegularFileInfoHeader func TgzFs(fs vfs.FileSystem, writer io.Writer) error { zip := gzip.NewWriter(writer) err := TarFlatFs(fs, zip) From ecc7bbb73af2b7ea1493ff6bd5be383cd07795d7 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:32:12 +0200 Subject: [PATCH 047/100] documentation + rename IsArtifact -> IsResource --- pkg/contexts/ocm/accessmethods/mvn/artifact.go | 8 ++++++-- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 72c0fbfc6b..cddd0a061d 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -34,7 +34,8 @@ func (a *Artifact) GavPath() string { return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version } -// FileName returns the Maven Artifact's name with classifier and extension. +// FileName returns the Maven Artifact's GAV-name with classifier and extension. +// Which is equal to the URL-path of the artifact in the repository. // Default extension is jar. func (a *Artifact) FileName() string { path := a.GavPath() + "/" + a.FilePrefix() @@ -67,7 +68,9 @@ func (a *Artifact) Purl() string { return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version } +// ClassifierExtensionFrom extracts the classifier and extension from the filename (without any path prefix). func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { + // TODO should work with pos (path.Basename)?!? s := strings.TrimPrefix(filename, a.FilePrefix()) if strings.HasPrefix(s, "-") { s = strings.TrimPrefix(s, "-") @@ -114,7 +117,8 @@ func ArtifactFromHint(gav string) *Artifact { return artifact } -func IsArtifact(fileName string) bool { +// IsResource returns true if the filename is not a checksum or signature file. +func IsResource(fileName string) bool { if strings.HasSuffix(fileName, ".asc") { return false } diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 10a58191c1..888cf44005 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -324,7 +324,7 @@ func filesAndHashes(fileList []string) map[string]crypto.Hash { // Which hash files are available? result := make(map[string]crypto.Hash) for _, file := range fileList { - if IsArtifact(file) { + if IsResource(file) { result[file] = bestAvailableHash(fileList, file) logger.Debug("found", "file", file) } From f002ae655729843e4f9ec40c34656f59a83fa068 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:32:42 +0200 Subject: [PATCH 048/100] add more options: ClassifierOption + ExtensionOption --- pkg/contexts/ocm/accessmethods/options/standard.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/contexts/ocm/accessmethods/options/standard.go b/pkg/contexts/ocm/accessmethods/options/standard.go index 00a0f63e61..80b2304e53 100644 --- a/pkg/contexts/ocm/accessmethods/options/standard.go +++ b/pkg/contexts/ocm/accessmethods/options/standard.go @@ -58,3 +58,9 @@ var HTTPRedirectOption = RegisterOption(NewBoolOptionType("noredirect", "http re // CommentOption. var CommentOption = RegisterOption(NewStringOptionType("comment", "comment field value")) + +// ClassifierOption the optional classifier of a maven resource +var ClassifierOption = RegisterOption(NewStringOptionType("accessClassifier", "classifier")) + +// ExtensionOption the optional extension of a maven resource +var ExtensionOption = RegisterOption(NewStringOptionType("accessExtension", "extension name")) From 09030b96fa3f81f559975083596bdd3768aa2eb9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:33:18 +0200 Subject: [PATCH 049/100] add classifier + extension --- pkg/contexts/ocm/accessmethods/mvn/cli.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/cli.go b/pkg/contexts/ocm/accessmethods/mvn/cli.go index df859a541a..dd81479a6b 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/cli.go +++ b/pkg/contexts/ocm/accessmethods/mvn/cli.go @@ -12,6 +12,9 @@ func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { options.GroupOption, options.PackageOption, options.VersionOption, + // optional + options.ClassifierOption, + options.ExtensionOption, ) } @@ -20,6 +23,9 @@ func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { flagsets.AddFieldByOptionP(opts, options.GroupOption, config, "groupId") flagsets.AddFieldByOptionP(opts, options.PackageOption, config, "artifactId") flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "version") + // optional + flagsets.AddFieldByOptionP(opts, options.ClassifierOption, config, "classifier") + flagsets.AddFieldByOptionP(opts, options.ExtensionOption, config, "extension") return nil } @@ -45,4 +51,12 @@ The type specific specification fields are: - **version** *string* The version name of the Maven (mvn) artifact + +- **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + +- **extension** *string* + + The optional extension of the Maven (mvn) artifact ` From 8f09903a8e911cbf0ccea02895c07e35264fbeb7 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:34:18 +0200 Subject: [PATCH 050/100] split up the integration tests --- .../ocm/accessmethods/mvn/integration_test.go | 79 +++++++++++++++++++ .../ocm/accessmethods/mvn/method_test.go | 37 +-------- 2 files changed, 80 insertions(+), 36 deletions(-) create mode 100644 pkg/contexts/ocm/accessmethods/mvn/integration_test.go diff --git a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go new file mode 100644 index 0000000000..f8fd99b45b --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go @@ -0,0 +1,79 @@ +//go:build integration +// +build integration + +package mvn_test + +import ( + "crypto" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + . "github.com/open-component-model/ocm/pkg/env" + . "github.com/open-component-model/ocm/pkg/env/builder" + "github.com/open-component-model/ocm/pkg/mime" + . "github.com/open-component-model/ocm/pkg/testutils" +) + +var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() { + var env *Builder + var cv ocm.ComponentVersionAccess + + BeforeEach(func() { + env = NewBuilder(TestData()) + cv = &cpi.DummyComponentVersionAccess{env.OCMContext()} + }) + + AfterEach(func() { + env.Cleanup() + }) + + // https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0 + It("one single pom only", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + files, err := acc.GavFiles() + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(1)) + Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) + }) + It("GetPackageMeta - com.sap.cloud.sdk", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(HavePrefix("file://")) + Expect(meta.Bin).To(ContainSubstring("mvn-sdk-modules-bom-5.7.0-")) + Expect(meta.Bin).To(HaveSuffix(".tar.gz")) + Expect(meta.Hash).To(Equal("345fe2e640663c3cd6ac87b7afb92e1c934f665f75ddcb9555bc33e1813ef00b")) + Expect(meta.HashType).To(Equal(crypto.SHA256)) + }) + + // https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6 + It("apache-maven, with bin + tar.gz etc.", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") + Expect(acc).ToNot(BeNil()) + Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) + files, err := acc.GavFiles() + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(8)) + Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) + Expect(files["apache-maven-3.9.6.pom"]).To(Equal(crypto.SHA1)) + }) + + // https://repo1.maven.org/maven2/com/sap/cloud/environment/servicebinding/java-sap-vcap-services/0.10.4 + It("accesses local artifact", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.environment.servicebinding", "java-sap-vcap-services", "0.10.4") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(HavePrefix("file://")) + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + /* manually also tested with repos: + - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx + - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! + */ + }) +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index ceafc56da4..e82c105acb 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -21,7 +21,7 @@ const ( FAILPATH = "/testdata" ) -var _ = Describe("accessmethods.mvn.AccessSpec tests", func() { +var _ = Describe("local accessmethods.mvn.AccessSpec tests", func() { var env *Builder var cv ocm.ComponentVersionAccess @@ -34,44 +34,11 @@ var _ = Describe("accessmethods.mvn.AccessSpec tests", func() { env.Cleanup() }) - It("get packaging", func() { - acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") - files, err := acc.GavFiles() - Expect(err).ToNot(HaveOccurred()) - Expect(files).To(HaveLen(1)) - Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) - }) - - It("get packaging", func() { - acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") - Expect(acc).ToNot(BeNil()) - Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) - files, err := acc.GavFiles() - Expect(err).ToNot(HaveOccurred()) - Expect(files).To(HaveLen(8)) - - //Expect(files[0]).To(Equal("sdk-modules-bom-5.7.0.pom")) - Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) - Expect(files["apache-maven-3.9.6.pom"]).To(Equal(crypto.SHA1)) - }) - - It("GetPackageMeta - com.sap.cloud.sdk", func() { - acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") - meta, err := acc.GetPackageMeta(ocm.DefaultContext()) - Expect(err).ToNot(HaveOccurred()) - Expect(meta.Bin).To(HavePrefix("file://")) - Expect(meta.Bin).To(ContainSubstring("mvn-sdk-modules-bom-5.7.0-")) - Expect(meta.Bin).To(HaveSuffix(".tar.gz")) - Expect(meta.Hash).To(Equal("345fe2e640663c3cd6ac87b7afb92e1c934f665f75ddcb9555bc33e1813ef00b")) - Expect(meta.HashType).To(Equal(crypto.SHA256)) - }) - It("accesses local artifact", func() { acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") m := Must(acc.AccessMethod(cv)) defer m.Close() Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) - r := Must(m.Reader()) defer r.Close() dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) @@ -91,7 +58,6 @@ var _ = Describe("accessmethods.mvn.AccessSpec tests", func() { m := Must(acc.AccessMethod(cv)) defer m.Close() Expect(m.MimeType()).To(Equal(mime.MIME_XML)) - r := Must(m.Reader()) defer r.Close() dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) @@ -108,7 +74,6 @@ var _ = Describe("accessmethods.mvn.AccessSpec tests", func() { It("detects digests mismatch", func() { acc := mvn.New("file://"+FAILPATH, "fail", "repository", "42", mvn.WithExtension("pom")) - m := Must(acc.AccessMethod(cv)) defer m.Close() _, err := m.Reader() From 5f846f1bae5636ee6498cc96b3608b1459adbe3a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 26 Apr 2024 15:49:57 +0200 Subject: [PATCH 051/100] using the new HashReader --- .../handlers/generic/mvn/blobhandler.go | 51 ++++++++----------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 96d8959d06..133ae9dcab 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -15,6 +15,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/utils/tarutils" @@ -90,16 +91,22 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi for _, file := range files { log.Debug("uploading", "file", file) artifact = artifact.ClassifierExtensionFrom(file) - reader, err := tempFs.Open(file) + + readHash, err := tempFs.Open(file) if err != nil { return nil, err } - defer reader.Close() - hash, err := GetHash(crypto.SHA256, tempFs, file) + defer readHash.Close() + // MD5 + SHA1 are still the most used ones in the mvn context + hr := iotools.NewHashReader(readHash, crypto.SHA256, crypto.SHA1, crypto.MD5) + _, _ = hr.CalcHashes() + + reader, err := tempFs.Open(file) if err != nil { return nil, err } - err = deploy(artifact, b.spec.Url, reader, username, password, crypto.SHA256, hash) + defer reader.Close() + err = deploy(artifact, b.spec.Url, reader, username, password, hr) if err != nil { return nil, err } @@ -109,35 +116,17 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil } -func GetHash(hash crypto.Hash, fs vfs.FileSystem, path string) (string, error) { - reader, err := fs.Open(path) - if err != nil { - return "", err - } - defer reader.Close() - digest := hash.New() - if _, err := io.Copy(digest, reader); err != nil { - return "", err - } - return fmt.Sprintf("%x", digest.Sum(nil)), nil -} - -func ChecksumHeader(hash crypto.Hash) string { - a := strings.ReplaceAll(hash.String(), "-", "") - return "X-Checksum-" + a[:1] + strings.ToLower(a[1:]) -} - -func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, hash crypto.Hash, digest string) error { - // https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-apis - // vs. https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifacts-from-archive - // Headers: X-Checksum-Deploy: true, X-Checksum-Sha1: sha1Value, X-Checksum-Sha256: sha256Value, X-Checksum: checksum value (type is resolved by length) +// deploy an artifact to the specified destination. See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact +func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, hashes *iotools.HashReader) error { req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) if err != nil { return err } req.SetBasicAuth(username, password) - req.Header.Set("X-Checksum", digest) - req.Header.Set(ChecksumHeader(hash), digest) + // give the remote server a chance to decide based upon the checksum policy + for k, v := range hashes.HttpHeader() { + req.Header.Set(k, v) + } // Execute the request client := &http.Client{} @@ -168,9 +157,11 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username s return err } - remoteDigest := artifactBody.Checksums[strings.ReplaceAll(strings.ToLower(hash.String()), "-", "")] + // let's check only SHA256 for now + digest := hashes.GetString(crypto.SHA256) + remoteDigest := artifactBody.Checksums[strings.ReplaceAll(strings.ToLower(crypto.SHA256.String()), "-", "")] if remoteDigest == "" { - log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", hash) + log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", crypto.SHA256) } else if remoteDigest != digest { return fmt.Errorf("failed to upload artifact: checksums do not match") } From 74df938f55d065864565c041c80c745b5e4698ca Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Apr 2024 09:26:15 +0200 Subject: [PATCH 052/100] Comment should end in a period (godot) --- pkg/contexts/ocm/accessmethods/options/standard.go | 4 ++-- pkg/utils/tarutils/pack.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/options/standard.go b/pkg/contexts/ocm/accessmethods/options/standard.go index 80b2304e53..a3dc1bc22e 100644 --- a/pkg/contexts/ocm/accessmethods/options/standard.go +++ b/pkg/contexts/ocm/accessmethods/options/standard.go @@ -59,8 +59,8 @@ var HTTPRedirectOption = RegisterOption(NewBoolOptionType("noredirect", "http re // CommentOption. var CommentOption = RegisterOption(NewStringOptionType("comment", "comment field value")) -// ClassifierOption the optional classifier of a maven resource +// ClassifierOption the optional classifier of a maven resource. var ClassifierOption = RegisterOption(NewStringOptionType("accessClassifier", "classifier")) -// ExtensionOption the optional extension of a maven resource +// ExtensionOption the optional extension of a maven resource. var ExtensionOption = RegisterOption(NewStringOptionType("accessExtension", "extension name")) diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index 7f5e137d06..c2e703a208 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -240,7 +240,7 @@ func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string, } // TgzFs creates a tar.gz archive from a filesystem with all files being in the root of the zipped archive. -// The writer is closed after the archive is written. The TAR-headers are normalized, see RegularFileInfoHeader +// The writer is closed after the archive is written. The TAR-headers are normalized, see RegularFileInfoHeader. func TgzFs(fs vfs.FileSystem, writer io.Writer) error { zip := gzip.NewWriter(writer) err := TarFlatFs(fs, zip) From 72fa77af98a26079bd965c7ca4a9e1ec8f181cdc Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Apr 2024 11:45:19 +0200 Subject: [PATCH 053/100] io.Copy(io.Discard, h.reader) is a bit too lazy --- pkg/iotools/hashReaderWriter.go | 17 +++++++++++++++-- pkg/iotools/hashReaderWriter_test.go | 9 +++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/iotools/hashReaderWriter.go b/pkg/iotools/hashReaderWriter.go index 375c75376b..2db2b2b95c 100644 --- a/pkg/iotools/hashReaderWriter.go +++ b/pkg/iotools/hashReaderWriter.go @@ -46,8 +46,21 @@ func (h *HashReader) ReadAll() ([]byte, error) { return io.ReadAll(h.reader) } +// CalcHashes returns the total number of bytes read and an error if any besides EOF. func (h *HashReader) CalcHashes() (int64, error) { - return io.Copy(io.Discard, h.reader) + b := make([]byte, 0, 512) + cnt := int64(0) + for { + n, err := h.Read(b[0:cap(b)]) // read a chunk, always from the beginning + b = b[:n] // reset slice to the actual read bytes + cnt += int64(n) + if err != nil { + if err == io.EOF { + err = nil + } + return cnt, err + } + } } //////////////////////////////////////////////////////////////////////////////// @@ -106,7 +119,7 @@ func getBytes(h hashes, algorithm crypto.Hash) []byte { func httpHeader(h hashes) map[string]string { headers := make(map[string]string, len(h.hashes())) - for algorithm, _ := range h.hashes() { + for algorithm := range h.hashes() { headers[headerName(algorithm)] = getString(h, algorithm) } return headers diff --git a/pkg/iotools/hashReaderWriter_test.go b/pkg/iotools/hashReaderWriter_test.go index cc535f72e5..bda2a6fc3d 100644 --- a/pkg/iotools/hashReaderWriter_test.go +++ b/pkg/iotools/hashReaderWriter_test.go @@ -13,7 +13,6 @@ import ( ) var _ = Describe("Hash Reader Writer tests", func() { - It("Ensure interface implementation", func() { var _ io.Reader = &iotools.HashReader{} var _ io.Reader = (*iotools.HashReader)(nil) @@ -62,6 +61,13 @@ var _ = Describe("Hash Reader Writer tests", func() { Expect(hr.GetBytes(0)).To(BeNil()) Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1) + cnt, err := hr.CalcHashes() + Expect(err).To(BeNil()) + Expect(cnt).To(Equal(int64(len(s)))) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1, crypto.MD5) hr.Read(buf) Expect(hr.GetBytes(crypto.SHA256)).To(BeNil()) @@ -70,5 +76,4 @@ var _ = Describe("Hash Reader Writer tests", func() { Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) }) - }) From e2013c2759bb5c1bfc65edaf7ce11156360b36bc Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Apr 2024 12:13:17 +0200 Subject: [PATCH 054/100] lint --- pkg/iotools/hashReaderWriter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/iotools/hashReaderWriter.go b/pkg/iotools/hashReaderWriter.go index 2db2b2b95c..675cdb68c0 100644 --- a/pkg/iotools/hashReaderWriter.go +++ b/pkg/iotools/hashReaderWriter.go @@ -6,6 +6,8 @@ import ( "hash" "io" "strings" + + "github.com/open-component-model/ocm/pkg/errors" ) type HashReader struct { @@ -55,7 +57,7 @@ func (h *HashReader) CalcHashes() (int64, error) { b = b[:n] // reset slice to the actual read bytes cnt += int64(n) if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { err = nil } return cnt, err From 7c9a4623fd12e9eed1ce584e7e060b8ff520ad20 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Apr 2024 12:47:26 +0200 Subject: [PATCH 055/100] avoid panic(nil pointer dereference) when path doesn't exist --- pkg/utils/tarutils/pack.go | 3 +++ pkg/utils/tarutils/pack_test.go | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index c2e703a208..45b06c7b08 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -226,6 +226,9 @@ func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string, error) { var files []string err := vfs.Walk(fs, root, func(path string, info vfs.FileInfo, err error) error { + if err != nil { + return err + } if !info.IsDir() { pathOrName := path if flat { diff --git a/pkg/utils/tarutils/pack_test.go b/pkg/utils/tarutils/pack_test.go index 3ca873cb1e..0fab744b9a 100644 --- a/pkg/utils/tarutils/pack_test.go +++ b/pkg/utils/tarutils/pack_test.go @@ -1,18 +1,15 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package tarutils_test import ( + "io/fs" "os" + "github.com/mandelsoft/vfs/pkg/osfs" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/open-component-model/ocm/pkg/testutils" - - "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/open-component-model/ocm/pkg/errors" + . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/utils/tarutils" ) @@ -44,4 +41,13 @@ var _ = Describe("tar utils mapping", func() { list := Must(tarutils.ListArchiveContent(file.Name())) Expect(list).To(ConsistOf("dir", "dir/dirlink", "dir/link", "dir/regular", "dir/subdir", "dir/subdir/file", "dir2", "dir2/file2", "file", "dir/dirlink/file2")) }) + + It("test ListSortedFilesInDir with non existing path", func() { + files, err := tarutils.ListSortedFilesInDir(osfs.New(), "/path/doesn't/exist!", true) + Expect(err).To(HaveOccurred()) + Expect(files).To(BeNil()) + Expect(errors.Is(err, fs.ErrNotExist)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("The system cannot find the path specified.")) + }) + }) From 154c620c083273b5fdc8bbd21cfd4415e9b9a77e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Apr 2024 14:18:24 +0200 Subject: [PATCH 056/100] there are different error messages on different OS --- pkg/utils/tarutils/pack_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/utils/tarutils/pack_test.go b/pkg/utils/tarutils/pack_test.go index 0fab744b9a..2487913a61 100644 --- a/pkg/utils/tarutils/pack_test.go +++ b/pkg/utils/tarutils/pack_test.go @@ -3,6 +3,7 @@ package tarutils_test import ( "io/fs" "os" + "runtime" "github.com/mandelsoft/vfs/pkg/osfs" . "github.com/onsi/ginkgo/v2" @@ -47,7 +48,11 @@ var _ = Describe("tar utils mapping", func() { Expect(err).To(HaveOccurred()) Expect(files).To(BeNil()) Expect(errors.Is(err, fs.ErrNotExist)).To(BeTrue()) - Expect(err.Error()).To(ContainSubstring("The system cannot find the path specified.")) + if runtime.GOOS == "windows" { + Expect(err.Error()).To(ContainSubstring("The system cannot find the path specified.")) + } else { + Expect(err.Error()).To(ContainSubstring("no such file or directory")) + } }) }) From b9552ec62ddb96933415d59b74d06a703c2d3ae4 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Apr 2024 11:35:43 +0200 Subject: [PATCH 057/100] DynamicLogger --- pkg/contexts/ocm/accessmethods/mvn/method.go | 10 +++++----- .../blobhandler/handlers/generic/mvn/blobhandler.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 888cf44005..3f064a239c 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -61,7 +61,7 @@ type AccessSpec struct { var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) -var logger = logging.Context().Logger(identity.REALM) +var log = logging.DynamicLogger(identity.REALM) // New creates a new Maven (mvn) repository access spec version v1. func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { @@ -156,7 +156,7 @@ type meta struct { func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { fs := vfsattr.Get(ctx) - log := logger.WithValues("BaseUrl", a.BaseUrl()) + log := log.WithValues("BaseUrl", a.BaseUrl()) fileMap, err := a.GavFiles(fs) if err != nil { return nil, err @@ -178,7 +178,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { for file, hash := range fileMap { artifact.ClassifierExtensionFrom(file) metadata.Bin = artifact.Url(a.Repository) - log = logger.WithValues("file", metadata.Bin) + log = log.WithValues("file", metadata.Bin) log.Debug("processing") metadata.MimeType = artifact.MimeType() if hash > 0 { @@ -278,7 +278,7 @@ func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, er // gavOnlineFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. func (a *AccessSpec) gavOnlineFiles() (map[string]crypto.Hash, error) { - log := logger.WithValues("BaseUrl", a.BaseUrl()) + log := log.WithValues("BaseUrl", a.BaseUrl()) log.Debug("gavOnlineFiles") reader, err := getReader(a.BaseUrl(), nil) @@ -326,7 +326,7 @@ func filesAndHashes(fileList []string) map[string]crypto.Hash { for _, file := range fileList { if IsResource(file) { result[file] = bestAvailableHash(fileList, file) - logger.Debug("found", "file", file) + log.Debug("found", "file", file) } } return result diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 133ae9dcab..f0415bffe0 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -31,7 +31,7 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { return &artifactHandler{repospec} } -var log = logging.Context().Logger(identity.REALM) +var log = logging.DynamicLogger(identity.REALM) func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { // check conditions @@ -52,7 +52,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } // setup logger - log = log.WithValues("repository", b.spec.Url) + log := log.WithValues("repository", b.spec.Url) // identify artifact artifact := mvn.ArtifactFromHint(hint) From 8831dd830a50c9a26adc0adc98a5a734fedef44d Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Apr 2024 11:55:21 +0200 Subject: [PATCH 058/100] inline Artifact in AccessSpec --- .../ocm/accessmethods/mvn/artifact.go | 30 ++++++++---- pkg/contexts/ocm/accessmethods/mvn/method.go | 47 +++++++------------ 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index cddd0a061d..d401555a54 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -7,16 +7,19 @@ import ( "github.com/open-component-model/ocm/pkg/mimeutils" ) -// Artifact holds the typical Maven coordinates groupId, artifactId, version and packaging. +// Artifact holds the typical Maven coordinates groupId, artifactId, version. Optional also classifier and extension. // https://maven.apache.org/ref/3.9.6/maven-core/artifact-handlers.html type Artifact struct { - GroupId string - ArtifactId string - Version string - Classifier string - Extension string - // Type string - // Packaging string + // ArtifactId is the name of Maven (mvn) artifact. + GroupId string `json:"groupId"` + // ArtifactId is the name of Maven (mvn) artifact. + ArtifactId string `json:"artifactId"` + // Version of the Maven (mvn) artifact. + Version string `json:"version"` + // Classifier of the Maven (mvn) artifact. + Classifier string `json:"classifier"` + // Extension of the Maven (mvn) artifact. + Extension string `json:"extension"` } // GAV returns the GAV coordinates of the Maven Artifact. @@ -97,6 +100,17 @@ func (a *Artifact) MimeType() string { return mime.MIME_TGZ } +// Copy creates a new Artifact with the same values. +func (a *Artifact) Copy() *Artifact { + return &Artifact{ + GroupId: a.GroupId, + ArtifactId: a.ArtifactId, + Version: a.Version, + Classifier: a.Classifier, + Extension: a.Extension, + } +} + // ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. func ArtifactFromHint(gav string) *Artifact { parts := strings.Split(gav, ":") diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 3f064a239c..693272bd1d 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -47,16 +47,8 @@ type AccessSpec struct { // Repository is the base URL of the Maven (mvn) repository. Repository string `json:"repository"` - // ArtifactId is the name of Maven (mvn) artifact. - GroupId string `json:"groupId"` - // ArtifactId is the name of Maven (mvn) artifact. - ArtifactId string `json:"artifactId"` - // Version of the Maven (mvn) artifact. - Version string `json:"version"` - // Classifier of the Maven (mvn) artifact. - Classifier string `json:"classifier"` - // Extension of the Maven (mvn) artifact. - Extension string `json:"extension"` + + Artifact `json:",inline"` } var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) @@ -67,12 +59,13 @@ var log = logging.DynamicLogger(identity.REALM) func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { accessSpec := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), - Repository: repository, - GroupId: groupId, - ArtifactId: artifactId, - Version: version, - Classifier: "", - Extension: "", + Artifact: Artifact{ + GroupId: groupId, + ArtifactId: artifactId, + Version: version, + Classifier: "", + Extension: "", + }, } for _, option := range options { option(accessSpec) @@ -129,21 +122,15 @@ func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.Comp } func (a *AccessSpec) BaseUrl() string { - return a.Repository + "/" + a.AsArtifact().GavPath() + return a.Repository + "/" + a.GavPath() } func (a *AccessSpec) ArtifactUrl() string { - return a.AsArtifact().Url(a.Repository) + return a.Url(a.Repository) } -func (a *AccessSpec) AsArtifact() *Artifact { - return &Artifact{ - GroupId: a.GroupId, - ArtifactId: a.ArtifactId, - Version: a.Version, - Classifier: a.Classifier, - Extension: a.Extension, - } +func (a *AccessSpec) NewArtifact() *Artifact { + return a.Artifact.Copy() } type meta struct { @@ -173,7 +160,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } defer vfs.Cleanup(tempFs) - artifact := a.AsArtifact() + artifact := a.NewArtifact() metadata := meta{} for file, hash := range fileMap { artifact.ClassifierExtensionFrom(file) @@ -295,7 +282,7 @@ func (a *AccessSpec) gavOnlineFiles() (map[string]crypto.Hash, error) { } var fileList []string var process func(*html.Node) - prefix := a.AsArtifact().FilePrefix() + prefix := a.FilePrefix() process = func(node *html.Node) { // check if the node is an element node and the tag is "" if node.Type == html.ElementNode && node.Data == "a" { @@ -386,10 +373,10 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A } } acc := blobaccess.DataAccessForReaderFunction(reader, meta.Bin) - return accessobj.CachedBlobAccessForWriter(c.GetContext(), a.AsArtifact().MimeType(), accessio.NewDataAccessWriter(acc)), nil + return accessobj.CachedBlobAccessForWriter(c.GetContext(), a.MimeType(), accessio.NewDataAccessWriter(acc)), nil } // FIXME add Digest! - return accspeccpi.NewDefaultMethodImpl(c, a, "", a.AsArtifact().MimeType(), factory), nil + return accspeccpi.NewDefaultMethodImpl(c, a, "", a.MimeType(), factory), nil } func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { From 96cabdb07af6d402991fb5b6479b76ea73f28370 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Apr 2024 12:15:19 +0200 Subject: [PATCH 059/100] refactoring method names etc. --- .../ocm/accessmethods/mvn/artifact.go | 23 +++++++++++-------- .../ocm/accessmethods/mvn/artifact_test.go | 9 ++++---- pkg/contexts/ocm/accessmethods/mvn/method.go | 10 ++++---- .../handlers/generic/mvn/blobhandler.go | 2 +- .../handlers/generic/mvn/blobhandler_test.go | 1 - 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index d401555a54..7c66ee9e96 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -27,6 +27,11 @@ func (a *Artifact) GAV() string { return a.GroupId + ":" + a.ArtifactId + ":" + a.Version } +// Serialize returns the Artifact as a string (GroupId:ArtifactId:Version:Classifier:Extension). +func (a *Artifact) Serialize() string { + return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + ":" + a.Classifier + ":" + a.Extension +} + // String returns the GAV coordinates of the Maven Artifact. func (a *Artifact) String() string { return a.GAV() @@ -37,11 +42,11 @@ func (a *Artifact) GavPath() string { return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version } -// FileName returns the Maven Artifact's GAV-name with classifier and extension. +// FilePath returns the Maven Artifact's GAV-name with classifier and extension. // Which is equal to the URL-path of the artifact in the repository. // Default extension is jar. -func (a *Artifact) FileName() string { - path := a.GavPath() + "/" + a.FilePrefix() +func (a *Artifact) FilePath() string { + path := a.GavPath() + "/" + a.FileNamePrefix() if a.Classifier != "" { path += "-" + a.Classifier } @@ -54,7 +59,7 @@ func (a *Artifact) FileName() string { } func (a *Artifact) Url(baseUrl string) string { - return baseUrl + "/" + a.FileName() + return baseUrl + "/" + a.FilePath() } // GroupPath returns GroupId with `/` instead of `.`. @@ -62,7 +67,7 @@ func (a *Artifact) GroupPath() string { return strings.ReplaceAll(a.GroupId, ".", "/") } -func (a *Artifact) FilePrefix() string { +func (a *Artifact) FileNamePrefix() string { return a.ArtifactId + "-" + a.Version } @@ -74,7 +79,7 @@ func (a *Artifact) Purl() string { // ClassifierExtensionFrom extracts the classifier and extension from the filename (without any path prefix). func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { // TODO should work with pos (path.Basename)?!? - s := strings.TrimPrefix(filename, a.FilePrefix()) + s := strings.TrimPrefix(filename, a.FileNamePrefix()) if strings.HasPrefix(s, "-") { s = strings.TrimPrefix(s, "-") i := strings.Index(s, ".") @@ -111,9 +116,9 @@ func (a *Artifact) Copy() *Artifact { } } -// ArtifactFromHint creates new Artifact from accessspec-hint. See 'GetReferenceHint'. -func ArtifactFromHint(gav string) *Artifact { - parts := strings.Split(gav, ":") +// DeSerialize creates an Artifact from it's serialized form (see Artifact.Serialize). +func DeSerialize(serializedArtifact string) *Artifact { + parts := strings.Split(serializedArtifact, ":") if len(parts) < 3 { return nil } diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go index 734f750add..44f867abd9 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go @@ -7,7 +7,7 @@ import ( var _ = Describe("Maven Test Environment", func() { - It("GAV, GroupPath, FileName", func() { + It("GAV, GroupPath, FilePath", func() { artifact := &Artifact{ GroupId: "ocm.software", ArtifactId: "hello-ocm", @@ -16,7 +16,7 @@ var _ = Describe("Maven Test Environment", func() { } Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) Expect(artifact.GroupPath()).To(Equal("ocm/software")) - Expect(artifact.FileName()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + Expect(artifact.FilePath()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) }) It("ClassifierExtensionFrom", func() { @@ -42,12 +42,13 @@ var _ = Describe("Maven Test Environment", func() { It("parse GAV", func() { gav := "org.apache.commons:commons-compress:1.26.1:cyclonedx:xml" - artifact := ArtifactFromHint(gav) + artifact := DeSerialize(gav) + Expect(artifact.Serialize()).To(Equal(gav)) Expect(artifact.GroupId).To(Equal("org.apache.commons")) Expect(artifact.ArtifactId).To(Equal("commons-compress")) Expect(artifact.Version).To(Equal("1.26.1")) Expect(artifact.Classifier).To(Equal("cyclonedx")) Expect(artifact.Extension).To(Equal("xml")) - Expect(artifact.FileName()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) + Expect(artifact.FilePath()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) }) }) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 693272bd1d..abc0fb08a5 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -59,6 +59,7 @@ var log = logging.DynamicLogger(identity.REALM) func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { accessSpec := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + Repository: repository, Artifact: Artifact{ GroupId: groupId, ArtifactId: artifactId, @@ -99,10 +100,9 @@ func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpe return a } -// GetReferenceHint returns the reference hint for the Maven (mvn) artifact. In the following form: -// groupId:artifactId:version:classifier:extension. +// GetReferenceHint returns the reference hint for the Maven (mvn) artifact. func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { - return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + ":" + a.Classifier + ":" + a.Extension + return a.Serialize() } func (_ *AccessSpec) GetType() string { @@ -215,7 +215,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } // pack all downloaded files into a tar.gz file - tgz, err := vfs.TempFile(fs, "", Type+"-"+artifact.FilePrefix()+"-*.tar.gz") + tgz, err := vfs.TempFile(fs, "", Type+"-"+artifact.FileNamePrefix()+"-*.tar.gz") if err != nil { return nil, err } @@ -282,7 +282,7 @@ func (a *AccessSpec) gavOnlineFiles() (map[string]crypto.Hash, error) { } var fileList []string var process func(*html.Node) - prefix := a.FilePrefix() + prefix := a.FileNamePrefix() process = func(node *html.Node) { // check if the node is an element node and the tag is "" if node.Type == html.ElementNode && node.Data == "a" { diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index f0415bffe0..b72092ec63 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -55,7 +55,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi log := log.WithValues("repository", b.spec.Url) // identify artifact - artifact := mvn.ArtifactFromHint(hint) + artifact := mvn.DeSerialize(hint) log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) log.Debug("identified") diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go index dbd3ba2c06..a632ace444 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go @@ -30,7 +30,6 @@ var _ = Describe("blobhandler generic mvn tests", func() { err := json.Unmarshal([]byte(resp), &body) Expect(err).To(BeNil()) Expect(body.Repo).To(Equal("ocm-mvn-test")) - //Expect(body.FileName).To(Equal("/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) Expect(body.MimeType).To(Equal("application/java-archive")) From 58653631d3454c5fd368a3911d9133ab01afed0a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Apr 2024 15:25:50 +0200 Subject: [PATCH 060/100] BasicAuth --- .../builtin/mvn/identity/identity.go | 15 ++++++++++++ .../ocm/accessmethods/mvn/integration_test.go | 4 ++-- pkg/contexts/ocm/accessmethods/mvn/method.go | 23 ++++++++++--------- .../handlers/generic/mvn/blobhandler.go | 23 ++++--------------- 4 files changed, 33 insertions(+), 32 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index a3bffeeaf4..b7acafe409 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -1,6 +1,7 @@ package identity import ( + "net/http" "path" . "net/url" @@ -8,6 +9,7 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" "github.com/open-component-model/ocm/pkg/listformat" "github.com/open-component-model/ocm/pkg/logging" ) @@ -59,3 +61,16 @@ func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Pro } return credentials.Properties() } + +func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) { + credentials := GetCredentials(ctx, repoUrl, groupId) + if credentials == nil { + return + } + username := credentials[ATTR_USERNAME] + password := credentials[ATTR_PASSWORD] + if username == "" || password == "" { + return + } + req.SetBasicAuth(username, password) +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go index f8fd99b45b..18db835afa 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go @@ -34,7 +34,7 @@ var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() // https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0 It("one single pom only", func() { acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") - files, err := acc.GavFiles() + files, err := acc.GavFiles(cv.GetContext()) Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(1)) Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) @@ -55,7 +55,7 @@ var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") Expect(acc).ToNot(BeNil()) Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) - files, err := acc.GavFiles() + files, err := acc.GavFiles(cv.GetContext()) Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(8)) Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index abc0fb08a5..6e202cb79e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -144,7 +144,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { fs := vfsattr.Get(ctx) log := log.WithValues("BaseUrl", a.BaseUrl()) - fileMap, err := a.GavFiles(fs) + fileMap, err := a.GavFiles(ctx, fs) if err != nil { return nil, err } @@ -170,7 +170,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { metadata.MimeType = artifact.MimeType() if hash > 0 { metadata.HashType = hash - metadata.Hash, err = getStringData(metadata.Bin+hashUrlExt(hash), fs) + metadata.Hash, err = getStringData(ctx, metadata.Bin+hashUrlExt(hash), fs) if err != nil { return nil, errors.Wrapf(err, "cannot read %s digest of: %s", hash, metadata.Bin) } @@ -190,7 +190,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { return nil, err } defer out.Close() - reader, err := getReader(metadata.Bin, fs) + reader, err := getReader(ctx, metadata.Bin, fs) if err != nil { return nil, err } @@ -247,12 +247,12 @@ func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[s return filtered } -func (a *AccessSpec) GavFiles(fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { +func (a *AccessSpec) GavFiles(ctx accspeccpi.Context, fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { if strings.HasPrefix(a.Repository, "file://") && len(fs) > 0 { dir := a.Repository[7:] return gavFilesFromDisk(fs[0], dir) } - return a.gavOnlineFiles() + return a.gavOnlineFiles(ctx) } func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, error) { @@ -264,11 +264,11 @@ func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, er } // gavOnlineFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. -func (a *AccessSpec) gavOnlineFiles() (map[string]crypto.Hash, error) { +func (a *AccessSpec) gavOnlineFiles(ctx accspeccpi.Context) (map[string]crypto.Hash, error) { log := log.WithValues("BaseUrl", a.BaseUrl()) log.Debug("gavOnlineFiles") - reader, err := getReader(a.BaseUrl(), nil) + reader, err := getReader(ctx, a.BaseUrl(), nil) if err != nil { return nil, err } @@ -334,8 +334,8 @@ func bestAvailableHash(list []string, filename string) crypto.Hash { //////////////////////////////////////////////////////////////////////////////// // getStringData reads all data from the given URL and returns it as a string. -func getStringData(url string, fs vfs.FileSystem) (string, error) { - r, err := getReader(url, fs) +func getStringData(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (string, error) { + r, err := getReader(ctx, url, fs) if err != nil { return "", err } @@ -360,7 +360,7 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A } reader := func() (io.ReadCloser, error) { - return getReader(meta.Bin, vfsattr.Get(c.GetContext())) + return getReader(c.GetContext(), meta.Bin, vfsattr.Get(c.GetContext())) } if meta.Hash != "" { getreader := reader @@ -379,7 +379,7 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A return accspeccpi.NewDefaultMethodImpl(c, a, "", a.MimeType(), factory), nil } -func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { +func getReader(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (io.ReadCloser, error) { if strings.HasPrefix(url, "file://") { path := url[7:] return fs.OpenFile(path, vfs.O_RDONLY, 0o600) @@ -389,6 +389,7 @@ func getReader(url string, fs vfs.FileSystem) (io.ReadCloser, error) { if err != nil { return nil, err } + identity.BasicAuth(req, ctx, url, "") httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index b72092ec63..062daa3957 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -14,6 +14,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/logging" @@ -53,32 +54,16 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi // setup logger log := log.WithValues("repository", b.spec.Url) - // identify artifact artifact := mvn.DeSerialize(hint) log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) log.Debug("identified") - // get credentials - cred := identity.GetCredentials(ctx.GetContext(), b.spec.Url, artifact.GroupPath()) - if cred == nil { - return nil, fmt.Errorf("no credentials found for %s. Couldn't upload '%s'", b.spec.Url, artifact) - } - username := cred[identity.ATTR_USERNAME] - password := cred[identity.ATTR_PASSWORD] - if username == "" || password == "" { - return nil, fmt.Errorf("credentials for %s are invalid. Username or password missing! Couldn't upload '%s'", b.spec.Url, artifact) - } - log = log.WithValues("user", username) - log.Debug("found credentials") - - // Create a new request blobReader, err := blob.Reader() if err != nil { return nil, err } defer blobReader.Close() - tempFs, err := tarutils.ExtractTgzToTempFs(blobReader) if err != nil { return nil, err @@ -106,7 +91,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return nil, err } defer reader.Close() - err = deploy(artifact, b.spec.Url, reader, username, password, hr) + err = deploy(artifact, b.spec.Url, reader, ctx.GetContext(), hr) if err != nil { return nil, err } @@ -117,12 +102,12 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } // deploy an artifact to the specified destination. See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact -func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, username string, password string, hashes *iotools.HashReader) error { +func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspeccpi.Context, hashes *iotools.HashReader) error { req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) if err != nil { return err } - req.SetBasicAuth(username, password) + identity.BasicAuth(req, ctx, url, artifact.GroupPath()) // give the remote server a chance to decide based upon the checksum policy for k, v := range hashes.HttpHeader() { req.Header.Set(k, v) From 2cbb278738c1cc3d263dbcb3547c315d5475f18a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 11:55:17 +0200 Subject: [PATCH 061/100] log credential errors on debug --- .../credentials/builtin/mvn/identity/identity.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index b7acafe409..3f642d90a6 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -43,6 +43,7 @@ the `+hostpath.IDENTITY_TYPE+` type.`, func GetConsumerId(rawURL, groupId string) cpi.ConsumerIdentity { url, err := Parse(rawURL) if err != nil { + debug("GetConsumerId", "error", err.Error(), "url", rawURL) return nil } @@ -56,7 +57,12 @@ func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Pro return nil } credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) - if credentials == nil || err != nil { + if err != nil { + debug("GetCredentials", "error", err.Error()) + return nil + } + if credentials == nil { + debug("no credentials found") return nil } return credentials.Properties() @@ -74,3 +80,8 @@ func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId strin } req.SetBasicAuth(username, password) } + +// debug uses a dynamic logger to log a debug message. +func debug(msg string, keypairs ...interface{}) { + logging.DynamicLogger(REALM).Debug(msg, keypairs...) +} From 6cb47ca26602339dd4aa1115883032065b295f6b Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:07:02 +0200 Subject: [PATCH 062/100] use `JoinPath` --- pkg/contexts/credentials/builtin/mvn/identity/identity.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 3f642d90a6..395da0854e 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -41,14 +41,13 @@ the `+hostpath.IDENTITY_TYPE+` type.`, } func GetConsumerId(rawURL, groupId string) cpi.ConsumerIdentity { - url, err := Parse(rawURL) + url, err := JoinPath(rawURL, groupId) if err != nil { debug("GetConsumerId", "error", err.Error(), "url", rawURL) return nil } - url.Path = path.Join(url.Path, groupId) - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url) } func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Properties { From 8a383f4b967d7977437512d1b919d49a1c9cf3b4 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:07:16 +0200 Subject: [PATCH 063/100] keep in sync with `mvn` --- .../credentials/builtin/npm/identity/identity.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index 32f9a16e8c..0ed2c139ae 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -1,8 +1,6 @@ package identity import ( - "path" - . "net/url" "github.com/open-component-model/ocm/pkg/common" @@ -45,13 +43,13 @@ the `+hostpath.IDENTITY_TYPE+` type.`, } func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { - url, err := Parse(rawURL) + url, err := JoinPath(rawURL, pkgName) if err != nil { + debug("GetConsumerId", "error", err.Error(), "url", rawURL) return nil } - url.Path = path.Join(url.Path, pkgName) - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url) } func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { @@ -65,3 +63,8 @@ func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) com } return credentials.Properties() } + +// debug uses a dynamic logger to log a debug message. +func debug(msg string, keypairs ...interface{}) { + logging.DynamicLogger(REALM).Debug(msg, keypairs...) +} From 6a8ceb4872cd837a4c12ce5011d68cf01817abc3 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:15:43 +0200 Subject: [PATCH 064/100] capital case with underscores becomes CamelCase --- .../builtin/mvn/identity/identity.go | 29 ++++++++------- .../builtin/npm/identity/identity.go | 36 +++++++++---------- .../repositories/npm/config_test.go | 8 ++--- .../repositories/npm/repository.go | 2 +- .../repositories/npm/repository_test.go | 8 ++--- pkg/npm/login.go | 8 ++--- 6 files changed, 45 insertions(+), 46 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 395da0854e..3fea8fb737 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -2,7 +2,6 @@ package identity import ( "net/http" - "path" . "net/url" @@ -15,27 +14,27 @@ import ( ) const ( - // CONSUMER_TYPE is the mvn repository type. - CONSUMER_TYPE = "Repository.maven.apache.org" + // ConsumerType is the mvn repository type. + ConsumerType = "Repository.maven.apache.org" - // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. - ATTR_USERNAME = cpi.ATTR_USERNAME - // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. - ATTR_PASSWORD = cpi.ATTR_PASSWORD + // Username is the username attribute. Required for login at any mvn registry. + Username = cpi.ATTR_USERNAME + // Password is the password attribute. Required for login at any mvn registry. + Password = cpi.ATTR_PASSWORD ) -// Logging Realm. +// REALM the logging realm / prefix. var REALM = logging.DefineSubRealm("Maven repository", "mvn") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - ATTR_USERNAME, "the basic auth user name", - ATTR_PASSWORD, "the basic auth password", + Username, "the basic auth user name", + Password, "the basic auth password", }) - cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `MVN repository + cpi.RegisterStandardIdentity(ConsumerType, hostpath.IdentityMatcher(ConsumerType), `MVN repository -It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +It matches the `+ConsumerType+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } @@ -47,7 +46,7 @@ func GetConsumerId(rawURL, groupId string) cpi.ConsumerIdentity { return nil } - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url) + return hostpath.GetConsumerIdentity(ConsumerType, url) } func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Properties { @@ -72,8 +71,8 @@ func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId strin if credentials == nil { return } - username := credentials[ATTR_USERNAME] - password := credentials[ATTR_PASSWORD] + username := credentials[Username] + password := credentials[Password] if username == "" || password == "" { return } diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index 0ed2c139ae..535352b6fa 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -11,33 +11,33 @@ import ( ) const ( - // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npmjs.com" + // ConsumerType is the npm repository type. + ConsumerType = "Registry.npmjs.com" - // ATTR_USERNAME is the username attribute. Required for login at any npm registry. - ATTR_USERNAME = cpi.ATTR_USERNAME - // ATTR_PASSWORD is the password attribute. Required for login at any npm registry. - ATTR_PASSWORD = cpi.ATTR_PASSWORD - // ATTR_EMAIL is the email attribute. Required for login at any npm registry. - ATTR_EMAIL = cpi.ATTR_EMAIL - // ATTR_TOKEN is the token attribute. May exist after login at any npm registry. - ATTR_TOKEN = cpi.ATTR_TOKEN + // Username is the username attribute. Required for login at any npm registry. + Username = cpi.ATTR_USERNAME + // Password is the password attribute. Required for login at any npm registry. + Password = cpi.ATTR_PASSWORD + // Email is the email attribute. Required for login at any npm registry. + Email = cpi.ATTR_EMAIL + // Token is the token attribute. May exist after login at any npm registry. + Token = cpi.ATTR_TOKEN ) -// Logging Realm. +// REALM the logging realm / prefix. var REALM = logging.DefineSubRealm("NPM registry", "NPM") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - ATTR_USERNAME, "the basic auth user name", - ATTR_PASSWORD, "the basic auth password", - ATTR_EMAIL, "NPM registry, require an email address", - ATTR_TOKEN, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!", + Username, "the basic auth user name", + Password, "the basic auth password", + Email, "NPM registry, require an email address", + Token, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!", }) - cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository + cpi.RegisterStandardIdentity(ConsumerType, hostpath.IdentityMatcher(ConsumerType), `NPM repository -It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +It matches the `+ConsumerType+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } @@ -49,7 +49,7 @@ func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { return nil } - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url) + return hostpath.GetConsumerIdentity(ConsumerType, url) } func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index e2c2fcfea9..40a88e96ee 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -3,12 +3,12 @@ package npm_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" + . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("Config deserialization Test Environment", func() { @@ -16,8 +16,8 @@ var _ = Describe("Config deserialization Test Environment", func() { ctx := credentials.New() repo := Must(npm.NewRepository(ctx, "testdata/.npmrc")) - Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"})) - Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"})) + Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.Token: "npm_TOKEN"})) + Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.Token: "bearer_TOKEN"})) }) It("propagates credentials", func() { @@ -30,7 +30,7 @@ var _ = Describe("Config deserialization Test Environment", func() { creds := Must(credentials.CredentialsForConsumer(ctx, id)) Expect(creds).NotTo(BeNil()) - Expect(creds.GetProperty(identity.ATTR_TOKEN)).To(Equal("npm_TOKEN")) + Expect(creds.GetProperty(identity.Token)).To(Equal("npm_TOKEN")) }) It("has description", func() { diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index c02d7d5971..88e3806072 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -81,7 +81,7 @@ func (r *Repository) Read(force bool) error { func newCredentials(token string) cpi.Credentials { props := common.Properties{ - npmCredentials.ATTR_TOKEN: token, + npmCredentials.Token: token, } return cpi.NewCredentials(props) } diff --git a/pkg/contexts/credentials/repositories/npm/repository_test.go b/pkg/contexts/credentials/repositories/npm/repository_test.go index b628c82ce7..d4844cedd0 100644 --- a/pkg/contexts/credentials/repositories/npm/repository_test.go +++ b/pkg/contexts/credentials/repositories/npm/repository_test.go @@ -6,7 +6,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" @@ -14,16 +13,17 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" local "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" "github.com/open-component-model/ocm/pkg/finalizer" + . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("NPM config - .npmrc", func() { props := common.Properties{ - npmCredentials.ATTR_TOKEN: "npm_TOKEN", + npmCredentials.Token: "npm_TOKEN", } props2 := common.Properties{ - npmCredentials.ATTR_TOKEN: "bearer_TOKEN", + npmCredentials.Token: "bearer_TOKEN", } var DefaultContext credentials.Context @@ -69,7 +69,7 @@ var _ = Describe("NPM config - .npmrc", func() { Must(ctx.RepositoryForConfig([]byte(specdata), nil)) - ci := cpi.NewConsumerIdentity(npmCredentials.CONSUMER_TYPE) + ci := cpi.NewConsumerIdentity(npmCredentials.ConsumerType) Expect(ci).NotTo(BeNil()) credentials := Must(cpi.CredentialsForConsumer(ctx.CredentialsContext(), ci)) Expect(credentials).NotTo(BeNil()) diff --git a/pkg/npm/login.go b/pkg/npm/login.go index 17f45b0298..fa17914055 100644 --- a/pkg/npm/login.go +++ b/pkg/npm/login.go @@ -66,16 +66,16 @@ func BearerToken(ctx cpi.ContextProvider, repoUrl string, pkgName string) (strin log.Debug("found credentials") // check if token exists, if not login and retrieve token - token := cred[identity.ATTR_TOKEN] + token := cred[identity.Token] if token != "" { log.Debug("token found, skipping login") return token, nil } // use user+pass+mail from credentials to login and retrieve bearer token - username := cred[identity.ATTR_USERNAME] - password := cred[identity.ATTR_PASSWORD] - email := cred[identity.ATTR_EMAIL] + username := cred[identity.Username] + password := cred[identity.Password] + email := cred[identity.Email] if username == "" || password == "" || email == "" { return "", fmt.Errorf("credentials for %s are invalid. Username, password or email missing! Couldn't upload '%s'", repoUrl, pkgName) } From b5da2e8d985c41dad06504d9c7442bd7268b722c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:30:02 +0200 Subject: [PATCH 065/100] wrap into anonymous function for proper `defer` handling --- .../handlers/generic/mvn/blobhandler.go | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 062daa3957..be700345a0 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -74,26 +74,28 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return nil, err } for _, file := range files { - log.Debug("uploading", "file", file) - artifact = artifact.ClassifierExtensionFrom(file) - - readHash, err := tempFs.Open(file) - if err != nil { - return nil, err - } - defer readHash.Close() - // MD5 + SHA1 are still the most used ones in the mvn context - hr := iotools.NewHashReader(readHash, crypto.SHA256, crypto.SHA1, crypto.MD5) - _, _ = hr.CalcHashes() - - reader, err := tempFs.Open(file) - if err != nil { - return nil, err - } - defer reader.Close() - err = deploy(artifact, b.spec.Url, reader, ctx.GetContext(), hr) - if err != nil { - return nil, err + e := func() (err error) { + log.Debug("uploading", "file", file) + artifact = artifact.ClassifierExtensionFrom(file) + readHash, err := tempFs.Open(file) + if err != nil { + return + } + defer readHash.Close() + // MD5 + SHA1 are still the most used ones in the mvn context + hr := iotools.NewHashReader(readHash, crypto.SHA256, crypto.SHA1, crypto.MD5) + _, _ = hr.CalcHashes() + + reader, err := tempFs.Open(file) + if err != nil { + return + } + defer reader.Close() + err = deploy(artifact, b.spec.Url, reader, ctx.GetContext(), hr) + return + }() + if e != nil { + return nil, e } } From 0a65ebf99f7299aa75e5a84804640007408866bd Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:30:28 +0200 Subject: [PATCH 066/100] fmt.Errorf not necessary --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index be700345a0..e6c71c5c67 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -16,6 +16,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" @@ -49,7 +50,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi return nil, nil } if b.spec.Url == "" { - return nil, fmt.Errorf("MVN repository url not provided") + return nil, errors.New("MVN repository url not provided") } // setup logger From 2fac60b24c1b3ed14c8e32ec35cb4a05d49c2504 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 12:32:47 +0200 Subject: [PATCH 067/100] handle errors of CalcHashes() --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index e6c71c5c67..42063960d8 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -85,8 +85,10 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi defer readHash.Close() // MD5 + SHA1 are still the most used ones in the mvn context hr := iotools.NewHashReader(readHash, crypto.SHA256, crypto.SHA1, crypto.MD5) - _, _ = hr.CalcHashes() - + _, err = hr.CalcHashes() + if err != nil { + return + } reader, err := tempFs.Open(file) if err != nil { return From 9752ee26d2e32ac716268204a3a3073c2320ca08 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 13:06:12 +0200 Subject: [PATCH 068/100] fetch latest go implementation --- pkg/mimeutils/type.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/mimeutils/type.go b/pkg/mimeutils/type.go index 4d53ee7051..44e25ed776 100644 --- a/pkg/mimeutils/type.go +++ b/pkg/mimeutils/type.go @@ -97,13 +97,14 @@ var builtinTypesLower = map[string]string{ var once sync.Once // guards initMime -var testInitMime func() +var testInitMime, osInitMime func() func initMime() { if fn := testInitMime; fn != nil { fn() } else { setMimeTypes(builtinTypesLower, builtinTypesLower) + osInitMime() } } @@ -145,7 +146,8 @@ func TypeByExtension(ext string) string { if c >= utf8RuneSelf { // Slow path. si, _ := mimeTypesLower.Load(strings.ToLower(ext)) - return si.(string) + s, _ := si.(string) + return s } if 'A' <= c && c <= 'Z' { lower = append(lower, c+('a'-'A')) @@ -154,7 +156,8 @@ func TypeByExtension(ext string) string { } } si, _ := mimeTypesLower.Load(string(lower)) - return si.(string) + s, _ := si.(string) + return s } // ExtensionsByType returns the extensions known to be associated with the MIME From 8d5571357686dcdf60aef2a16a89b3a38b3e9f9a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 13:28:43 +0200 Subject: [PATCH 069/100] wrap into anonymous function for proper `defer` handling --- pkg/contexts/ocm/accessmethods/mvn/method.go | 47 +++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 6e202cb79e..4e2676d675 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -185,32 +185,35 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } // download the artifact into the temporary file system - out, err := tempFs.Create(file) - if err != nil { - return nil, err - } - defer out.Close() - reader, err := getReader(ctx, metadata.Bin, fs) - if err != nil { - return nil, err - } - defer reader.Close() - - if hash > 0 { - dreader := iotools.NewDigestReaderWithHash(hash, reader) - _, err = io.Copy(out, dreader) + e := func() (err error) { + out, err := tempFs.Create(file) if err != nil { - return nil, err - } - sum := dreader.Digest().Encoded() - if metadata.Hash != sum { - return nil, errors.Newf("checksum mismatch for %s", metadata.Bin) + return } - } else { - _, err = io.Copy(out, reader) + defer out.Close() + reader, err := getReader(ctx, metadata.Bin, fs) if err != nil { - return nil, err + return + } + defer reader.Close() + if hash > 0 { + dreader := iotools.NewDigestReaderWithHash(hash, reader) + _, err = io.Copy(out, dreader) + if err != nil { + return + } + sum := dreader.Digest().Encoded() + if metadata.Hash != sum { + return errors.Newf("checksum mismatch for %s", metadata.Bin) + } + } else { + _, err = io.Copy(out, reader) + return } + return + }() + if e != nil { + return nil, e } } From a2c79187c4c1a066b59e27f83f8a454963ab87b3 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 13:30:38 +0200 Subject: [PATCH 070/100] Don't panic! --- pkg/contexts/ocm/accessmethods/mvn/artifact.go | 9 +++++---- pkg/contexts/ocm/accessmethods/mvn/method.go | 5 ++++- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 5 ++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go index 7c66ee9e96..37c509664b 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ b/pkg/contexts/ocm/accessmethods/mvn/artifact.go @@ -1,6 +1,7 @@ package mvn import ( + "fmt" "strings" "github.com/open-component-model/ocm/pkg/mime" @@ -77,14 +78,14 @@ func (a *Artifact) Purl() string { } // ClassifierExtensionFrom extracts the classifier and extension from the filename (without any path prefix). -func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { - // TODO should work with pos (path.Basename)?!? +func (a *Artifact) ClassifierExtensionFrom(filename string) (*Artifact, error) { + // TODO should work with both (path.Basename)?!? s := strings.TrimPrefix(filename, a.FileNamePrefix()) if strings.HasPrefix(s, "-") { s = strings.TrimPrefix(s, "-") i := strings.Index(s, ".") if i < 0 { - panic("no extension after classifier found in filename: " + filename) + return nil, fmt.Errorf("no extension after classifier found in filename: %s", filename) } a.Classifier = s[:i] s = strings.TrimPrefix(s, a.Classifier) @@ -92,7 +93,7 @@ func (a *Artifact) ClassifierExtensionFrom(filename string) *Artifact { a.Classifier = "" } a.Extension = strings.TrimPrefix(s, ".") - return a + return a, nil } // MimeType returns the MIME type of the Maven Artifact based on the file extension. diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 4e2676d675..a2c97da970 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -163,7 +163,10 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { artifact := a.NewArtifact() metadata := meta{} for file, hash := range fileMap { - artifact.ClassifierExtensionFrom(file) + _, err := artifact.ClassifierExtensionFrom(file) + if err != nil { + return nil, err + } metadata.Bin = artifact.Url(a.Repository) log = log.WithValues("file", metadata.Bin) log.Debug("processing") diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index 42063960d8..fcdbd5c5e9 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -77,7 +77,10 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi for _, file := range files { e := func() (err error) { log.Debug("uploading", "file", file) - artifact = artifact.ClassifierExtensionFrom(file) + artifact, err = artifact.ClassifierExtensionFrom(file) + if err != nil { + return + } readHash, err := tempFs.Open(file) if err != nil { return From 6287afeed042faf2565304ced9fa86ba4cf33381 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 14:12:05 +0200 Subject: [PATCH 071/100] errors.New --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index fcdbd5c5e9..c0b89600e2 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -156,7 +156,7 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspe if remoteDigest == "" { log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", crypto.SHA256) } else if remoteDigest != digest { - return fmt.Errorf("failed to upload artifact: checksums do not match") + return errors.New("failed to upload artifact: checksums do not match") } log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest) return nil From 17b88356540c4cb4e97a14f752c739b7bdb0b4d3 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 14:12:36 +0200 Subject: [PATCH 072/100] capital case with underscores becomes CamelCase --- .../ocm/blobhandler/handlers/generic/mvn/blobhandler.go | 2 +- .../ocm/blobhandler/handlers/generic/mvn/registration.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index c0b89600e2..f61f86e3fb 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -23,7 +23,7 @@ import ( "github.com/open-component-model/ocm/pkg/utils/tarutils" ) -const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.MVN_ARTIFACT +const BlobHandlerName = "ocm/" + resourcetypes.MVN_ARTIFACT type artifactHandler struct { spec *Config diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go index 7bd4ec73e1..7fc5ffb8f9 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go @@ -33,7 +33,7 @@ func (c *Config) UnmarshalJSON(data []byte) error { } func init() { - cpi.RegisterBlobHandlerRegistrationHandler(BLOB_HANDLER_NAME, &RegistrationHandler{}) + cpi.RegisterBlobHandlerRegistrationHandler(BlobHandlerName, &RegistrationHandler{}) } type RegistrationHandler struct{} @@ -63,7 +63,7 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co func (r *RegistrationHandler) GetHandlers(_ cpi.Context) registrations.HandlerInfos { return registrations.NewLeafHandlerInfo("uploading mvn artifacts", ` -The `+BLOB_HANDLER_NAME+` uploader is able to upload mvn artifacts (whole GAV only!) +The `+BlobHandlerName+` uploader is able to upload mvn artifacts (whole GAV only!) as artifact archive according to the mvn artifact spec. If registered the default mime type is: `+mime.MIME_TGZ+` From 7f257f01cdf5c851a882d3ba1da6847139d4ba9b Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 16:11:32 +0200 Subject: [PATCH 073/100] stop copy & paste from go-internal libraries --- pkg/mimeutils/type.go | 205 +++--------------------------------------- 1 file changed, 10 insertions(+), 195 deletions(-) diff --git a/pkg/mimeutils/type.go b/pkg/mimeutils/type.go index 44e25ed776..cd142c8192 100644 --- a/pkg/mimeutils/type.go +++ b/pkg/mimeutils/type.go @@ -1,87 +1,13 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Modifications Copyright 2024 SAP SE or an SAP affiliate company. - -// Package mimeutils implements parts of the MIME spec. -// Copied from mime to achieve a platform independent reproducible list of -// known mime types. package mimeutils import ( - "fmt" "mime" - "sort" - "strings" - "sync" + "github.com/open-component-model/ocm/pkg/logging" ocmmime "github.com/open-component-model/ocm/pkg/mime" ) -var ( - mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress" - mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress" - - // extensions maps from MIME type to list of lowercase file - // extensions: "image/jpeg" => [".jpg", ".jpeg"] - extensionsMu sync.Mutex // Guards stores (but not loads) on extensions. - extensions sync.Map // map[string][]string; slice values are append-only. -) - -func clearSyncMap(m *sync.Map) { - m.Range(func(k, _ any) bool { - m.Delete(k) - return true - }) -} - -// setMimeTypes is used by initMime's non-test path, and by tests. -func setMimeTypes(lowerExt, mixExt map[string]string) { - clearSyncMap(&mimeTypes) - clearSyncMap(&mimeTypesLower) - clearSyncMap(&extensions) - - for k, v := range lowerExt { - mimeTypesLower.Store(k, v) - } - for k, v := range mixExt { - mimeTypes.Store(k, v) - } - - extensionsMu.Lock() - defer extensionsMu.Unlock() - for k, v := range lowerExt { - justType, _, err := mime.ParseMediaType(v) - if err != nil { - panic(err) - } - var exts []string - if ei, ok := extensions.Load(justType); ok { - exts = ei.([]string) - } - extensions.Store(justType, append(exts, k)) - } -} - -var builtinTypesLower = map[string]string{ - // default list - ".avif": "image/avif", - ".css": "text/css; charset=utf-8", - ".gif": "image/gif", - ".htm": "text/html; charset=utf-8", - ".html": "text/html; charset=utf-8", - ".jpeg": "image/jpeg", - ".jpg": "image/jpeg", - ".js": "text/javascript; charset=utf-8", - ".json": "application/json", - ".mjs": "text/javascript; charset=utf-8", - ".pdf": "application/pdf", - ".png": "image/png", - ".svg": "image/svg+xml", - ".wasm": "application/wasm", - ".webp": "image/webp", - ".xml": "text/xml; charset=utf-8", +var ocmTypes = map[string]string{ // added entries ".txt": ocmmime.MIME_TEXT, ".yaml": ocmmime.MIME_YAML_OFFICIAL, @@ -95,127 +21,16 @@ var builtinTypesLower = map[string]string{ ".module": ocmmime.MIME_JSON, // gradle module metadata } -var once sync.Once // guards initMime - -var testInitMime, osInitMime func() - -func initMime() { - if fn := testInitMime; fn != nil { - fn() - } else { - setMimeTypes(builtinTypesLower, builtinTypesLower) - osInitMime() - } -} - -// TypeByExtension returns the MIME type associated with the file extension ext. -// The extension ext should begin with a leading dot, as in ".html". -// When ext has no associated type, TypeByExtension returns "". -// -// Extensions are looked up first case-sensitively, then case-insensitively. -// -// The built-in table is small but on unix it is augmented by the local -// system's MIME-info database or mime.types file(s) if available under one or -// more of these names: -// -// /usr/local/share/mime/globs2 -// /usr/share/mime/globs2 -// /etc/mime.types -// /etc/apache2/mime.types -// /etc/apache/mime.types -// -// On Windows, MIME types are extracted from the registry. -// -// Text types have the charset parameter set to "utf-8" by default. -func TypeByExtension(ext string) string { - once.Do(initMime) - - // Case-sensitive lookup. - if v, ok := mimeTypes.Load(ext); ok { - return v.(string) - } - - // Case-insensitive lookup. - // Optimistically assume a short ASCII extension and be - // allocation-free in that case. - var buf [10]byte - lower := buf[:0] - const utf8RuneSelf = 0x80 // from utf8 package, but not importing it. - for i := 0; i < len(ext); i++ { - c := ext[i] - if c >= utf8RuneSelf { - // Slow path. - si, _ := mimeTypesLower.Load(strings.ToLower(ext)) - s, _ := si.(string) - return s - } - if 'A' <= c && c <= 'Z' { - lower = append(lower, c+('a'-'A')) - } else { - lower = append(lower, c) +func init() { + for k, v := range ocmTypes { + err := mime.AddExtensionType(k, v) + if err != nil { + logging.DynamicLogger(logging.DefineSubRealm("mimeutils")).Error("failed to add extension type", "extension", k, "type", v, "error", err) } } - si, _ := mimeTypesLower.Load(string(lower)) - s, _ := si.(string) - return s -} - -// ExtensionsByType returns the extensions known to be associated with the MIME -// type typ. The returned extensions will each begin with a leading dot, as in -// ".html". When typ has no associated extensions, ExtensionsByType returns an -// nil slice. -func ExtensionsByType(typ string) ([]string, error) { - justType, _, err := mime.ParseMediaType(typ) - if err != nil { - return nil, err - } - - once.Do(initMime) - s, ok := extensions.Load(justType) - if !ok { - return nil, nil - } - ret := append([]string(nil), s.([]string)...) - sort.Strings(ret) - return ret, nil } -// AddExtensionType sets the MIME type associated with -// the extension ext to typ. The extension should begin with -// a leading dot, as in ".html". -func AddExtensionType(ext, typ string) error { - if !strings.HasPrefix(ext, ".") { - return fmt.Errorf("mime: extension %q missing leading dot", ext) - } - once.Do(initMime) - return setExtensionType(ext, typ) -} - -func setExtensionType(extension, mimeType string) error { - justType, param, err := mime.ParseMediaType(mimeType) - if err != nil { - return err - } - if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" { - param["charset"] = "utf-8" - mimeType = mime.FormatMediaType(mimeType, param) - } - extLower := strings.ToLower(extension) - - mimeTypes.Store(extension, mimeType) - mimeTypesLower.Store(extLower, mimeType) - - extensionsMu.Lock() - defer extensionsMu.Unlock() - var exts []string - if ei, ok := extensions.Load(justType); ok { - exts = ei.([]string) - } - for _, v := range exts { - if v == extLower { - return nil - } - } - extensions.Store(justType, append(exts, extLower)) - return nil +// Deprecated: use mime.TypeByExtension instead. +func TypeByExtension(ext string) string { + return mime.TypeByExtension(ext) } From 3c15fd6294e32f9d50917e11cd71c777f1db4621 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 13 May 2024 16:14:28 +0200 Subject: [PATCH 074/100] artifact --> coordinates --- .../builtin/mvn/identity/identity.go | 47 +++--- pkg/contexts/ocm/accessmethods/mvn/README.md | 2 +- .../ocm/accessmethods/mvn/artifact.go | 158 ------------------ .../ocm/accessmethods/mvn/coordinates.go | 144 ++++++++++++++++ .../{artifact_test.go => coordinates_test.go} | 17 +- pkg/contexts/ocm/accessmethods/mvn/method.go | 26 +-- .../handlers/generic/mvn/blobhandler.go | 30 ++-- 7 files changed, 212 insertions(+), 212 deletions(-) delete mode 100644 pkg/contexts/ocm/accessmethods/mvn/artifact.go create mode 100644 pkg/contexts/ocm/accessmethods/mvn/coordinates.go rename pkg/contexts/ocm/accessmethods/mvn/{artifact_test.go => coordinates_test.go} (78%) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 3fea8fb737..a1424e1f02 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -1,6 +1,7 @@ package identity import ( + "errors" "net/http" . "net/url" @@ -39,47 +40,51 @@ the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -func GetConsumerId(rawURL, groupId string) cpi.ConsumerIdentity { +var identityMatcher = hostpath.IdentityMatcher(ConsumerType) + +func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { + return identityMatcher(pattern, cur, id) +} + +func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) { url, err := JoinPath(rawURL, groupId) if err != nil { - debug("GetConsumerId", "error", err.Error(), "url", rawURL) - return nil + return nil, err } - - return hostpath.GetConsumerIdentity(ConsumerType, url) + return hostpath.GetConsumerIdentity(ConsumerType, url), nil } -func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) common.Properties { - id := GetConsumerId(repoUrl, groupId) +func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (common.Properties, error) { + id, err := GetConsumerId(repoUrl, groupId) + if err != nil { + return nil, err + } if id == nil { - return nil + return nil, nil } credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) if err != nil { - debug("GetCredentials", "error", err.Error()) - return nil + return nil, err } if credentials == nil { - debug("no credentials found") - return nil + return nil, nil } - return credentials.Properties() + return credentials.Properties(), nil } -func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) { - credentials := GetCredentials(ctx, repoUrl, groupId) +func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) (err error) { + credentials, err := GetCredentials(ctx, repoUrl, groupId) + if err != nil { + return + } if credentials == nil { return } username := credentials[Username] password := credentials[Password] if username == "" || password == "" { - return + return errors.New("missing username or password in credentials") } req.SetBasicAuth(username, password) -} - -// debug uses a dynamic logger to log a debug message. -func debug(msg string, keypairs ...interface{}) { - logging.DynamicLogger(REALM).Debug(msg, keypairs...) + return } diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md index 79cc8522b0..ecb696d7a6 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/README.md +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -10,7 +10,7 @@ Provided blobs use the following media type: `application/x-tgz` ### Description This method implements the access of a resource hosted by a maven repository or a -complete resource set denoted by a GAV. +complete resource set denoted by a GAV (GroupId, ArtifactId, Version). ### Specification Versions diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact.go b/pkg/contexts/ocm/accessmethods/mvn/artifact.go deleted file mode 100644 index 37c509664b..0000000000 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact.go +++ /dev/null @@ -1,158 +0,0 @@ -package mvn - -import ( - "fmt" - "strings" - - "github.com/open-component-model/ocm/pkg/mime" - "github.com/open-component-model/ocm/pkg/mimeutils" -) - -// Artifact holds the typical Maven coordinates groupId, artifactId, version. Optional also classifier and extension. -// https://maven.apache.org/ref/3.9.6/maven-core/artifact-handlers.html -type Artifact struct { - // ArtifactId is the name of Maven (mvn) artifact. - GroupId string `json:"groupId"` - // ArtifactId is the name of Maven (mvn) artifact. - ArtifactId string `json:"artifactId"` - // Version of the Maven (mvn) artifact. - Version string `json:"version"` - // Classifier of the Maven (mvn) artifact. - Classifier string `json:"classifier"` - // Extension of the Maven (mvn) artifact. - Extension string `json:"extension"` -} - -// GAV returns the GAV coordinates of the Maven Artifact. -func (a *Artifact) GAV() string { - return a.GroupId + ":" + a.ArtifactId + ":" + a.Version -} - -// Serialize returns the Artifact as a string (GroupId:ArtifactId:Version:Classifier:Extension). -func (a *Artifact) Serialize() string { - return a.GroupId + ":" + a.ArtifactId + ":" + a.Version + ":" + a.Classifier + ":" + a.Extension -} - -// String returns the GAV coordinates of the Maven Artifact. -func (a *Artifact) String() string { - return a.GAV() -} - -// GavPath returns the Maven repository path. -func (a *Artifact) GavPath() string { - return a.GroupPath() + "/" + a.ArtifactId + "/" + a.Version -} - -// FilePath returns the Maven Artifact's GAV-name with classifier and extension. -// Which is equal to the URL-path of the artifact in the repository. -// Default extension is jar. -func (a *Artifact) FilePath() string { - path := a.GavPath() + "/" + a.FileNamePrefix() - if a.Classifier != "" { - path += "-" + a.Classifier - } - if a.Extension != "" { - path += "." + a.Extension - } else { - path += ".jar" - } - return path -} - -func (a *Artifact) Url(baseUrl string) string { - return baseUrl + "/" + a.FilePath() -} - -// GroupPath returns GroupId with `/` instead of `.`. -func (a *Artifact) GroupPath() string { - return strings.ReplaceAll(a.GroupId, ".", "/") -} - -func (a *Artifact) FileNamePrefix() string { - return a.ArtifactId + "-" + a.Version -} - -// Purl returns the Package URL of the Maven Artifact. -func (a *Artifact) Purl() string { - return "pkg:maven/" + a.GroupId + "/" + a.ArtifactId + "@" + a.Version -} - -// ClassifierExtensionFrom extracts the classifier and extension from the filename (without any path prefix). -func (a *Artifact) ClassifierExtensionFrom(filename string) (*Artifact, error) { - // TODO should work with both (path.Basename)?!? - s := strings.TrimPrefix(filename, a.FileNamePrefix()) - if strings.HasPrefix(s, "-") { - s = strings.TrimPrefix(s, "-") - i := strings.Index(s, ".") - if i < 0 { - return nil, fmt.Errorf("no extension after classifier found in filename: %s", filename) - } - a.Classifier = s[:i] - s = strings.TrimPrefix(s, a.Classifier) - } else { - a.Classifier = "" - } - a.Extension = strings.TrimPrefix(s, ".") - return a, nil -} - -// MimeType returns the MIME type of the Maven Artifact based on the file extension. -// Default is application/x-tgz. -func (a *Artifact) MimeType() string { - m := mimeutils.TypeByExtension("." + a.Extension) - if m != "" { - return m - } - return mime.MIME_TGZ -} - -// Copy creates a new Artifact with the same values. -func (a *Artifact) Copy() *Artifact { - return &Artifact{ - GroupId: a.GroupId, - ArtifactId: a.ArtifactId, - Version: a.Version, - Classifier: a.Classifier, - Extension: a.Extension, - } -} - -// DeSerialize creates an Artifact from it's serialized form (see Artifact.Serialize). -func DeSerialize(serializedArtifact string) *Artifact { - parts := strings.Split(serializedArtifact, ":") - if len(parts) < 3 { - return nil - } - artifact := &Artifact{ - GroupId: parts[0], - ArtifactId: parts[1], - Version: parts[2], - } - if len(parts) >= 4 { - artifact.Classifier = parts[3] - } - if len(parts) >= 5 { - artifact.Extension = parts[4] - } - return artifact -} - -// IsResource returns true if the filename is not a checksum or signature file. -func IsResource(fileName string) bool { - if strings.HasSuffix(fileName, ".asc") { - return false - } - if strings.HasSuffix(fileName, ".md5") { - return false - } - if strings.HasSuffix(fileName, ".sha1") { - return false - } - if strings.HasSuffix(fileName, ".sha256") { - return false - } - if strings.HasSuffix(fileName, ".sha512") { - return false - } - return true -} diff --git a/pkg/contexts/ocm/accessmethods/mvn/coordinates.go b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go new file mode 100644 index 0000000000..dccd4267c0 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go @@ -0,0 +1,144 @@ +package mvn + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/mimeutils" +) + +// Coordinates holds the typical Maven coordinates groupId, artifactId, version. Optional also classifier and extension. +// https://maven.apache.org/ref/3.9.6/maven-core/artifact-handlers.html +type Coordinates struct { + // GroupId of the Maven (mvn) artifact. + GroupId string `json:"groupId"` + // ArtifactId of the Maven (mvn) artifact. + ArtifactId string `json:"artifactId"` + // Version of the Maven (mvn) artifact. + Version string `json:"version"` + // Classifier of the Maven (mvn) artifact. + Classifier string `json:"classifier"` + // Extension of the Maven (mvn) artifact. + Extension string `json:"extension"` +} + +// GAV returns the GAV coordinates of the Maven Coordinates. +func (c *Coordinates) GAV() string { + return c.GroupId + ":" + c.ArtifactId + ":" + c.Version +} + +// String returns the Coordinates as a string (GroupId:ArtifactId:Version:Classifier:Extension). +func (c *Coordinates) String() string { + return c.GroupId + ":" + c.ArtifactId + ":" + c.Version + ":" + c.Classifier + ":" + c.Extension +} + +// GavPath returns the Maven repository path. +func (c *Coordinates) GavPath() string { + return c.GroupPath() + "/" + c.ArtifactId + "/" + c.Version +} + +// FilePath returns the Maven Coordinates's GAV-name with classifier and extension. +// Which is equal to the URL-path of the artifact in the repository. +// Default extension is jar. +func (c *Coordinates) FilePath() string { + path := c.GavPath() + "/" + c.FileNamePrefix() + if c.Classifier != "" { + path += "-" + c.Classifier + } + if c.Extension != "" { + path += "." + c.Extension + } else { + path += ".jar" + } + return path +} + +func (c *Coordinates) Url(baseUrl string) string { + return baseUrl + "/" + c.FilePath() +} + +// GroupPath returns GroupId with `/` instead of `.`. +func (c *Coordinates) GroupPath() string { + return strings.ReplaceAll(c.GroupId, ".", "/") +} + +func (c *Coordinates) FileNamePrefix() string { + return c.ArtifactId + "-" + c.Version +} + +// Purl returns the Package URL of the Maven Coordinates. +func (c *Coordinates) Purl() string { + return "pkg:maven/" + c.GroupId + "/" + c.ArtifactId + "@" + c.Version +} + +// SetClassifierExtensionBy extracts the classifier and extension from the filename (without any path prefix). +func (c *Coordinates) SetClassifierExtensionBy(filename string) error { + s := strings.TrimPrefix(path.Base(filename), c.FileNamePrefix()) + if strings.HasPrefix(s, "-") { + s = strings.TrimPrefix(s, "-") + i := strings.Index(s, ".") + if i < 0 { + return fmt.Errorf("no extension after classifier found in filename: %s", filename) + } + c.Classifier = s[:i] + s = strings.TrimPrefix(s, c.Classifier) + } else { + c.Classifier = "" + } + c.Extension = strings.TrimPrefix(s, ".") + return nil +} + +// MimeType returns the MIME type of the Maven Coordinates based on the file extension. +// Default is application/x-tgz. +func (c *Coordinates) MimeType() string { + m := mimeutils.TypeByExtension("." + c.Extension) + if m != "" { + return m + } + return mime.MIME_TGZ +} + +// Copy creates a new Coordinates with the same values. +func (c *Coordinates) Copy() *Coordinates { + return &Coordinates{ + GroupId: c.GroupId, + ArtifactId: c.ArtifactId, + Version: c.Version, + Classifier: c.Classifier, + Extension: c.Extension, + } +} + +// Parse creates an Coordinates from it's serialized form (see Coordinates.String). +func Parse(serializedArtifact string) (*Coordinates, error) { + parts := strings.Split(serializedArtifact, ":") + if len(parts) < 3 { + return nil, fmt.Errorf("invalid artifact string: %s", serializedArtifact) + } + artifact := &Coordinates{ + GroupId: parts[0], + ArtifactId: parts[1], + Version: parts[2], + } + if len(parts) >= 4 { + artifact.Classifier = parts[3] + } + if len(parts) >= 5 { + artifact.Extension = parts[4] + } + return artifact, nil +} + +// IsResource returns true if the filename is not a checksum or signature file. +func IsResource(fileName string) bool { + switch filepath.Ext(fileName) { + case ".asc", ".md5", ".sha1", ".sha256", ".sha512": + return false + default: + return true + } +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go b/pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go similarity index 78% rename from pkg/contexts/ocm/accessmethods/mvn/artifact_test.go rename to pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go index 44f867abd9..63df1268ae 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/artifact_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go @@ -8,7 +8,7 @@ import ( var _ = Describe("Maven Test Environment", func() { It("GAV, GroupPath, FilePath", func() { - artifact := &Artifact{ + artifact := &Coordinates{ GroupId: "ocm.software", ArtifactId: "hello-ocm", Version: "0.0.1", @@ -19,31 +19,32 @@ var _ = Describe("Maven Test Environment", func() { Expect(artifact.FilePath()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) }) - It("ClassifierExtensionFrom", func() { - artifact := &Artifact{ + It("SetClassifierExtensionBy", func() { + artifact := &Coordinates{ GroupId: "ocm.software", ArtifactId: "hello-ocm", Version: "0.0.1", } - artifact.ClassifierExtensionFrom("hello-ocm-0.0.1.pom") + artifact.SetClassifierExtensionBy("hello-ocm-0.0.1.pom") Expect(artifact.Classifier).To(Equal("")) Expect(artifact.Extension).To(Equal("pom")) - artifact.ClassifierExtensionFrom("hello-ocm-0.0.1-tests.jar") + artifact.SetClassifierExtensionBy("hello-ocm-0.0.1-tests.jar") Expect(artifact.Classifier).To(Equal("tests")) Expect(artifact.Extension).To(Equal("jar")) artifact.ArtifactId = "apache-maven" artifact.Version = "3.9.6" - artifact.ClassifierExtensionFrom("apache-maven-3.9.6-bin.tar.gz") + artifact.SetClassifierExtensionBy("apache-maven-3.9.6-bin.tar.gz") Expect(artifact.Classifier).To(Equal("bin")) Expect(artifact.Extension).To(Equal("tar.gz")) }) It("parse GAV", func() { gav := "org.apache.commons:commons-compress:1.26.1:cyclonedx:xml" - artifact := DeSerialize(gav) - Expect(artifact.Serialize()).To(Equal(gav)) + artifact, err := Parse(gav) + Expect(err).To(BeNil()) + Expect(artifact.String()).To(Equal(gav)) Expect(artifact.GroupId).To(Equal("org.apache.commons")) Expect(artifact.ArtifactId).To(Equal("commons-compress")) Expect(artifact.Version).To(Equal("1.26.1")) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index a2c97da970..ec1c945d34 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -48,7 +48,7 @@ type AccessSpec struct { // Repository is the base URL of the Maven (mvn) repository. Repository string `json:"repository"` - Artifact `json:",inline"` + Coordinates `json:",inline"` } var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) @@ -60,7 +60,7 @@ func New(repository, groupId, artifactId, version string, options ...func(*Acces accessSpec := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), Repository: repository, - Artifact: Artifact{ + Coordinates: Coordinates{ GroupId: groupId, ArtifactId: artifactId, Version: version, @@ -102,7 +102,7 @@ func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpe // GetReferenceHint returns the reference hint for the Maven (mvn) artifact. func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { - return a.Serialize() + return a.String() } func (_ *AccessSpec) GetType() string { @@ -129,8 +129,8 @@ func (a *AccessSpec) ArtifactUrl() string { return a.Url(a.Repository) } -func (a *AccessSpec) NewArtifact() *Artifact { - return a.Artifact.Copy() +func (a *AccessSpec) NewArtifact() *Coordinates { + return a.Coordinates.Copy() } type meta struct { @@ -163,7 +163,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { artifact := a.NewArtifact() metadata := meta{} for file, hash := range fileMap { - _, err := artifact.ClassifierExtensionFrom(file) + err := artifact.SetClassifierExtensionBy(file) if err != nil { return nil, err } @@ -244,13 +244,12 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[string]crypto.Hash { - filtered := make(map[string]crypto.Hash) - for file, hash := range fileMap { - if strings.Contains(file, "-"+classifier+".") { - filtered[file] = hash + for file, _ := range fileMap { + if !strings.Contains(file, "-"+classifier+".") { + delete(fileMap, file) } } - return filtered + return fileMap } func (a *AccessSpec) GavFiles(ctx accspeccpi.Context, fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { @@ -395,7 +394,10 @@ func getReader(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (io.ReadCl if err != nil { return nil, err } - identity.BasicAuth(req, ctx, url, "") + err = identity.BasicAuth(req, ctx, url, "") + if err != nil { + return nil, err + } httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go index f61f86e3fb..1e27561d32 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -56,7 +56,10 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi // setup logger log := log.WithValues("repository", b.spec.Url) // identify artifact - artifact := mvn.DeSerialize(hint) + artifact, err := mvn.Parse(hint) + if err != nil { + return nil, err + } log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) log.Debug("identified") @@ -77,7 +80,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi for _, file := range files { e := func() (err error) { log.Debug("uploading", "file", file) - artifact, err = artifact.ClassifierExtensionFrom(file) + err = artifact.SetClassifierExtensionBy(file) if err != nil { return } @@ -110,12 +113,15 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi } // deploy an artifact to the specified destination. See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact -func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspeccpi.Context, hashes *iotools.HashReader) error { +func deploy(artifact *mvn.Coordinates, url string, reader io.ReadCloser, ctx accspeccpi.Context, hashes *iotools.HashReader) (err error) { req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) if err != nil { - return err + return + } + err = identity.BasicAuth(req, ctx, url, artifact.GroupPath()) + if err != nil { + return } - identity.BasicAuth(req, ctx, url, artifact.GroupPath()) // give the remote server a chance to decide based upon the checksum policy for k, v := range hashes.HttpHeader() { req.Header.Set(k, v) @@ -125,15 +131,15 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspe client := &http.Client{} resp, err := client.Do(req) if err != nil { - return err + return } defer resp.Body.Close() // Check the response if resp.StatusCode != http.StatusCreated { - all, err := io.ReadAll(resp.Body) - if err != nil { - return err + all, e := io.ReadAll(resp.Body) + if e != nil { + return e } return fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) } @@ -142,12 +148,12 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspe // Validate the response - especially the hash values with the ones we've tried to send respBody, err := io.ReadAll(resp.Body) if err != nil { - return err + return } var artifactBody Body err = json.Unmarshal(respBody, &artifactBody) if err != nil { - return err + return } // let's check only SHA256 for now @@ -159,7 +165,7 @@ func deploy(artifact *mvn.Artifact, url string, reader io.ReadCloser, ctx accspe return errors.New("failed to upload artifact: checksums do not match") } log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest) - return nil + return } // Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). From d9ff883350204f5efab0b83eb1978ca9a6b4f8b2 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 09:32:06 +0200 Subject: [PATCH 075/100] drop mimeutils --- pkg/blobaccess/wget/access.go | 3 +- .../ocm/accessmethods/mvn/coordinates.go | 8 ++--- pkg/mime/types.go | 29 +++++++++++++++ pkg/mimeutils/type.go | 36 ------------------- 4 files changed, 34 insertions(+), 42 deletions(-) delete mode 100644 pkg/mimeutils/type.go diff --git a/pkg/blobaccess/wget/access.go b/pkg/blobaccess/wget/access.go index d02c45c0ac..3ee353d3aa 100644 --- a/pkg/blobaccess/wget/access.go +++ b/pkg/blobaccess/wget/access.go @@ -15,7 +15,6 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/errors" ocmmime "github.com/open-component-model/ocm/pkg/mime" - "github.com/open-component-model/ocm/pkg/mimeutils" "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/utils" ) @@ -132,7 +131,7 @@ func BlobAccessForWget(url string, opts ...Option) (_ blobaccess.BlobAccess, rer "extract mime type from url") ext, err := utils.GetFileExtensionFromUrl(url) if err == nil && ext != "" { - eff.MimeType = mimeutils.TypeByExtension(ext) + eff.MimeType = mime.TypeByExtension(ext) } else if err != nil { log.Debug(err.Error()) } diff --git a/pkg/contexts/ocm/accessmethods/mvn/coordinates.go b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go index dccd4267c0..d39e09c2e5 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/coordinates.go +++ b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go @@ -2,12 +2,12 @@ package mvn import ( "fmt" + "mime" "path" "path/filepath" "strings" - "github.com/open-component-model/ocm/pkg/mime" - "github.com/open-component-model/ocm/pkg/mimeutils" + ocmmime "github.com/open-component-model/ocm/pkg/mime" ) // Coordinates holds the typical Maven coordinates groupId, artifactId, version. Optional also classifier and extension. @@ -95,11 +95,11 @@ func (c *Coordinates) SetClassifierExtensionBy(filename string) error { // MimeType returns the MIME type of the Maven Coordinates based on the file extension. // Default is application/x-tgz. func (c *Coordinates) MimeType() string { - m := mimeutils.TypeByExtension("." + c.Extension) + m := mime.TypeByExtension("." + c.Extension) if m != "" { return m } - return mime.MIME_TGZ + return ocmmime.MIME_TGZ } // Copy creates a new Coordinates with the same values. diff --git a/pkg/mime/types.go b/pkg/mime/types.go index 783ed12ab2..715f6845b2 100644 --- a/pkg/mime/types.go +++ b/pkg/mime/types.go @@ -1,5 +1,11 @@ package mime +import ( + "mime" + + "github.com/open-component-model/ocm/pkg/logging" +) + const ( MIME_TEXT = "text/plain" MIME_OCTET = "application/octet-stream" @@ -19,3 +25,26 @@ const ( MIME_JAR = "application/x-jar" ) + +func init() { + ocmTypes := map[string]string{ + // added entries + ".txt": MIME_TEXT, + ".yaml": MIME_YAML_OFFICIAL, + ".gzip": MIME_GZIP, + ".tar": MIME_TAR, + ".tgz": MIME_TGZ, + ".tar.gz": MIME_TGZ, + ".pom": MIME_XML, + ".zip": MIME_GZIP, + ".jar": MIME_JAR, + ".module": MIME_JSON, // gradle module metadata + } + + for k, v := range ocmTypes { + err := mime.AddExtensionType(k, v) + if err != nil { + logging.DynamicLogger(logging.DefineSubRealm("mimeutils")).Error("failed to add extension type", "extension", k, "type", v, "error", err) + } + } +} diff --git a/pkg/mimeutils/type.go b/pkg/mimeutils/type.go deleted file mode 100644 index cd142c8192..0000000000 --- a/pkg/mimeutils/type.go +++ /dev/null @@ -1,36 +0,0 @@ -package mimeutils - -import ( - "mime" - - "github.com/open-component-model/ocm/pkg/logging" - ocmmime "github.com/open-component-model/ocm/pkg/mime" -) - -var ocmTypes = map[string]string{ - // added entries - ".txt": ocmmime.MIME_TEXT, - ".yaml": ocmmime.MIME_YAML_OFFICIAL, - ".gzip": ocmmime.MIME_GZIP, - ".tar": ocmmime.MIME_TAR, - ".tgz": ocmmime.MIME_TGZ, - ".tar.gz": ocmmime.MIME_TGZ, - ".pom": ocmmime.MIME_XML, - ".zip": ocmmime.MIME_GZIP, - ".jar": ocmmime.MIME_JAR, - ".module": ocmmime.MIME_JSON, // gradle module metadata -} - -func init() { - for k, v := range ocmTypes { - err := mime.AddExtensionType(k, v) - if err != nil { - logging.DynamicLogger(logging.DefineSubRealm("mimeutils")).Error("failed to add extension type", "extension", k, "type", v, "error", err) - } - } -} - -// Deprecated: use mime.TypeByExtension instead. -func TypeByExtension(ext string) string { - return mime.TypeByExtension(ext) -} From c46862afc1d9b9c64f58c2be8cd6611f23a4daee Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 10:19:24 +0200 Subject: [PATCH 076/100] gofmt --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index ec1c945d34..c1c5eb36ba 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -244,7 +244,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[string]crypto.Hash { - for file, _ := range fileMap { + for file := range fileMap { if !strings.Contains(file, "-"+classifier+".") { delete(fileMap, file) } From 17bd6ad0e14567805e04f1c3ec0ac14f16618d6d Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 11:18:30 +0200 Subject: [PATCH 077/100] drop mimeutils --- .reuse/dep5 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.reuse/dep5 b/.reuse/dep5 index 30a2631ec6..e75df02db4 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -28,10 +28,6 @@ Files: ** Copyright: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors License: Apache-2.0 -Files: pkg/mimeutils/* -Copyright: Copyright 2010 The Go Authors. All rights reserved. -License: BSD-3-Clause - Files: pkg/contexts/ocm/blobhandler/handlers/generic/npm/publish.go Copyright: Copyright 2021 - cloverstd License: MIT From ca0c9cf6a7052ef75454aadd9ea78ead859d34a5 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 11:25:00 +0200 Subject: [PATCH 078/100] keep npm unchanged --- .../builtin/npm/identity/identity.go | 47 +++++++++---------- .../repositories/npm/config_test.go | 8 ++-- .../repositories/npm/repository.go | 2 +- .../repositories/npm/repository_test.go | 8 ++-- .../handlers/generic/npm/blobhandler.go | 3 +- pkg/npm/login.go | 8 ++-- 6 files changed, 36 insertions(+), 40 deletions(-) diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index 535352b6fa..32f9a16e8c 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -1,6 +1,8 @@ package identity import ( + "path" + . "net/url" "github.com/open-component-model/ocm/pkg/common" @@ -11,45 +13,45 @@ import ( ) const ( - // ConsumerType is the npm repository type. - ConsumerType = "Registry.npmjs.com" + // CONSUMER_TYPE is the npm repository type. + CONSUMER_TYPE = "Registry.npmjs.com" - // Username is the username attribute. Required for login at any npm registry. - Username = cpi.ATTR_USERNAME - // Password is the password attribute. Required for login at any npm registry. - Password = cpi.ATTR_PASSWORD - // Email is the email attribute. Required for login at any npm registry. - Email = cpi.ATTR_EMAIL - // Token is the token attribute. May exist after login at any npm registry. - Token = cpi.ATTR_TOKEN + // ATTR_USERNAME is the username attribute. Required for login at any npm registry. + ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any npm registry. + ATTR_PASSWORD = cpi.ATTR_PASSWORD + // ATTR_EMAIL is the email attribute. Required for login at any npm registry. + ATTR_EMAIL = cpi.ATTR_EMAIL + // ATTR_TOKEN is the token attribute. May exist after login at any npm registry. + ATTR_TOKEN = cpi.ATTR_TOKEN ) -// REALM the logging realm / prefix. +// Logging Realm. var REALM = logging.DefineSubRealm("NPM registry", "NPM") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - Username, "the basic auth user name", - Password, "the basic auth password", - Email, "NPM registry, require an email address", - Token, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!", + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_EMAIL, "NPM registry, require an email address", + ATTR_TOKEN, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!", }) - cpi.RegisterStandardIdentity(ConsumerType, hostpath.IdentityMatcher(ConsumerType), `NPM repository + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository -It matches the `+ConsumerType+` consumer type and additionally acts like +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { - url, err := JoinPath(rawURL, pkgName) + url, err := Parse(rawURL) if err != nil { - debug("GetConsumerId", "error", err.Error(), "url", rawURL) return nil } - return hostpath.GetConsumerIdentity(ConsumerType, url) + url.Path = path.Join(url.Path, pkgName) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) } func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { @@ -63,8 +65,3 @@ func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) com } return credentials.Properties() } - -// debug uses a dynamic logger to log a debug message. -func debug(msg string, keypairs ...interface{}) { - logging.DynamicLogger(REALM).Debug(msg, keypairs...) -} diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index 40a88e96ee..e2c2fcfea9 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -3,12 +3,12 @@ package npm_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" - . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("Config deserialization Test Environment", func() { @@ -16,8 +16,8 @@ var _ = Describe("Config deserialization Test Environment", func() { ctx := credentials.New() repo := Must(npm.NewRepository(ctx, "testdata/.npmrc")) - Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.Token: "npm_TOKEN"})) - Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.Token: "bearer_TOKEN"})) + Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"})) + Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"})) }) It("propagates credentials", func() { @@ -30,7 +30,7 @@ var _ = Describe("Config deserialization Test Environment", func() { creds := Must(credentials.CredentialsForConsumer(ctx, id)) Expect(creds).NotTo(BeNil()) - Expect(creds.GetProperty(identity.Token)).To(Equal("npm_TOKEN")) + Expect(creds.GetProperty(identity.ATTR_TOKEN)).To(Equal("npm_TOKEN")) }) It("has description", func() { diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index 88e3806072..c02d7d5971 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -81,7 +81,7 @@ func (r *Repository) Read(force bool) error { func newCredentials(token string) cpi.Credentials { props := common.Properties{ - npmCredentials.Token: token, + npmCredentials.ATTR_TOKEN: token, } return cpi.NewCredentials(props) } diff --git a/pkg/contexts/credentials/repositories/npm/repository_test.go b/pkg/contexts/credentials/repositories/npm/repository_test.go index d4844cedd0..b628c82ce7 100644 --- a/pkg/contexts/credentials/repositories/npm/repository_test.go +++ b/pkg/contexts/credentials/repositories/npm/repository_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" @@ -13,17 +14,16 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" local "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" "github.com/open-component-model/ocm/pkg/finalizer" - . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("NPM config - .npmrc", func() { props := common.Properties{ - npmCredentials.Token: "npm_TOKEN", + npmCredentials.ATTR_TOKEN: "npm_TOKEN", } props2 := common.Properties{ - npmCredentials.Token: "bearer_TOKEN", + npmCredentials.ATTR_TOKEN: "bearer_TOKEN", } var DefaultContext credentials.Context @@ -69,7 +69,7 @@ var _ = Describe("NPM config - .npmrc", func() { Must(ctx.RepositoryForConfig([]byte(specdata), nil)) - ci := cpi.NewConsumerIdentity(npmCredentials.ConsumerType) + ci := cpi.NewConsumerIdentity(npmCredentials.CONSUMER_TYPE) Expect(ci).NotTo(BeNil()) credentials := Must(cpi.CredentialsForConsumer(ctx.CredentialsContext(), ci)) Expect(credentials).NotTo(BeNil()) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 40f028edf5..76032ffca3 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -11,13 +11,12 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" - "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" npmLogin "github.com/open-component-model/ocm/pkg/npm" ) -const BLOB_HANDLER_NAME = "ocm/" + resourcetypes.NPM_PACKAGE +const BLOB_HANDLER_NAME = "ocm/npmPackage" type artifactHandler struct { spec *Config diff --git a/pkg/npm/login.go b/pkg/npm/login.go index fa17914055..17f45b0298 100644 --- a/pkg/npm/login.go +++ b/pkg/npm/login.go @@ -66,16 +66,16 @@ func BearerToken(ctx cpi.ContextProvider, repoUrl string, pkgName string) (strin log.Debug("found credentials") // check if token exists, if not login and retrieve token - token := cred[identity.Token] + token := cred[identity.ATTR_TOKEN] if token != "" { log.Debug("token found, skipping login") return token, nil } // use user+pass+mail from credentials to login and retrieve bearer token - username := cred[identity.Username] - password := cred[identity.Password] - email := cred[identity.Email] + username := cred[identity.ATTR_USERNAME] + password := cred[identity.ATTR_PASSWORD] + email := cred[identity.ATTR_EMAIL] if username == "" || password == "" || email == "" { return "", fmt.Errorf("credentials for %s are invalid. Username, password or email missing! Couldn't upload '%s'", repoUrl, pkgName) } From 2c49e81e956e32c17990972457f02a105d7e33f1 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 11:33:21 +0200 Subject: [PATCH 079/100] keep capitals --- .../builtin/mvn/identity/identity.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index a1424e1f02..36168af944 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -15,13 +15,13 @@ import ( ) const ( - // ConsumerType is the mvn repository type. - ConsumerType = "Repository.maven.apache.org" + // CONSUMER_TYPE is the mvn repository type. + CONSUMER_TYPE = "MavenRepository" - // Username is the username attribute. Required for login at any mvn registry. - Username = cpi.ATTR_USERNAME - // Password is the password attribute. Required for login at any mvn registry. - Password = cpi.ATTR_PASSWORD + // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. + ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. + ATTR_PASSWORD = cpi.ATTR_PASSWORD ) // REALM the logging realm / prefix. @@ -29,18 +29,18 @@ var REALM = logging.DefineSubRealm("Maven repository", "mvn") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - Username, "the basic auth user name", - Password, "the basic auth password", + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", }) - cpi.RegisterStandardIdentity(ConsumerType, hostpath.IdentityMatcher(ConsumerType), `MVN repository + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `MVN repository -It matches the `+ConsumerType+` consumer type and additionally acts like +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -var identityMatcher = hostpath.IdentityMatcher(ConsumerType) +var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { return identityMatcher(pattern, cur, id) @@ -51,7 +51,7 @@ func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) { if err != nil { return nil, err } - return hostpath.GetConsumerIdentity(ConsumerType, url), nil + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url), nil } func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (common.Properties, error) { @@ -80,8 +80,8 @@ func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId strin if credentials == nil { return } - username := credentials[Username] - password := credentials[Password] + username := credentials[ATTR_USERNAME] + password := credentials[ATTR_PASSWORD] if username == "" || password == "" { return errors.New("missing username or password in credentials") } From ca7d30b65e4f57a7531c08ef34c085ab2d5a5cde Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 11:39:24 +0200 Subject: [PATCH 080/100] drop mimeutils --- LICENSES/BSD-3-Clause.txt | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 LICENSES/BSD-3-Clause.txt diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt deleted file mode 100644 index ea890afbc7..0000000000 --- a/LICENSES/BSD-3-Clause.txt +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) . - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From fda631f1f29f98851b9b7b5e233b34740dcb640e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 12:25:21 +0200 Subject: [PATCH 081/100] ensure that there are actually files to be processed --- pkg/contexts/ocm/accessmethods/mvn/method.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index c1c5eb36ba..8109d54d2e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -152,6 +152,9 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { if a.Classifier != "" { fileMap = filterByClassifier(fileMap, a.Classifier) } + if len(fileMap) < 1 { + return nil, errors.New("no maven artifact files found") + } singleBinary := len(fileMap) == 1 tempFs, err := osfs.NewTempFileSystem() From c527927eb76a34bda1e302502f3ff40470bc97c6 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 12:25:46 +0200 Subject: [PATCH 082/100] add missing `close()` --- pkg/contexts/ocm/accessmethods/mvn/method.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 8109d54d2e..219b888b6a 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -347,6 +347,7 @@ func getStringData(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (strin if err != nil { return "", err } + defer r.Close() b, err := io.ReadAll(r) if err != nil { return "", err From 9e3bde6c3afa1c87896143bfbb0078389d1a6399 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 14 May 2024 12:26:31 +0200 Subject: [PATCH 083/100] just log the http.body instead of writing it into the error message --- pkg/contexts/ocm/accessmethods/mvn/method.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 219b888b6a..75abfc53c5 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -411,10 +411,10 @@ func getReader(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (io.ReadCl defer resp.Body.Close() buf := &bytes.Buffer{} _, err = io.Copy(buf, io.LimitReader(resp.Body, 2000)) - if err != nil { - return nil, errors.Newf("http %s error - %s", resp.Status, url) + if err == nil { + log.Error("http", "code", resp.Status, "url", url, "body", buf.String()) } - return nil, errors.Newf("http %s error - %s returned: %s", resp.Status, url, buf.String()) + return nil, errors.Newf("http %s error - %s", resp.Status, url) } return resp.Body, nil } From fe97064a96e6e858b7e95bdf5b7f99e58dc602f9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 15 May 2024 12:48:41 +0200 Subject: [PATCH 084/100] rm `// +build ` --- hack/tools.go | 1 - pkg/cobrautils/flag/path_array_test.go | 1 - pkg/cobrautils/flag/path_array_win_test.go | 1 - pkg/cobrautils/flag/path_test.go | 1 - pkg/cobrautils/flag/path_win_test.go | 1 - pkg/contexts/ocm/accessmethods/mvn/integration_test.go | 1 - 6 files changed, 6 deletions(-) diff --git a/hack/tools.go b/hack/tools.go index b76c619e68..5b15c7ab40 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -1,5 +1,4 @@ //go:build tools -// +build tools package tools diff --git a/pkg/cobrautils/flag/path_array_test.go b/pkg/cobrautils/flag/path_array_test.go index 79bc5352bc..c403dc7b67 100644 --- a/pkg/cobrautils/flag/path_array_test.go +++ b/pkg/cobrautils/flag/path_array_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package flag_test diff --git a/pkg/cobrautils/flag/path_array_win_test.go b/pkg/cobrautils/flag/path_array_win_test.go index a0fd2bf1ad..888954e4a9 100644 --- a/pkg/cobrautils/flag/path_array_win_test.go +++ b/pkg/cobrautils/flag/path_array_win_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package flag_test diff --git a/pkg/cobrautils/flag/path_test.go b/pkg/cobrautils/flag/path_test.go index 8440a9ce96..1de8cecc33 100644 --- a/pkg/cobrautils/flag/path_test.go +++ b/pkg/cobrautils/flag/path_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package flag_test diff --git a/pkg/cobrautils/flag/path_win_test.go b/pkg/cobrautils/flag/path_win_test.go index 4220aa270c..bea185436b 100644 --- a/pkg/cobrautils/flag/path_win_test.go +++ b/pkg/cobrautils/flag/path_win_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package flag_test diff --git a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go index 18db835afa..1381c19e50 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package mvn_test From 52676e29a4f4efe28fc5c92818282fc4d64baede Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 11:03:11 +0200 Subject: [PATCH 085/100] refactor GetCredentials --- .../builtin/mvn/identity/identity.go | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 36168af944..e4fcab93fd 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -6,7 +6,6 @@ import ( . "net/url" - "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" @@ -54,37 +53,31 @@ func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) { return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url), nil } -func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (common.Properties, error) { +func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (cpi.Credentials, error) { id, err := GetConsumerId(repoUrl, groupId) if err != nil { return nil, err } if id == nil { + logging.DynamicLogger(REALM).Debug("No consumer identity found.", "url", repoUrl, "groupId", groupId) return nil, nil } - credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) - if err != nil { - return nil, err - } - if credentials == nil { - return nil, nil - } - return credentials.Properties(), nil + return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) + } func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) (err error) { credentials, err := GetCredentials(ctx, repoUrl, groupId) if err != nil { - return + return err } if credentials == nil { - return + logging.DynamicLogger(REALM).Debug("No credentials found. BasicAuth not required?", "url", repoUrl, "groupId", groupId, "req", req) + return nil } - username := credentials[ATTR_USERNAME] - password := credentials[ATTR_PASSWORD] - if username == "" || password == "" { + if !credentials.ExistsProperty(ATTR_USERNAME) || !credentials.ExistsProperty(ATTR_PASSWORD) { return errors.New("missing username or password in credentials") } - req.SetBasicAuth(username, password) + req.SetBasicAuth(credentials.GetProperty(ATTR_USERNAME), credentials.GetProperty(ATTR_PASSWORD)) return } From c1f64e7ca15d0e0e0dd54300a55aa6ba140d1420 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 11:05:11 +0200 Subject: [PATCH 086/100] follow 'optionutils' --- pkg/contexts/ocm/accessmethods/mvn/method.go | 36 +++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 75abfc53c5..10b474641a 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -26,6 +26,7 @@ import ( "github.com/open-component-model/ocm/pkg/iotools" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/runtime" "github.com/open-component-model/ocm/pkg/utils/tarutils" ) @@ -51,12 +52,15 @@ type AccessSpec struct { Coordinates `json:",inline"` } +// Option defines the interface function "ApplyTo()" +type Option = optionutils.Option[*AccessSpec] + var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) var log = logging.DynamicLogger(identity.REALM) // New creates a new Maven (mvn) repository access spec version v1. -func New(repository, groupId, artifactId, version string, options ...func(*AccessSpec)) *AccessSpec { +func New(repository, groupId, artifactId, version string, options ...Option) *AccessSpec { accessSpec := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), Repository: repository, @@ -68,24 +72,32 @@ func New(repository, groupId, artifactId, version string, options ...func(*Acces Extension: "", }, } - for _, option := range options { - option(accessSpec) - } + optionutils.ApplyOptions(accessSpec, options...) return accessSpec } +// classifier Option for Maven (mvn) Coordinates. +type classifier string + +func (c classifier) ApplyTo(a *AccessSpec) { + a.Classifier = string(c) +} + // WithClassifier sets the classifier of the Maven (mvn) artifact. -func WithClassifier(classifier string) func(*AccessSpec) { - return func(a *AccessSpec) { - a.Classifier = classifier - } +func WithClassifier(c string) Option { + return classifier(c) +} + +// extension Option for Maven (mvn) Coordinates. +type extension string + +func (e extension) ApplyTo(a *AccessSpec) { + a.Extension = string(e) } // WithExtension sets the extension of the Maven (mvn) artifact. -func WithExtension(extension string) func(*AccessSpec) { - return func(a *AccessSpec) { - a.Extension = extension - } +func WithExtension(e string) Option { + return extension(e) } func (a *AccessSpec) Describe(_ accspeccpi.Context) string { From 78efca17f65dd0d8ebe96c05339d5840dc108218 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 12:14:16 +0200 Subject: [PATCH 087/100] don't log complete request --- pkg/contexts/credentials/builtin/mvn/identity/identity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index e4fcab93fd..2c6e27aa61 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -72,7 +72,7 @@ func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId strin return err } if credentials == nil { - logging.DynamicLogger(REALM).Debug("No credentials found. BasicAuth not required?", "url", repoUrl, "groupId", groupId, "req", req) + logging.DynamicLogger(REALM).Debug("No credentials found. BasicAuth not required?", "url", repoUrl, "groupId", groupId) return nil } if !credentials.ExistsProperty(ATTR_USERNAME) || !credentials.ExistsProperty(ATTR_PASSWORD) { From d4156d677f537504dcb849d76d5a9b5c5416721c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 12:15:26 +0200 Subject: [PATCH 088/100] refactor for-loop --- pkg/contexts/ocm/accessmethods/mvn/method.go | 66 +++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 10b474641a..4e36fd4c3e 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -152,6 +152,28 @@ type meta struct { Bin string `json:"bin"` } +func update(a *AccessSpec, file string, hash crypto.Hash, metadata *meta, ctx accspeccpi.Context, fs vfs.FileSystem) error { + artifact := a.NewArtifact() + err := artifact.SetClassifierExtensionBy(file) + if err != nil { + return err + } + metadata.Bin = artifact.Url(a.Repository) + log := log.WithValues("file", metadata.Bin) + log.Debug("processing") + metadata.MimeType = artifact.MimeType() + if hash > 0 { + metadata.HashType = hash + metadata.Hash, err = getStringData(ctx, metadata.Bin+hashUrlExt(hash), fs) + if err != nil { + return errors.Wrapf(err, "cannot read %s digest of: %s", hash, metadata.Bin) + } + } else { + log.Warn("no digest available") + } + return nil +} + func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { fs := vfsattr.Get(ctx) @@ -164,43 +186,31 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { if a.Classifier != "" { fileMap = filterByClassifier(fileMap, a.Classifier) } - if len(fileMap) < 1 { + + switch l := len(fileMap); { + case l <= 0: return nil, errors.New("no maven artifact files found") + case l == 1 && (a.Extension != "" || a.Classifier != ""): + metadata := meta{} + for file, hash := range fileMap { + update(a, file, hash, &metadata, ctx, fs) + } + return &metadata, nil + // default: continue below with: create tempFs where all files can be downloaded to and packed together as tar.gz } - singleBinary := len(fileMap) == 1 + if (a.Extension == "") != (a.Classifier == "") { // XOR + log.Warn("Either classifier or extension have been specified, which results in an incomplete GAV!") + } tempFs, err := osfs.NewTempFileSystem() if err != nil { return nil, err } defer vfs.Cleanup(tempFs) - artifact := a.NewArtifact() metadata := meta{} for file, hash := range fileMap { - err := artifact.SetClassifierExtensionBy(file) - if err != nil { - return nil, err - } - metadata.Bin = artifact.Url(a.Repository) - log = log.WithValues("file", metadata.Bin) - log.Debug("processing") - metadata.MimeType = artifact.MimeType() - if hash > 0 { - metadata.HashType = hash - metadata.Hash, err = getStringData(ctx, metadata.Bin+hashUrlExt(hash), fs) - if err != nil { - return nil, errors.Wrapf(err, "cannot read %s digest of: %s", hash, metadata.Bin) - } - } else { - log.Warn("no digest available") - } - - // single binary dependency, this will never be a complete GAV - no maven uploader support! - if a.Extension != "" || singleBinary && a.Classifier != "" { - // in case you want to transport pom, then you should NOT set the extension - return &metadata, nil - } + update(a, file, hash, &metadata, ctx, fs) // download the artifact into the temporary file system e := func() (err error) { @@ -222,7 +232,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } sum := dreader.Digest().Encoded() if metadata.Hash != sum { - return errors.Newf("checksum mismatch for %s", metadata.Bin) + return errors.Newf("%s digest mismatch: expected %s, found %s", metadata.HashType, metadata.Hash, sum) } } else { _, err = io.Copy(out, reader) @@ -236,7 +246,7 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } // pack all downloaded files into a tar.gz file - tgz, err := vfs.TempFile(fs, "", Type+"-"+artifact.FileNamePrefix()+"-*.tar.gz") + tgz, err := vfs.TempFile(fs, "", Type+"-"+a.NewArtifact().FileNamePrefix()+"-*.tar.gz") if err != nil { return nil, err } From 8abe2d92dbb8a8c264f8ea8e5feb36a7a0069524 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 12:51:09 +0200 Subject: [PATCH 089/100] test: "apache-maven, 'bin' zip + tar.gz only!" --- .../ocm/accessmethods/mvn/integration_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go index 1381c19e50..b2eca2b1d0 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go @@ -15,6 +15,7 @@ import ( . "github.com/open-component-model/ocm/pkg/env/builder" "github.com/open-component-model/ocm/pkg/mime" . "github.com/open-component-model/ocm/pkg/testutils" + "github.com/open-component-model/ocm/pkg/utils/tarutils" ) var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() { @@ -75,4 +76,23 @@ var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! */ }) + + // https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6 + It("apache-maven, 'bin' zip + tar.gz only!", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6", mvn.WithClassifier("bin")) + Expect(acc).ToNot(BeNil()) + Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) + files, err := acc.GavFiles(cv.GetContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(8)) // the repo contains 8 files... + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(err).ToNot(HaveOccurred()) + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + r := Must(m.Reader()) + defer r.Close() + list, err := tarutils.ListArchiveContentFromReader(r) + Expect(err).ToNot(HaveOccurred()) + Expect(list).To(HaveLen(2)) // ...but with the classifier set, we're interested only in two! + }) }) From 5712881302ebec05dff963bb3439a28628103d9b Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 13:13:18 +0200 Subject: [PATCH 090/100] . --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 4e36fd4c3e..6681f6b51d 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -52,7 +52,7 @@ type AccessSpec struct { Coordinates `json:",inline"` } -// Option defines the interface function "ApplyTo()" +// Option defines the interface function "ApplyTo()". type Option = optionutils.Option[*AccessSpec] var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) From 394f657ff109c503a02ac1f057d6e4f2802c4475 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 14:00:00 +0200 Subject: [PATCH 091/100] NpmRegistry --- pkg/contexts/credentials/builtin/npm/identity/identity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index 32f9a16e8c..ee9db72f19 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -14,7 +14,7 @@ import ( const ( // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npmjs.com" + CONSUMER_TYPE = "NpmRegistry" // ATTR_USERNAME is the username attribute. Required for login at any npm registry. ATTR_USERNAME = cpi.ATTR_USERNAME From 9d320e86e4e22f53afa8bb89cf6a8047ff826dbf Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 14:38:11 +0200 Subject: [PATCH 092/100] lower case 'npm' --- pkg/contexts/credentials/builtin/npm/identity/identity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index ee9db72f19..308cf8d299 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -27,7 +27,7 @@ const ( ) // Logging Realm. -var REALM = logging.DefineSubRealm("NPM registry", "NPM") +var REALM = logging.DefineSubRealm("NPM registry", "npm") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ From 898d7fdc1365605d1f654e380965b09148c3ce89 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 14:38:58 +0200 Subject: [PATCH 093/100] mvn options --- pkg/contexts/ocm/accessmethods/options/standard.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/options/standard.go b/pkg/contexts/ocm/accessmethods/options/standard.go index a3dc1bc22e..7e8680f12b 100644 --- a/pkg/contexts/ocm/accessmethods/options/standard.go +++ b/pkg/contexts/ocm/accessmethods/options/standard.go @@ -60,7 +60,7 @@ var HTTPRedirectOption = RegisterOption(NewBoolOptionType("noredirect", "http re var CommentOption = RegisterOption(NewStringOptionType("comment", "comment field value")) // ClassifierOption the optional classifier of a maven resource. -var ClassifierOption = RegisterOption(NewStringOptionType("accessClassifier", "classifier")) +var ClassifierOption = RegisterOption(NewStringOptionType("accessClassifier", "mvn classifier")) // ExtensionOption the optional extension of a maven resource. -var ExtensionOption = RegisterOption(NewStringOptionType("accessExtension", "extension name")) +var ExtensionOption = RegisterOption(NewStringOptionType("accessExtension", "mvn extension name")) From 9f7ac4fc6498ff79e1a145b91952d7e31dc138a9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 16 May 2024 14:57:45 +0200 Subject: [PATCH 094/100] generate docs --- .../plugin_accessmethod_compose.md | 3 ++ docs/pluginreference/plugin_descriptor.md | 3 ++ .../plugin_valueset_compose.md | 3 ++ .../ocm_add_resource-configuration.md | 38 +++++++++++++++++++ docs/reference/ocm_add_resources.md | 38 +++++++++++++++++++ .../reference/ocm_add_source-configuration.md | 38 +++++++++++++++++++ docs/reference/ocm_add_sources.md | 38 +++++++++++++++++++ docs/reference/ocm_controller.md | 1 + docs/reference/ocm_controller_install.md | 1 + docs/reference/ocm_controller_uninstall.md | 33 ++++++++++++++++ docs/reference/ocm_credential-handling.md | 27 +++++++++---- docs/reference/ocm_get_credentials.md | 27 +++++++++---- docs/reference/ocm_logging.md | 3 +- docs/reference/ocm_ocm-accessmethods.md | 35 +++++++++++++++++ docs/reference/ocm_ocm-uploadhandlers.md | 17 +++++++-- .../ocm_transfer_commontransportarchive.md | 17 +++++++-- .../ocm_transfer_componentversions.md | 17 +++++++-- 17 files changed, 310 insertions(+), 29 deletions(-) create mode 100644 docs/reference/ocm_controller_uninstall.md diff --git a/docs/pluginreference/plugin_accessmethod_compose.md b/docs/pluginreference/plugin_accessmethod_compose.md index 5ffbea5834..d9c0455651 100644 --- a/docs/pluginreference/plugin_accessmethod_compose.md +++ b/docs/pluginreference/plugin_accessmethod_compose.md @@ -35,6 +35,9 @@ by the plugin name. The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/pluginreference/plugin_descriptor.md b/docs/pluginreference/plugin_descriptor.md index f04a2169f4..c3ecbdd8a4 100644 --- a/docs/pluginreference/plugin_descriptor.md +++ b/docs/pluginreference/plugin_descriptor.md @@ -120,6 +120,9 @@ It uses the following fields: The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/pluginreference/plugin_valueset_compose.md b/docs/pluginreference/plugin_valueset_compose.md index 0ad1025ed4..09bf007037 100644 --- a/docs/pluginreference/plugin_valueset_compose.md +++ b/docs/pluginreference/plugin_valueset_compose.md @@ -35,6 +35,9 @@ by the plugin name. The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/reference/ocm_add_resource-configuration.md b/docs/reference/ocm_add_resource-configuration.md index fdc3daaee8..679bcc2f27 100644 --- a/docs/reference/ocm_add_resource-configuration.md +++ b/docs/reference/ocm_add_resource-configuration.md @@ -24,6 +24,9 @@ resource-configuration, resourceconfig, rsccfg, rcfg ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -680,6 +683,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_resources.md b/docs/reference/ocm_add_resources.md index d755180f4e..9e8878f8e8 100644 --- a/docs/reference/ocm_add_resources.md +++ b/docs/reference/ocm_add_resources.md @@ -30,6 +30,9 @@ resources, resource, res, r ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -690,6 +693,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_source-configuration.md b/docs/reference/ocm_add_source-configuration.md index 25a8f5f5a0..f7d2b4fc32 100644 --- a/docs/reference/ocm_add_source-configuration.md +++ b/docs/reference/ocm_add_source-configuration.md @@ -24,6 +24,9 @@ source-configuration, sourceconfig, srccfg, scfg ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -680,6 +683,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_sources.md b/docs/reference/ocm_add_sources.md index ed829fb938..746d02c5ee 100644 --- a/docs/reference/ocm_add_sources.md +++ b/docs/reference/ocm_add_sources.md @@ -29,6 +29,9 @@ sources, source, src, s ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -688,6 +691,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_controller.md b/docs/reference/ocm_controller.md index 38c7c859a5..e74255bbaa 100644 --- a/docs/reference/ocm_controller.md +++ b/docs/reference/ocm_controller.md @@ -22,4 +22,5 @@ ocm controller [] ... ##### Sub Commands * [ocm controller install](ocm_controller_install.md) — Install either a specific or latest version of the ocm-controller. Optionally install prerequisites required by the controller. +* [ocm controller uninstall](ocm_controller_uninstall.md) — Uninstalls the ocm-controller and all of its dependencies diff --git a/docs/reference/ocm_controller_install.md b/docs/reference/ocm_controller_install.md index 701fd35056..8ed8c1c02c 100644 --- a/docs/reference/ocm_controller_install.md +++ b/docs/reference/ocm_controller_install.md @@ -19,6 +19,7 @@ ocm controller install controller {--version v0.0.1} -i, --install-prerequisites install prerequisites required by ocm-controller (default true) -n, --namespace string the namespace into which the controller is installed (default "ocm-system") -a, --release-api-url string the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases") + -l, --silent don't fail on error -s, --skip-pre-flight-check skip the pre-flight check for clusters -t, --timeout duration maximum time to wait for deployment to be ready (default 1m0s) -v, --version string the version of the controller to install (default "latest") diff --git a/docs/reference/ocm_controller_uninstall.md b/docs/reference/ocm_controller_uninstall.md new file mode 100644 index 0000000000..e13c34927c --- /dev/null +++ b/docs/reference/ocm_controller_uninstall.md @@ -0,0 +1,33 @@ +## ocm controller uninstall — Uninstalls The Ocm-Controller And All Of Its Dependencies + +### Synopsis + +``` +ocm controller uninstall controller +``` + +### Options + +``` + -u, --base-url string the base url to the ocm-controller's release page (default "https://github.com/open-component-model/ocm-controller/releases") + --cert-manager-base-url string the base url to the cert-manager's release page (default "https://github.com/cert-manager/cert-manager/releases") + --cert-manager-release-api-url string the base url to the cert-manager's API release page (default "https://api.github.com/repos/cert-manager/cert-manager/releases") + --cert-manager-version string version for cert-manager (default "v1.13.2") + -c, --controller-name string name of the controller that's used for status check (default "ocm-controller") + -d, --dry-run if enabled, prints the downloaded manifest file + -h, --help help for uninstall + -n, --namespace string the namespace into which the controller is installed (default "ocm-system") + -a, --release-api-url string the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases") + -l, --silent don't fail on error + -t, --timeout duration maximum time to wait for deployment to be ready (default 1m0s) + -p, --uninstall-prerequisites uninstall prerequisites required by ocm-controller + -v, --version string the version of the controller to install (default "latest") +``` + +### SEE ALSO + +##### Parents + +* [ocm controller](ocm_controller.md) — Commands acting on the ocm-controller +* [ocm](ocm.md) — Open Component Model command line client + diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index 998d942fda..41c466d1a4 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -157,25 +157,23 @@ The following credential consumer types are used/supported: - certificateAuthority: TLS certificate authority - - OCIRegistry: OCI registry credential matcher + - MavenRepository: MVN repository - It matches the OCIRegistry consumer type and additionally acts like + It matches the MavenRepository consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + Credential consumers of the consumer type MavenRepository evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password - - identityToken: the bearer token used for non-basic auth authorization - - certificateAuthority: the certificate authority certificate used to verify certificates - - Registry.npmjs.com: NPM repository + - NpmRegistry: NPM repository - It matches the Registry.npmjs.com consumer type and additionally acts like + It matches the NpmRegistry consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + Credential consumers of the consumer type NpmRegistry evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password @@ -183,6 +181,19 @@ The following credential consumer types are used/supported: - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + - OCIRegistry: OCI registry credential matcher + + It matches the OCIRegistry consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - identityToken: the bearer token used for non-basic auth authorization + - certificateAuthority: the certificate authority certificate used to verify certificates + + - S3: S3 credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index b33dcb9eb2..39fd7e8d16 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -83,25 +83,23 @@ Matchers exist for the following usage contexts or consumer types: - certificateAuthority: TLS certificate authority - - OCIRegistry: OCI registry credential matcher + - MavenRepository: MVN repository - It matches the OCIRegistry consumer type and additionally acts like + It matches the MavenRepository consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + Credential consumers of the consumer type MavenRepository evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password - - identityToken: the bearer token used for non-basic auth authorization - - certificateAuthority: the certificate authority certificate used to verify certificates - - Registry.npmjs.com: NPM repository + - NpmRegistry: NPM repository - It matches the Registry.npmjs.com consumer type and additionally acts like + It matches the NpmRegistry consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + Credential consumers of the consumer type NpmRegistry evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password @@ -109,6 +107,19 @@ Matchers exist for the following usage contexts or consumer types: - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + - OCIRegistry: OCI registry credential matcher + + It matches the OCIRegistry consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - identityToken: the bearer token used for non-basic auth authorization + - certificateAuthority: the certificate authority certificate used to verify certificates + + - S3: S3 credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_logging.md b/docs/reference/ocm_logging.md index 08a6292a2c..f72d0b3fcb 100644 --- a/docs/reference/ocm_logging.md +++ b/docs/reference/ocm_logging.md @@ -18,7 +18,6 @@ The following *tags* are used by the command line tool: The following *realms* are used by the command line tool: - ocm: general realm used for the ocm go library. - - ocm/NPM: NPM registry - ocm/accessmethod/ociartifact: access method ociArtifact - ocm/accessmethod/wget: access method for wget - ocm/blobaccess/wget: blob access for wget @@ -28,6 +27,8 @@ The following *realms* are used by the command line tool: - ocm/credentials/dockerconfig: docker config handling as credential repository - ocm/credentials/vault: HashiCorp Vault Access - ocm/downloader: Downloaders + - ocm/mvn: Maven repository + - ocm/npm: NPM registry - ocm/oci/mapping: OCM to OCI Registry Mapping - ocm/oci/ocireg: OCI repository handling - ocm/plugins: OCM plugin handling diff --git a/docs/reference/ocm_ocm-accessmethods.md b/docs/reference/ocm_ocm-accessmethods.md index f94353937d..1c02801075 100644 --- a/docs/reference/ocm_ocm-accessmethods.md +++ b/docs/reference/ocm_ocm-accessmethods.md @@ -183,6 +183,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_ocm-uploadhandlers.md b/docs/reference/ocm_ocm-uploadhandlers.md index 767da82a6d..d3b3219957 100644 --- a/docs/reference/ocm_ocm-uploadhandlers.md +++ b/docs/reference/ocm_ocm-uploadhandlers.md @@ -60,10 +60,6 @@ The following handler names are possible: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -73,6 +69,19 @@ The following handler names are possible: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_commontransportarchive.md b/docs/reference/ocm_transfer_commontransportarchive.md index faa7bfe6a1..c3f6c2d20a 100644 --- a/docs/reference/ocm_transfer_commontransportarchive.md +++ b/docs/reference/ocm_transfer_commontransportarchive.md @@ -134,10 +134,6 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -147,6 +143,19 @@ The uploader name may be a path expression with the following possibilities: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index cbbae63fe8..f251617d44 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -191,10 +191,6 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -204,6 +200,19 @@ The uploader name may be a path expression with the following possibilities: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using From 8307feb6de07008bd01a90eeb2502b1875f17bcc Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 17 May 2024 12:01:51 +0200 Subject: [PATCH 095/100] AccessSpec.Describe --- pkg/contexts/ocm/accessmethods/mvn/method.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 6681f6b51d..d4fb864450 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -101,7 +101,7 @@ func WithExtension(e string) Option { } func (a *AccessSpec) Describe(_ accspeccpi.Context) string { - return fmt.Sprintf("Maven (mvn) package %s:%s:%s in repository %s", a.GroupId, a.ArtifactId, a.Version, a.Repository) + return fmt.Sprintf("Maven (mvn) package '%s' in repository '%s' path '%s'", a.Coordinates.String(), a.Repository, a.Coordinates.FilePath()) } func (_ *AccessSpec) IsLocal(accspeccpi.Context) bool { From 49a12ab14558b61914aba824f6ed3e2add1f2eba Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 17 May 2024 12:02:39 +0200 Subject: [PATCH 096/100] utils.FileSystem --- pkg/contexts/ocm/accessmethods/mvn/method.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index d4fb864450..8a067a528f 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -28,6 +28,7 @@ import ( "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" "github.com/open-component-model/ocm/pkg/utils/tarutils" ) @@ -278,9 +279,9 @@ func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[s } func (a *AccessSpec) GavFiles(ctx accspeccpi.Context, fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { - if strings.HasPrefix(a.Repository, "file://") && len(fs) > 0 { + if strings.HasPrefix(a.Repository, "file://") { dir := a.Repository[7:] - return gavFilesFromDisk(fs[0], dir) + return gavFilesFromDisk(utils.FileSystem(fs...), dir) } return a.gavOnlineFiles(ctx) } @@ -339,7 +340,7 @@ func filesAndHashes(fileList []string) map[string]crypto.Hash { sort.Strings(fileList) // Which hash files are available? - result := make(map[string]crypto.Hash) + result := make(map[string]crypto.Hash, len(fileList)/2) for _, file := range fileList { if IsResource(file) { result[file] = bestAvailableHash(fileList, file) From b488a48f05e76747c7d2918a1627ecb642add395 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 17 May 2024 12:03:38 +0200 Subject: [PATCH 097/100] clearly separate fail from success paths --- .../credentials/builtin/mvn/identity/identity.go | 1 - pkg/contexts/ocm/accessmethods/mvn/method_test.go | 9 +++++++-- .../fail/{ => test}/repository/42/repository-42.pom | 0 .../fail/{ => test}/repository/42/repository-42.pom.sha1 | 0 4 files changed, 7 insertions(+), 3 deletions(-) rename pkg/contexts/ocm/accessmethods/mvn/testdata/fail/{ => test}/repository/42/repository-42.pom (100%) rename pkg/contexts/ocm/accessmethods/mvn/testdata/fail/{ => test}/repository/42/repository-42.pom.sha1 (100%) diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go index 2c6e27aa61..0a5bf6fc74 100644 --- a/pkg/contexts/credentials/builtin/mvn/identity/identity.go +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -63,7 +63,6 @@ func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (cpi.Crede return nil, nil } return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) - } func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) (err error) { diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go index e82c105acb..7b4ca85a80 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method_test.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -18,7 +18,7 @@ import ( const ( mvnPATH = "/testdata/.m2/repository" - FAILPATH = "/testdata" + FAILPATH = "/testdata/fail" ) var _ = Describe("local accessmethods.mvn.AccessSpec tests", func() { @@ -72,8 +72,13 @@ var _ = Describe("local accessmethods.mvn.AccessSpec tests", func() { Expect(dr.Digest().String()).To(Equal("SHA-1:34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) }) + It("Describe", func() { + acc := mvn.New("file://"+FAILPATH, "test", "repository", "42", mvn.WithExtension("pom")) + Expect(acc.Describe(nil)).To(Equal("Maven (mvn) package 'test:repository:42::pom' in repository 'file:///testdata/fail' path 'test/repository/42/repository-42.pom'")) + }) + It("detects digests mismatch", func() { - acc := mvn.New("file://"+FAILPATH, "fail", "repository", "42", mvn.WithExtension("pom")) + acc := mvn.New("file://"+FAILPATH, "test", "repository", "42", mvn.WithExtension("pom")) m := Must(acc.AccessMethod(cv)) defer m.Close() _, err := m.Reader() diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom similarity index 100% rename from pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom rename to pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom.sha1 similarity index 100% rename from pkg/contexts/ocm/accessmethods/mvn/testdata/fail/repository/42/repository-42.pom.sha1 rename to pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom.sha1 From edc557a0858dec12d77c49285ee28ac098671b6a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 17 May 2024 14:39:27 +0200 Subject: [PATCH 098/100] not using named return --- pkg/contexts/ocm/accessmethods/mvn/method.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index 8a067a528f..d04d13d0ae 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -214,22 +214,22 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { update(a, file, hash, &metadata, ctx, fs) // download the artifact into the temporary file system - e := func() (err error) { + e := func() error { out, err := tempFs.Create(file) if err != nil { - return + return err } defer out.Close() reader, err := getReader(ctx, metadata.Bin, fs) if err != nil { - return + return err } defer reader.Close() if hash > 0 { dreader := iotools.NewDigestReaderWithHash(hash, reader) _, err = io.Copy(out, dreader) if err != nil { - return + return err } sum := dreader.Digest().Encoded() if metadata.Hash != sum { @@ -237,9 +237,9 @@ func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { } } else { _, err = io.Copy(out, reader) - return + return err } - return + return err }() if e != nil { return nil, e From 69df5a7c05f5d4030c082c0be8f5634c89bf92f6 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 17 May 2024 15:52:55 +0200 Subject: [PATCH 099/100] add path --- pkg/contexts/ocm/accessmethods/mvn/method.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go index d04d13d0ae..55da45fe0b 100644 --- a/pkg/contexts/ocm/accessmethods/mvn/method.go +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "path" "sort" "strings" @@ -280,7 +281,7 @@ func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[s func (a *AccessSpec) GavFiles(ctx accspeccpi.Context, fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { if strings.HasPrefix(a.Repository, "file://") { - dir := a.Repository[7:] + dir := path.Join(a.Repository[7:], a.GavPath()) return gavFilesFromDisk(utils.FileSystem(fs...), dir) } return a.gavOnlineFiles(ctx) From 09913068162c4e35af1a316dc0c7cf4b71fc624a Mon Sep 17 00:00:00 2001 From: Fabian Burth Date: Tue, 21 May 2024 15:15:12 +0200 Subject: [PATCH 100/100] Fix Makefile --- hack/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/Makefile b/hack/Makefile index aeab22f85b..d2de231042 100644 --- a/hack/Makefile +++ b/hack/Makefile @@ -45,12 +45,12 @@ ifneq ("v$(GO_BINDATA)",$(GO_BINDATA_VERSION)) endif VAULT_VERSION := 1.16.2 VAULT := $(shell ($(LOCALBIN)/vault --version 2>/dev/null || echo 0.0) | sed 's/.*Vault v\([0-9\.]*\).*/\1/') -ifeq ($(VAULT), $(VAULT_VERSION)) +ifneq ($(VAULT), $(VAULT_VERSION)) deps += vault endif OCI_REGISTRY_VERSION := 3.0.0-alpha.1 OCI_REGISTRY := $(shell (registry --version 2>/dev/null || echo 0.0) | sed 's/.* v\([0-9a-z\.\-]*\).*/\1/') -ifeq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION)) +ifneq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION)) deps += oci-registry endif