diff --git a/.reuse/dep5 b/.reuse/dep5 index 30a2631ec6..e75df02db4 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -28,10 +28,6 @@ Files: ** Copyright: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors License: Apache-2.0 -Files: pkg/mimeutils/* -Copyright: Copyright 2010 The Go Authors. All rights reserved. -License: BSD-3-Clause - Files: pkg/contexts/ocm/blobhandler/handlers/generic/npm/publish.go Copyright: Copyright 2021 - cloverstd License: MIT diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt deleted file mode 100644 index ea890afbc7..0000000000 --- a/LICENSES/BSD-3-Clause.txt +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) . - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/pluginreference/plugin_accessmethod_compose.md b/docs/pluginreference/plugin_accessmethod_compose.md index 5ffbea5834..d9c0455651 100644 --- a/docs/pluginreference/plugin_accessmethod_compose.md +++ b/docs/pluginreference/plugin_accessmethod_compose.md @@ -35,6 +35,9 @@ by the plugin name. The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/pluginreference/plugin_descriptor.md b/docs/pluginreference/plugin_descriptor.md index f04a2169f4..c3ecbdd8a4 100644 --- a/docs/pluginreference/plugin_descriptor.md +++ b/docs/pluginreference/plugin_descriptor.md @@ -120,6 +120,9 @@ It uses the following fields: The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/pluginreference/plugin_valueset_compose.md b/docs/pluginreference/plugin_valueset_compose.md index 0ad1025ed4..09bf007037 100644 --- a/docs/pluginreference/plugin_valueset_compose.md +++ b/docs/pluginreference/plugin_valueset_compose.md @@ -35,6 +35,9 @@ by the plugin name. The following predefined option types can be used: + - accessClassifier: [*string*] mvn classifier + - accessExtension: [*string*] mvn extension name + - accessGroup: [*string*] GroupID or namespace - accessHostname: [*string*] hostname used for access - accessPackage: [*string*] package or object name - accessRegistry: [*string*] registry base URL diff --git a/docs/reference/ocm_add_resource-configuration.md b/docs/reference/ocm_add_resource-configuration.md index fdc3daaee8..679bcc2f27 100644 --- a/docs/reference/ocm_add_resource-configuration.md +++ b/docs/reference/ocm_add_resource-configuration.md @@ -24,6 +24,9 @@ resource-configuration, resourceconfig, rsccfg, rcfg ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -680,6 +683,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_resources.md b/docs/reference/ocm_add_resources.md index d755180f4e..9e8878f8e8 100644 --- a/docs/reference/ocm_add_resources.md +++ b/docs/reference/ocm_add_resources.md @@ -30,6 +30,9 @@ resources, resource, res, r ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -690,6 +693,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_source-configuration.md b/docs/reference/ocm_add_source-configuration.md index 25a8f5f5a0..f7d2b4fc32 100644 --- a/docs/reference/ocm_add_source-configuration.md +++ b/docs/reference/ocm_add_source-configuration.md @@ -24,6 +24,9 @@ source-configuration, sourceconfig, srccfg, scfg ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -680,6 +683,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_add_sources.md b/docs/reference/ocm_add_sources.md index ed829fb938..746d02c5ee 100644 --- a/docs/reference/ocm_add_sources.md +++ b/docs/reference/ocm_add_sources.md @@ -29,6 +29,9 @@ sources, source, src, s ``` --access YAML blob access specification (YAML) + --accessClassifier string mvn classifier + --accessExtension string mvn extension name + --accessGroup string GroupID or namespace --accessHostname string hostname used for access --accessPackage string package or object name --accessRegistry string registry base URL @@ -688,6 +691,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_controller_install.md b/docs/reference/ocm_controller_install.md index 701fd35056..8ed8c1c02c 100644 --- a/docs/reference/ocm_controller_install.md +++ b/docs/reference/ocm_controller_install.md @@ -19,6 +19,7 @@ ocm controller install controller {--version v0.0.1} -i, --install-prerequisites install prerequisites required by ocm-controller (default true) -n, --namespace string the namespace into which the controller is installed (default "ocm-system") -a, --release-api-url string the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases") + -l, --silent don't fail on error -s, --skip-pre-flight-check skip the pre-flight check for clusters -t, --timeout duration maximum time to wait for deployment to be ready (default 1m0s) -v, --version string the version of the controller to install (default "latest") diff --git a/docs/reference/ocm_controller_uninstall.md b/docs/reference/ocm_controller_uninstall.md index 2d4c0f1734..e13c34927c 100644 --- a/docs/reference/ocm_controller_uninstall.md +++ b/docs/reference/ocm_controller_uninstall.md @@ -18,6 +18,7 @@ ocm controller uninstall controller -h, --help help for uninstall -n, --namespace string the namespace into which the controller is installed (default "ocm-system") -a, --release-api-url string the base url to the ocm-controller's API release page (default "https://api.github.com/repos/open-component-model/ocm-controller/releases") + -l, --silent don't fail on error -t, --timeout duration maximum time to wait for deployment to be ready (default 1m0s) -p, --uninstall-prerequisites uninstall prerequisites required by ocm-controller -v, --version string the version of the controller to install (default "latest") diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index fdf51d3363..28e70c6c12 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -156,25 +156,23 @@ The following credential consumer types are used/supported: - certificateAuthority: TLS certificate authority - - OCIRegistry: OCI registry credential matcher + - MavenRepository: MVN repository - It matches the OCIRegistry consumer type and additionally acts like + It matches the MavenRepository consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + Credential consumers of the consumer type MavenRepository evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password - - identityToken: the bearer token used for non-basic auth authorization - - certificateAuthority: the certificate authority certificate used to verify certificates - - Registry.npmjs.com: NPM repository + - NpmRegistry: NPM repository - It matches the Registry.npmjs.com consumer type and additionally acts like + It matches the NpmRegistry consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + Credential consumers of the consumer type NpmRegistry evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password @@ -182,6 +180,19 @@ The following credential consumer types are used/supported: - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + - OCIRegistry: OCI registry credential matcher + + It matches the OCIRegistry consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - identityToken: the bearer token used for non-basic auth authorization + - certificateAuthority: the certificate authority certificate used to verify certificates + + - S3: S3 credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index cf3d1cbc4d..c97d24bb06 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -82,25 +82,23 @@ Matchers exist for the following usage contexts or consumer types: - certificateAuthority: TLS certificate authority - - OCIRegistry: OCI registry credential matcher + - MavenRepository: MVN repository - It matches the OCIRegistry consumer type and additionally acts like + It matches the MavenRepository consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + Credential consumers of the consumer type MavenRepository evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password - - identityToken: the bearer token used for non-basic auth authorization - - certificateAuthority: the certificate authority certificate used to verify certificates - - Registry.npmjs.com: NPM repository + - NpmRegistry: NPM repository - It matches the Registry.npmjs.com consumer type and additionally acts like + It matches the NpmRegistry consumer type and additionally acts like the hostpath type. - Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + Credential consumers of the consumer type NpmRegistry evaluate the following credential properties: - username: the basic auth user name - password: the basic auth password @@ -108,6 +106,19 @@ Matchers exist for the following usage contexts or consumer types: - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + - OCIRegistry: OCI registry credential matcher + + It matches the OCIRegistry consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type OCIRegistry evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - identityToken: the bearer token used for non-basic auth authorization + - certificateAuthority: the certificate authority certificate used to verify certificates + + - S3: S3 credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_logging.md b/docs/reference/ocm_logging.md index 08a6292a2c..f72d0b3fcb 100644 --- a/docs/reference/ocm_logging.md +++ b/docs/reference/ocm_logging.md @@ -18,7 +18,6 @@ The following *tags* are used by the command line tool: The following *realms* are used by the command line tool: - ocm: general realm used for the ocm go library. - - ocm/NPM: NPM registry - ocm/accessmethod/ociartifact: access method ociArtifact - ocm/accessmethod/wget: access method for wget - ocm/blobaccess/wget: blob access for wget @@ -28,6 +27,8 @@ The following *realms* are used by the command line tool: - ocm/credentials/dockerconfig: docker config handling as credential repository - ocm/credentials/vault: HashiCorp Vault Access - ocm/downloader: Downloaders + - ocm/mvn: Maven repository + - ocm/npm: NPM registry - ocm/oci/mapping: OCM to OCI Registry Mapping - ocm/oci/ocireg: OCI repository handling - ocm/plugins: OCM plugin handling diff --git a/docs/reference/ocm_ocm-accessmethods.md b/docs/reference/ocm_ocm-accessmethods.md index f94353937d..1c02801075 100644 --- a/docs/reference/ocm_ocm-accessmethods.md +++ b/docs/reference/ocm_ocm-accessmethods.md @@ -183,6 +183,41 @@ shown below. Options used to configure fields: --globalAccess, --hint, --mediaType, --reference +- Access type mvn + + This method implements the access of a Maven (mvn) artifact in a Maven repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repository** *string* + + Base URL of the Maven (mvn) repository + + - **groupId** *string* + + The groupId of the Maven (mvn) artifact + + - **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + + - **version** *string* + + The version name of the Maven (mvn) artifact + + - **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + + - **extension** *string* + + The optional extension of the Maven (mvn) artifact + + Options used to configure fields: --accessClassifier, --accessExtension, --accessGroup, --accessPackage, --accessRepository, --accessVersion + - Access type none dummy resource with no access diff --git a/docs/reference/ocm_ocm-uploadhandlers.md b/docs/reference/ocm_ocm-uploadhandlers.md index 767da82a6d..d3b3219957 100644 --- a/docs/reference/ocm_ocm-uploadhandlers.md +++ b/docs/reference/ocm_ocm-uploadhandlers.md @@ -60,10 +60,6 @@ The following handler names are possible: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -73,6 +69,19 @@ The following handler names are possible: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_commontransportarchive.md b/docs/reference/ocm_transfer_commontransportarchive.md index faa7bfe6a1..c3f6c2d20a 100644 --- a/docs/reference/ocm_transfer_commontransportarchive.md +++ b/docs/reference/ocm_transfer_commontransportarchive.md @@ -134,10 +134,6 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -147,6 +143,19 @@ The uploader name may be a path expression with the following possibilities: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index cbbae63fe8..f251617d44 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -191,10 +191,6 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/npmPackage: uploading npm artifacts The ocm/npmPackage uploader is able to upload npm artifacts @@ -204,6 +200,19 @@ The uploader name may be a path expression with the following possibilities: It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npm repository. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/mvnArtifact: uploading mvn artifacts + + The ocm/mvnArtifact uploader is able to upload mvn artifacts (whole GAV only!) + as artifact archive according to the mvn artifact spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the mvn repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/hack/Makefile b/hack/Makefile index aeab22f85b..d2de231042 100644 --- a/hack/Makefile +++ b/hack/Makefile @@ -45,12 +45,12 @@ ifneq ("v$(GO_BINDATA)",$(GO_BINDATA_VERSION)) endif VAULT_VERSION := 1.16.2 VAULT := $(shell ($(LOCALBIN)/vault --version 2>/dev/null || echo 0.0) | sed 's/.*Vault v\([0-9\.]*\).*/\1/') -ifeq ($(VAULT), $(VAULT_VERSION)) +ifneq ($(VAULT), $(VAULT_VERSION)) deps += vault endif OCI_REGISTRY_VERSION := 3.0.0-alpha.1 OCI_REGISTRY := $(shell (registry --version 2>/dev/null || echo 0.0) | sed 's/.* v\([0-9a-z\.\-]*\).*/\1/') -ifeq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION)) +ifneq ($(OCI_REGISTRY), $(OCI_REGISTRY_VERSION)) deps += oci-registry endif diff --git a/hack/tools.go b/hack/tools.go index b76c619e68..5b15c7ab40 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -1,5 +1,4 @@ //go:build tools -// +build tools package tools diff --git a/pkg/blobaccess/wget/access.go b/pkg/blobaccess/wget/access.go index d2611910c3..fcd2b22552 100644 --- a/pkg/blobaccess/wget/access.go +++ b/pkg/blobaccess/wget/access.go @@ -16,7 +16,6 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/wget/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" ocmmime "github.com/open-component-model/ocm/pkg/mime" - "github.com/open-component-model/ocm/pkg/mimeutils" "github.com/open-component-model/ocm/pkg/optionutils" "github.com/open-component-model/ocm/pkg/utils" ) @@ -133,7 +132,7 @@ func BlobAccessForWget(url string, opts ...Option) (_ blobaccess.BlobAccess, rer "extract mime type from url") ext, err := utils.GetFileExtensionFromUrl(url) if err == nil && ext != "" { - eff.MimeType = mimeutils.TypeByExtension(ext) + eff.MimeType = mime.TypeByExtension(ext) } else if err != nil { log.Debug(err.Error()) } diff --git a/pkg/cobrautils/flag/path_array_test.go b/pkg/cobrautils/flag/path_array_test.go index 79bc5352bc..c403dc7b67 100644 --- a/pkg/cobrautils/flag/path_array_test.go +++ b/pkg/cobrautils/flag/path_array_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package flag_test diff --git a/pkg/cobrautils/flag/path_array_win_test.go b/pkg/cobrautils/flag/path_array_win_test.go index a0fd2bf1ad..888954e4a9 100644 --- a/pkg/cobrautils/flag/path_array_win_test.go +++ b/pkg/cobrautils/flag/path_array_win_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package flag_test diff --git a/pkg/cobrautils/flag/path_test.go b/pkg/cobrautils/flag/path_test.go index 8440a9ce96..1de8cecc33 100644 --- a/pkg/cobrautils/flag/path_test.go +++ b/pkg/cobrautils/flag/path_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package flag_test diff --git a/pkg/cobrautils/flag/path_win_test.go b/pkg/cobrautils/flag/path_win_test.go index 4220aa270c..bea185436b 100644 --- a/pkg/cobrautils/flag/path_win_test.go +++ b/pkg/cobrautils/flag/path_win_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package flag_test diff --git a/pkg/contexts/credentials/builtin/mvn/identity/identity.go b/pkg/contexts/credentials/builtin/mvn/identity/identity.go new file mode 100644 index 0000000000..0a5bf6fc74 --- /dev/null +++ b/pkg/contexts/credentials/builtin/mvn/identity/identity.go @@ -0,0 +1,82 @@ +package identity + +import ( + "errors" + "net/http" + + . "net/url" + + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" + "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/logging" +) + +const ( + // CONSUMER_TYPE is the mvn repository type. + CONSUMER_TYPE = "MavenRepository" + + // ATTR_USERNAME is the username attribute. Required for login at any mvn registry. + ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any mvn registry. + ATTR_PASSWORD = cpi.ATTR_PASSWORD +) + +// REALM the logging realm / prefix. +var REALM = logging.DefineSubRealm("Maven repository", "mvn") + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + }) + + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `MVN repository + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) + +func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { + return identityMatcher(pattern, cur, id) +} + +func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) { + url, err := JoinPath(rawURL, groupId) + if err != nil { + return nil, err + } + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url), nil +} + +func GetCredentials(ctx cpi.ContextProvider, repoUrl, groupId string) (cpi.Credentials, error) { + id, err := GetConsumerId(repoUrl, groupId) + if err != nil { + return nil, err + } + if id == nil { + logging.DynamicLogger(REALM).Debug("No consumer identity found.", "url", repoUrl, "groupId", groupId) + return nil, nil + } + return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) +} + +func BasicAuth(req *http.Request, ctx accspeccpi.Context, repoUrl, groupId string) (err error) { + credentials, err := GetCredentials(ctx, repoUrl, groupId) + if err != nil { + return err + } + if credentials == nil { + logging.DynamicLogger(REALM).Debug("No credentials found. BasicAuth not required?", "url", repoUrl, "groupId", groupId) + return nil + } + if !credentials.ExistsProperty(ATTR_USERNAME) || !credentials.ExistsProperty(ATTR_PASSWORD) { + return errors.New("missing username or password in credentials") + } + req.SetBasicAuth(credentials.GetProperty(ATTR_USERNAME), credentials.GetProperty(ATTR_PASSWORD)) + return +} diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index 32f9a16e8c..308cf8d299 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -14,7 +14,7 @@ import ( const ( // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npmjs.com" + CONSUMER_TYPE = "NpmRegistry" // ATTR_USERNAME is the username attribute. Required for login at any npm registry. ATTR_USERNAME = cpi.ATTR_USERNAME @@ -27,7 +27,7 @@ const ( ) // Logging Realm. -var REALM = logging.DefineSubRealm("NPM registry", "NPM") +var REALM = logging.DefineSubRealm("NPM registry", "npm") func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ diff --git a/pkg/contexts/ocm/accessmethods/init.go b/pkg/contexts/ocm/accessmethods/init.go index 52052609d6..f0a70ce7dd 100644 --- a/pkg/contexts/ocm/accessmethods/init.go +++ b/pkg/contexts/ocm/accessmethods/init.go @@ -6,6 +6,7 @@ import ( _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localfsblob" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localociblob" + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/none" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" diff --git a/pkg/contexts/ocm/accessmethods/mvn/README.md b/pkg/contexts/ocm/accessmethods/mvn/README.md new file mode 100644 index 0000000000..ecb696d7a6 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/README.md @@ -0,0 +1,119 @@ +# `mvn` - Maven artifacts (Java packages, jars) in a Maven (mvn) repository (e.g. mvnrepository.com) + +### Synopsis +``` +type: mvn/v1 +``` + +Provided blobs use the following media type: `application/x-tgz` + +### Description + +This method implements the access of a resource hosted by a maven repository or a +complete resource set denoted by a GAV (GroupId, ArtifactId, Version). + +### Specification Versions + +Supported specification version is `v1` + +#### Version `v1` + +The type specific specification fields are: + +- **`repository`** *string* + + Base URL of the Maven (mvn) repository + +- **`groupId`** *string* + + The groupId of the Maven (mvn) artifact + +- **`artifactId`** *string* + + The artifactId of the Maven (mvn) artifact + +- **`version`** *string* + + The version name of the Maven (mvn) artifact + +- **`classifier`** *string* + + The optional classifier of the Maven (mvn) artifact + +- **`extension`** *string* + + The optional extension of the Maven (mvn) artifact + +If classifier/extension is given a dedicated resource is described, +otherwise the complete resource set described by a GAV. +Only complete resource sets can be uploaded again to a Maven repository. + +#### Examples + +##### Complete resource set denoted by a GAV + +```yaml +name: acme.org/complete/gav +version: 0.0.1 +provider: + name: acme.org +resources: + - name: java-sap-vcap-services + type: mvnArtifact + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: com.sap.cloud.environment.servicebinding + artifactId: java-sap-vcap-services + version: 0.10.4 +``` + +##### Single pom.xml file + +This can't be uploaded again into a Maven repository, but it can be used to describe the dependencies of a project. +The mime type will be `application/xml`. + +```yaml +name: acme.org/single/pom +version: 0.0.1 +provider: + name: acme.org +resources: + - name: sap-cloud-sdk + type: pom + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: com.sap.cloud.sdk + artifactId: sdk-modules-bom + version: 5.7.0 + classifier: '' + extension: pom +``` + +##### Single binary file + +In case you want to download and install maven itself, you can use the following example. +This can't be uploaded again into a Maven repository. +The mime type will be `application/gzip`. + +```yaml +name: acme.org/bin/zip +version: 0.0.1 +provider: + name: acme.org +resources: + - name: maven + type: bin + version: 0.0.1 + access: + type: mvn + repository: https://repo1.maven.org/maven2 + groupId: org.apache.maven + artifactId: apache-maven + version: 3.9.6 + classifier: bin + extension: zip +``` diff --git a/pkg/contexts/ocm/accessmethods/mvn/cli.go b/pkg/contexts/ocm/accessmethods/mvn/cli.go new file mode 100644 index 0000000000..dd81479a6b --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/cli.go @@ -0,0 +1,62 @@ +package mvn + +import ( + "github.com/open-component-model/ocm/pkg/cobrautils/flagsets" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/options" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + Type, AddConfig, + options.RepositoryOption, + options.GroupOption, + options.PackageOption, + options.VersionOption, + // optional + options.ClassifierOption, + options.ExtensionOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository") + flagsets.AddFieldByOptionP(opts, options.GroupOption, config, "groupId") + flagsets.AddFieldByOptionP(opts, options.PackageOption, config, "artifactId") + flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "version") + // optional + flagsets.AddFieldByOptionP(opts, options.ClassifierOption, config, "classifier") + flagsets.AddFieldByOptionP(opts, options.ExtensionOption, config, "extension") + return nil +} + +var usage = ` +This method implements the access of a Maven (mvn) artifact in a Maven repository. +` + +var formatV1 = ` +The type specific specification fields are: + +- **repository** *string* + + Base URL of the Maven (mvn) repository + +- **groupId** *string* + + The groupId of the Maven (mvn) artifact + +- **artifactId** *string* + + The artifactId of the Maven (mvn) artifact + +- **version** *string* + + The version name of the Maven (mvn) artifact + +- **classifier** *string* + + The optional classifier of the Maven (mvn) artifact + +- **extension** *string* + + The optional extension of the Maven (mvn) artifact +` diff --git a/pkg/contexts/ocm/accessmethods/mvn/coordinates.go b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go new file mode 100644 index 0000000000..d39e09c2e5 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/coordinates.go @@ -0,0 +1,144 @@ +package mvn + +import ( + "fmt" + "mime" + "path" + "path/filepath" + "strings" + + ocmmime "github.com/open-component-model/ocm/pkg/mime" +) + +// Coordinates holds the typical Maven coordinates groupId, artifactId, version. Optional also classifier and extension. +// https://maven.apache.org/ref/3.9.6/maven-core/artifact-handlers.html +type Coordinates struct { + // GroupId of the Maven (mvn) artifact. + GroupId string `json:"groupId"` + // ArtifactId of the Maven (mvn) artifact. + ArtifactId string `json:"artifactId"` + // Version of the Maven (mvn) artifact. + Version string `json:"version"` + // Classifier of the Maven (mvn) artifact. + Classifier string `json:"classifier"` + // Extension of the Maven (mvn) artifact. + Extension string `json:"extension"` +} + +// GAV returns the GAV coordinates of the Maven Coordinates. +func (c *Coordinates) GAV() string { + return c.GroupId + ":" + c.ArtifactId + ":" + c.Version +} + +// String returns the Coordinates as a string (GroupId:ArtifactId:Version:Classifier:Extension). +func (c *Coordinates) String() string { + return c.GroupId + ":" + c.ArtifactId + ":" + c.Version + ":" + c.Classifier + ":" + c.Extension +} + +// GavPath returns the Maven repository path. +func (c *Coordinates) GavPath() string { + return c.GroupPath() + "/" + c.ArtifactId + "/" + c.Version +} + +// FilePath returns the Maven Coordinates's GAV-name with classifier and extension. +// Which is equal to the URL-path of the artifact in the repository. +// Default extension is jar. +func (c *Coordinates) FilePath() string { + path := c.GavPath() + "/" + c.FileNamePrefix() + if c.Classifier != "" { + path += "-" + c.Classifier + } + if c.Extension != "" { + path += "." + c.Extension + } else { + path += ".jar" + } + return path +} + +func (c *Coordinates) Url(baseUrl string) string { + return baseUrl + "/" + c.FilePath() +} + +// GroupPath returns GroupId with `/` instead of `.`. +func (c *Coordinates) GroupPath() string { + return strings.ReplaceAll(c.GroupId, ".", "/") +} + +func (c *Coordinates) FileNamePrefix() string { + return c.ArtifactId + "-" + c.Version +} + +// Purl returns the Package URL of the Maven Coordinates. +func (c *Coordinates) Purl() string { + return "pkg:maven/" + c.GroupId + "/" + c.ArtifactId + "@" + c.Version +} + +// SetClassifierExtensionBy extracts the classifier and extension from the filename (without any path prefix). +func (c *Coordinates) SetClassifierExtensionBy(filename string) error { + s := strings.TrimPrefix(path.Base(filename), c.FileNamePrefix()) + if strings.HasPrefix(s, "-") { + s = strings.TrimPrefix(s, "-") + i := strings.Index(s, ".") + if i < 0 { + return fmt.Errorf("no extension after classifier found in filename: %s", filename) + } + c.Classifier = s[:i] + s = strings.TrimPrefix(s, c.Classifier) + } else { + c.Classifier = "" + } + c.Extension = strings.TrimPrefix(s, ".") + return nil +} + +// MimeType returns the MIME type of the Maven Coordinates based on the file extension. +// Default is application/x-tgz. +func (c *Coordinates) MimeType() string { + m := mime.TypeByExtension("." + c.Extension) + if m != "" { + return m + } + return ocmmime.MIME_TGZ +} + +// Copy creates a new Coordinates with the same values. +func (c *Coordinates) Copy() *Coordinates { + return &Coordinates{ + GroupId: c.GroupId, + ArtifactId: c.ArtifactId, + Version: c.Version, + Classifier: c.Classifier, + Extension: c.Extension, + } +} + +// Parse creates an Coordinates from it's serialized form (see Coordinates.String). +func Parse(serializedArtifact string) (*Coordinates, error) { + parts := strings.Split(serializedArtifact, ":") + if len(parts) < 3 { + return nil, fmt.Errorf("invalid artifact string: %s", serializedArtifact) + } + artifact := &Coordinates{ + GroupId: parts[0], + ArtifactId: parts[1], + Version: parts[2], + } + if len(parts) >= 4 { + artifact.Classifier = parts[3] + } + if len(parts) >= 5 { + artifact.Extension = parts[4] + } + return artifact, nil +} + +// IsResource returns true if the filename is not a checksum or signature file. +func IsResource(fileName string) bool { + switch filepath.Ext(fileName) { + case ".asc", ".md5", ".sha1", ".sha256", ".sha512": + return false + default: + return true + } +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go b/pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go new file mode 100644 index 0000000000..63df1268ae --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/coordinates_test.go @@ -0,0 +1,55 @@ +package mvn + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Maven Test Environment", func() { + + It("GAV, GroupPath, FilePath", func() { + artifact := &Coordinates{ + GroupId: "ocm.software", + ArtifactId: "hello-ocm", + Version: "0.0.1", + Extension: "jar", + } + Expect(artifact.GAV()).To(Equal("ocm.software:hello-ocm:0.0.1")) + Expect(artifact.GroupPath()).To(Equal("ocm/software")) + Expect(artifact.FilePath()).To(Equal("ocm/software/hello-ocm/0.0.1/hello-ocm-0.0.1.jar")) + }) + + It("SetClassifierExtensionBy", func() { + artifact := &Coordinates{ + GroupId: "ocm.software", + ArtifactId: "hello-ocm", + Version: "0.0.1", + } + artifact.SetClassifierExtensionBy("hello-ocm-0.0.1.pom") + Expect(artifact.Classifier).To(Equal("")) + Expect(artifact.Extension).To(Equal("pom")) + + artifact.SetClassifierExtensionBy("hello-ocm-0.0.1-tests.jar") + Expect(artifact.Classifier).To(Equal("tests")) + Expect(artifact.Extension).To(Equal("jar")) + + artifact.ArtifactId = "apache-maven" + artifact.Version = "3.9.6" + artifact.SetClassifierExtensionBy("apache-maven-3.9.6-bin.tar.gz") + Expect(artifact.Classifier).To(Equal("bin")) + Expect(artifact.Extension).To(Equal("tar.gz")) + }) + + It("parse GAV", func() { + gav := "org.apache.commons:commons-compress:1.26.1:cyclonedx:xml" + artifact, err := Parse(gav) + Expect(err).To(BeNil()) + Expect(artifact.String()).To(Equal(gav)) + Expect(artifact.GroupId).To(Equal("org.apache.commons")) + Expect(artifact.ArtifactId).To(Equal("commons-compress")) + Expect(artifact.Version).To(Equal("1.26.1")) + Expect(artifact.Classifier).To(Equal("cyclonedx")) + Expect(artifact.Extension).To(Equal("xml")) + Expect(artifact.FilePath()).To(Equal("org/apache/commons/commons-compress/1.26.1/commons-compress-1.26.1-cyclonedx.xml")) + }) +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/integration_test.go b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go new file mode 100644 index 0000000000..b2eca2b1d0 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/integration_test.go @@ -0,0 +1,98 @@ +//go:build integration + +package mvn_test + +import ( + "crypto" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + . "github.com/open-component-model/ocm/pkg/env" + . "github.com/open-component-model/ocm/pkg/env/builder" + "github.com/open-component-model/ocm/pkg/mime" + . "github.com/open-component-model/ocm/pkg/testutils" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +var _ = Describe("online accessmethods.mvn.AccessSpec integration tests", func() { + var env *Builder + var cv ocm.ComponentVersionAccess + + BeforeEach(func() { + env = NewBuilder(TestData()) + cv = &cpi.DummyComponentVersionAccess{env.OCMContext()} + }) + + AfterEach(func() { + env.Cleanup() + }) + + // https://repo1.maven.org/maven2/com/sap/cloud/sdk/sdk-modules-bom/5.7.0 + It("one single pom only", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + files, err := acc.GavFiles(cv.GetContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(1)) + Expect(files["sdk-modules-bom-5.7.0.pom"]).To(Equal(crypto.SHA1)) + }) + It("GetPackageMeta - com.sap.cloud.sdk", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(HavePrefix("file://")) + Expect(meta.Bin).To(ContainSubstring("mvn-sdk-modules-bom-5.7.0-")) + Expect(meta.Bin).To(HaveSuffix(".tar.gz")) + Expect(meta.Hash).To(Equal("345fe2e640663c3cd6ac87b7afb92e1c934f665f75ddcb9555bc33e1813ef00b")) + Expect(meta.HashType).To(Equal(crypto.SHA256)) + }) + + // https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6 + It("apache-maven, with bin + tar.gz etc.", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6") + Expect(acc).ToNot(BeNil()) + Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) + files, err := acc.GavFiles(cv.GetContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(8)) + Expect(files["apache-maven-3.9.6-src.zip"]).To(Equal(crypto.SHA512)) + Expect(files["apache-maven-3.9.6.pom"]).To(Equal(crypto.SHA1)) + }) + + // https://repo1.maven.org/maven2/com/sap/cloud/environment/servicebinding/java-sap-vcap-services/0.10.4 + It("accesses local artifact", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "com.sap.cloud.environment.servicebinding", "java-sap-vcap-services", "0.10.4") + meta, err := acc.GetPackageMeta(ocm.DefaultContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Bin).To(HavePrefix("file://")) + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + /* manually also tested with repos: + - https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.26.1/ // cyclonedx + - https://repo1.maven.org/maven2/cn/afternode/commons/commons/1.6/ // gradle module! + */ + }) + + // https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6 + It("apache-maven, 'bin' zip + tar.gz only!", func() { + acc := mvn.New("https://repo1.maven.org/maven2", "org.apache.maven", "apache-maven", "3.9.6", mvn.WithClassifier("bin")) + Expect(acc).ToNot(BeNil()) + Expect(acc.BaseUrl()).To(Equal("https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.6")) + files, err := acc.GavFiles(cv.GetContext()) + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(8)) // the repo contains 8 files... + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(err).ToNot(HaveOccurred()) + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + r := Must(m.Reader()) + defer r.Close() + list, err := tarutils.ListArchiveContentFromReader(r) + Expect(err).ToNot(HaveOccurred()) + Expect(list).To(HaveLen(2)) // ...but with the classifier set, we're interested only in two! + }) +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/method.go b/pkg/contexts/ocm/accessmethods/mvn/method.go new file mode 100644 index 0000000000..55da45fe0b --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/method.go @@ -0,0 +1,444 @@ +package mvn + +import ( + "bytes" + "context" + "crypto" + "fmt" + "io" + "net/http" + "path" + "sort" + "strings" + + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/opencontainers/go-digest" + "golang.org/x/exp/slices" + "golang.org/x/net/html" + + "github.com/open-component-model/ocm/pkg/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" + "github.com/open-component-model/ocm/pkg/contexts/datacontext/attrs/vfsattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/logging" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/optionutils" + "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +// Type is the access type of Maven (mvn) repository. +const ( + Type = "mvn" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) + +func init() { + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) +} + +// AccessSpec describes the access for a Maven (mvn) artifact. +type AccessSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // Repository is the base URL of the Maven (mvn) repository. + Repository string `json:"repository"` + + Coordinates `json:",inline"` +} + +// Option defines the interface function "ApplyTo()". +type Option = optionutils.Option[*AccessSpec] + +var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) + +var log = logging.DynamicLogger(identity.REALM) + +// New creates a new Maven (mvn) repository access spec version v1. +func New(repository, groupId, artifactId, version string, options ...Option) *AccessSpec { + accessSpec := &AccessSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + Repository: repository, + Coordinates: Coordinates{ + GroupId: groupId, + ArtifactId: artifactId, + Version: version, + Classifier: "", + Extension: "", + }, + } + optionutils.ApplyOptions(accessSpec, options...) + return accessSpec +} + +// classifier Option for Maven (mvn) Coordinates. +type classifier string + +func (c classifier) ApplyTo(a *AccessSpec) { + a.Classifier = string(c) +} + +// WithClassifier sets the classifier of the Maven (mvn) artifact. +func WithClassifier(c string) Option { + return classifier(c) +} + +// extension Option for Maven (mvn) Coordinates. +type extension string + +func (e extension) ApplyTo(a *AccessSpec) { + a.Extension = string(e) +} + +// WithExtension sets the extension of the Maven (mvn) artifact. +func WithExtension(e string) Option { + return extension(e) +} + +func (a *AccessSpec) Describe(_ accspeccpi.Context) string { + return fmt.Sprintf("Maven (mvn) package '%s' in repository '%s' path '%s'", a.Coordinates.String(), a.Repository, a.Coordinates.FilePath()) +} + +func (_ *AccessSpec) IsLocal(accspeccpi.Context) bool { + return false +} + +func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpec { + return a +} + +// GetReferenceHint returns the reference hint for the Maven (mvn) artifact. +func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { + return a.String() +} + +func (_ *AccessSpec) GetType() string { + return Type +} + +func (a *AccessSpec) AccessMethod(c accspeccpi.ComponentVersionAccess) (accspeccpi.AccessMethod, error) { + return accspeccpi.AccessMethodForImplementation(newMethod(c, a)) +} + +func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.ComponentVersionAccess) string { + meta, _ := a.GetPackageMeta(access.GetContext()) + if meta != nil { + return meta.Hash + } + return "" +} + +func (a *AccessSpec) BaseUrl() string { + return a.Repository + "/" + a.GavPath() +} + +func (a *AccessSpec) ArtifactUrl() string { + return a.Url(a.Repository) +} + +func (a *AccessSpec) NewArtifact() *Coordinates { + return a.Coordinates.Copy() +} + +type meta struct { + MimeType string `json:"packaging"` + HashType crypto.Hash `json:"hashType"` + Hash string `json:"hash"` + Bin string `json:"bin"` +} + +func update(a *AccessSpec, file string, hash crypto.Hash, metadata *meta, ctx accspeccpi.Context, fs vfs.FileSystem) error { + artifact := a.NewArtifact() + err := artifact.SetClassifierExtensionBy(file) + if err != nil { + return err + } + metadata.Bin = artifact.Url(a.Repository) + log := log.WithValues("file", metadata.Bin) + log.Debug("processing") + metadata.MimeType = artifact.MimeType() + if hash > 0 { + metadata.HashType = hash + metadata.Hash, err = getStringData(ctx, metadata.Bin+hashUrlExt(hash), fs) + if err != nil { + return errors.Wrapf(err, "cannot read %s digest of: %s", hash, metadata.Bin) + } + } else { + log.Warn("no digest available") + } + return nil +} + +func (a *AccessSpec) GetPackageMeta(ctx accspeccpi.Context) (*meta, error) { + fs := vfsattr.Get(ctx) + + log := log.WithValues("BaseUrl", a.BaseUrl()) + fileMap, err := a.GavFiles(ctx, fs) + if err != nil { + return nil, err + } + + if a.Classifier != "" { + fileMap = filterByClassifier(fileMap, a.Classifier) + } + + switch l := len(fileMap); { + case l <= 0: + return nil, errors.New("no maven artifact files found") + case l == 1 && (a.Extension != "" || a.Classifier != ""): + metadata := meta{} + for file, hash := range fileMap { + update(a, file, hash, &metadata, ctx, fs) + } + return &metadata, nil + // default: continue below with: create tempFs where all files can be downloaded to and packed together as tar.gz + } + + if (a.Extension == "") != (a.Classifier == "") { // XOR + log.Warn("Either classifier or extension have been specified, which results in an incomplete GAV!") + } + tempFs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + defer vfs.Cleanup(tempFs) + + metadata := meta{} + for file, hash := range fileMap { + update(a, file, hash, &metadata, ctx, fs) + + // download the artifact into the temporary file system + e := func() error { + out, err := tempFs.Create(file) + if err != nil { + return err + } + defer out.Close() + reader, err := getReader(ctx, metadata.Bin, fs) + if err != nil { + return err + } + defer reader.Close() + if hash > 0 { + dreader := iotools.NewDigestReaderWithHash(hash, reader) + _, err = io.Copy(out, dreader) + if err != nil { + return err + } + sum := dreader.Digest().Encoded() + if metadata.Hash != sum { + return errors.Newf("%s digest mismatch: expected %s, found %s", metadata.HashType, metadata.Hash, sum) + } + } else { + _, err = io.Copy(out, reader) + return err + } + return err + }() + if e != nil { + return nil, e + } + } + + // pack all downloaded files into a tar.gz file + tgz, err := vfs.TempFile(fs, "", Type+"-"+a.NewArtifact().FileNamePrefix()+"-*.tar.gz") + if err != nil { + return nil, err + } + defer tgz.Close() + + dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) + defer dw.Close() + err = tarutils.TgzFs(tempFs, dw) + if err != nil { + return nil, err + } + + metadata.Bin = "file://" + tgz.Name() + metadata.MimeType = mime.MIME_TGZ + metadata.Hash = dw.Digest().Encoded() + metadata.HashType = crypto.SHA256 + log.Debug("created", "file", metadata.Bin) + + return &metadata, nil +} + +func filterByClassifier(fileMap map[string]crypto.Hash, classifier string) map[string]crypto.Hash { + for file := range fileMap { + if !strings.Contains(file, "-"+classifier+".") { + delete(fileMap, file) + } + } + return fileMap +} + +func (a *AccessSpec) GavFiles(ctx accspeccpi.Context, fs ...vfs.FileSystem) (map[string]crypto.Hash, error) { + if strings.HasPrefix(a.Repository, "file://") { + dir := path.Join(a.Repository[7:], a.GavPath()) + return gavFilesFromDisk(utils.FileSystem(fs...), dir) + } + return a.gavOnlineFiles(ctx) +} + +func gavFilesFromDisk(fs vfs.FileSystem, dir string) (map[string]crypto.Hash, error) { + files, err := tarutils.ListSortedFilesInDir(fs, dir, true) + if err != nil { + return nil, err + } + return filesAndHashes(files), nil +} + +// gavOnlineFiles returns the files of the Maven (mvn) artifact in the repository and their available digests. +func (a *AccessSpec) gavOnlineFiles(ctx accspeccpi.Context) (map[string]crypto.Hash, error) { + log := log.WithValues("BaseUrl", a.BaseUrl()) + log.Debug("gavOnlineFiles") + + reader, err := getReader(ctx, a.BaseUrl(), nil) + if err != nil { + return nil, err + } + defer reader.Close() + + // Which files are listed in the repository? + log.Debug("parse-html") + htmlDoc, err := html.Parse(reader) + if err != nil { + return nil, err + } + var fileList []string + var process func(*html.Node) + prefix := a.FileNamePrefix() + process = func(node *html.Node) { + // check if the node is an element node and the tag is "" + if node.Type == html.ElementNode && node.Data == "a" { + for _, attribute := range node.Attr { + if attribute.Key == "href" { + // check if the href starts with artifactId-version + if strings.HasPrefix(attribute.Val, prefix) { + fileList = append(fileList, attribute.Val) + } + } + } + } + for nextChild := node.FirstChild; nextChild != nil; nextChild = nextChild.NextSibling { + process(nextChild) // recursive call! + } + } + process(htmlDoc) + + return filesAndHashes(fileList), nil +} + +func filesAndHashes(fileList []string) map[string]crypto.Hash { + // Sort the list of files, to ensure always the same results for e.g. identical tar.gz files. + sort.Strings(fileList) + + // Which hash files are available? + result := make(map[string]crypto.Hash, len(fileList)/2) + for _, file := range fileList { + if IsResource(file) { + result[file] = bestAvailableHash(fileList, file) + log.Debug("found", "file", file) + } + } + return result +} + +// bestAvailableHash returns the best available hash for the given file. +// It first checks for SHA-512, then SHA-256, SHA-1, and finally MD5. If nothing is found, it returns 0. +func bestAvailableHash(list []string, filename string) crypto.Hash { + hashes := [5]crypto.Hash{crypto.SHA512, crypto.SHA256, crypto.SHA1, crypto.MD5} + for _, hash := range hashes { + if slices.Contains(list, filename+hashUrlExt(hash)) { + return hash + } + } + return 0 +} + +//////////////////////////////////////////////////////////////////////////////// + +// getStringData reads all data from the given URL and returns it as a string. +func getStringData(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (string, error) { + r, err := getReader(ctx, url, fs) + if err != nil { + return "", err + } + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + return "", err + } + return string(b), nil +} + +// hashUrlExt returns the 'maven' hash extension for the given hash. +// Maven usually uses sha1, sha256, sha512, md5 instead of SHA-1, SHA-256, SHA-512, MD5. +func hashUrlExt(h crypto.Hash) string { + return "." + strings.ReplaceAll(strings.ToLower(h.String()), "-", "") +} + +func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { + factory := func() (blobaccess.BlobAccess, error) { + meta, err := a.GetPackageMeta(c.GetContext()) + if err != nil { + return nil, err + } + + reader := func() (io.ReadCloser, error) { + return getReader(c.GetContext(), meta.Bin, vfsattr.Get(c.GetContext())) + } + if meta.Hash != "" { + getreader := reader + reader = func() (io.ReadCloser, error) { + readCloser, err := getreader() + if err != nil { + return nil, err + } + return iotools.VerifyingReaderWithHash(readCloser, meta.HashType, meta.Hash), nil + } + } + acc := blobaccess.DataAccessForReaderFunction(reader, meta.Bin) + return accessobj.CachedBlobAccessForWriter(c.GetContext(), a.MimeType(), accessio.NewDataAccessWriter(acc)), nil + } + // FIXME add Digest! + return accspeccpi.NewDefaultMethodImpl(c, a, "", a.MimeType(), factory), nil +} + +func getReader(ctx accspeccpi.Context, url string, fs vfs.FileSystem) (io.ReadCloser, error) { + if strings.HasPrefix(url, "file://") { + path := url[7:] + return fs.OpenFile(path, vfs.O_RDONLY, 0o600) + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + err = identity.BasicAuth(req, ctx, url, "") + if err != nil { + return nil, err + } + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + buf := &bytes.Buffer{} + _, err = io.Copy(buf, io.LimitReader(resp.Body, 2000)) + if err == nil { + log.Error("http", "code", resp.Status, "url", url, "body", buf.String()) + } + return nil, errors.Newf("http %s error - %s", resp.Status, url) + } + return resp.Body, nil +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/method_test.go b/pkg/contexts/ocm/accessmethods/mvn/method_test.go new file mode 100644 index 0000000000..7b4ca85a80 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/method_test.go @@ -0,0 +1,87 @@ +package mvn_test + +import ( + "crypto" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + . "github.com/open-component-model/ocm/pkg/env" + . "github.com/open-component-model/ocm/pkg/env/builder" + "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/mime" + . "github.com/open-component-model/ocm/pkg/testutils" +) + +const ( + mvnPATH = "/testdata/.m2/repository" + FAILPATH = "/testdata/fail" +) + +var _ = Describe("local accessmethods.mvn.AccessSpec tests", func() { + var env *Builder + var cv ocm.ComponentVersionAccess + + BeforeEach(func() { + env = NewBuilder(TestData()) + cv = &cpi.DummyComponentVersionAccess{env.OCMContext()} + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("accesses local artifact", func() { + acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0") + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_TGZ)) + r := Must(m.Reader()) + defer r.Close() + dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) + for { + var buf [8096]byte + _, err := dr.Read(buf[:]) + if err != nil { + break + } + } + Expect(dr.Size()).To(Equal(int64(1109))) + Expect(dr.Digest().String()).To(Equal("SHA-1:4ee125ffe4f7690588833f1217a13cc741e4df5f")) + }) + + It("accesses local artifact with extension", func() { + acc := mvn.New("file://"+mvnPATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0", mvn.WithExtension("pom")) + m := Must(acc.AccessMethod(cv)) + defer m.Close() + Expect(m.MimeType()).To(Equal(mime.MIME_XML)) + r := Must(m.Reader()) + defer r.Close() + dr := iotools.NewDigestReaderWithHash(crypto.SHA1, r) + for { + var buf [8096]byte + _, err := dr.Read(buf[:]) + if err != nil { + break + } + } + Expect(dr.Size()).To(Equal(int64(7153))) + Expect(dr.Digest().String()).To(Equal("SHA-1:34ccdeb9c008f8aaef90873fc636b09d3ae5c709")) + }) + + It("Describe", func() { + acc := mvn.New("file://"+FAILPATH, "test", "repository", "42", mvn.WithExtension("pom")) + Expect(acc.Describe(nil)).To(Equal("Maven (mvn) package 'test:repository:42::pom' in repository 'file:///testdata/fail' path 'test/repository/42/repository-42.pom'")) + }) + + It("detects digests mismatch", func() { + acc := mvn.New("file://"+FAILPATH, "test", "repository", "42", mvn.WithExtension("pom")) + m := Must(acc.AccessMethod(cv)) + defer m.Close() + _, err := m.Reader() + Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found b3242b8c31f8ce14f729b8fd132ac77bc4bc5bf7"))) + }) +}) diff --git a/pkg/contexts/ocm/accessmethods/mvn/suite_test.go b/pkg/contexts/ocm/accessmethods/mvn/suite_test.go new file mode 100644 index 0000000000..c62cadc7b0 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/suite_test.go @@ -0,0 +1,13 @@ +package mvn_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Maven (mvn) Test Suite") +} diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom new file mode 100644 index 0000000000..b3baaee32f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom @@ -0,0 +1,210 @@ + + + 4.0.0 + com.sap.cloud.sdk + sdk-modules-bom + 5.7.0 + pom + SAP Cloud SDK - Modules BOM + Bill of Materials (BOM) of the SAP Cloud SDK modules. + https://sap.github.io/cloud-sdk/docs/java/getting-started + + SAP SE + https://www.sap.com + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + SAP + cloudsdk@sap.com + SAP SE + https://www.sap.com + + + + + + + + UTF-8 + Public + Stable + + 5.7.0 + + + + + com.sap.cloud.sdk + sdk-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + dwc-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-core + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + caching + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient4 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-apache-httpclient5 + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + cloudplatform-connectivity-scp-cf + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-destination-service + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-oauth + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-dwc + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + connectivity-ztis + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience-api + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + resilience4j + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + security + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + servlet-jakarta + ${sdk.version} + + + com.sap.cloud.sdk.cloudplatform + tenant + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-core + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + s4hana-connectivity + ${sdk.version} + + + com.sap.cloud.sdk.s4hana + rfc + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + datamodel-metadata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator-utility + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-client + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + odata-v4-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-core + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + openapi-generator + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + fluent-result + ${sdk.version} + + + com.sap.cloud.sdk.datamodel + + soap + ${sdk.version} + + + + diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 new file mode 100644 index 0000000000..35f63a2e1f --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/.m2/repository/com/sap/cloud/sdk/sdk-modules-bom/5.7.0/sdk-modules-bom-5.7.0.pom.sha1 @@ -0,0 +1 @@ +34ccdeb9c008f8aaef90873fc636b09d3ae5c709 \ No newline at end of file diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom new file mode 100644 index 0000000000..218894d775 --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom @@ -0,0 +1,14 @@ + + + 4.0.0 + fail + repository + 42 + pom + ocm + test + + SAP SE + https://www.sap.com + + diff --git a/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom.sha1 b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom.sha1 new file mode 100644 index 0000000000..3d6c52fe9e --- /dev/null +++ b/pkg/contexts/ocm/accessmethods/mvn/testdata/fail/test/repository/42/repository-42.pom.sha1 @@ -0,0 +1 @@ +44a77645201d1a8fc5213ace787c220eabbd0967 \ No newline at end of file diff --git a/pkg/contexts/ocm/accessmethods/options/standard.go b/pkg/contexts/ocm/accessmethods/options/standard.go index c13b298134..7e8680f12b 100644 --- a/pkg/contexts/ocm/accessmethods/options/standard.go +++ b/pkg/contexts/ocm/accessmethods/options/standard.go @@ -18,6 +18,9 @@ var ReferenceOption = RegisterOption(NewStringOptionType("reference", "reference // PackageOption. var PackageOption = RegisterOption(NewStringOptionType("accessPackage", "package or object name")) +// GroupOption. +var GroupOption = RegisterOption(NewStringOptionType("accessGroup", "GroupID or namespace")) + // RepositoryOption. var RepositoryOption = RegisterOption(NewStringOptionType("accessRepository", "repository URL")) @@ -53,7 +56,11 @@ var HTTPBodyOption = RegisterOption(NewStringOptionType("body", "body of a http var HTTPRedirectOption = RegisterOption(NewBoolOptionType("noredirect", "http redirect behavior")) -//////////////////////////////////////////////////////////////////////////////// - // CommentOption. var CommentOption = RegisterOption(NewStringOptionType("comment", "comment field value")) + +// ClassifierOption the optional classifier of a maven resource. +var ClassifierOption = RegisterOption(NewStringOptionType("accessClassifier", "mvn classifier")) + +// ExtensionOption the optional extension of a maven resource. +var ExtensionOption = RegisterOption(NewStringOptionType("accessExtension", "mvn extension name")) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go new file mode 100644 index 0000000000..1e27561d32 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler.go @@ -0,0 +1,180 @@ +package mvn + +import ( + "context" + "crypto" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/mvn/identity" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/accspeccpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/iotools" + "github.com/open-component-model/ocm/pkg/logging" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +const BlobHandlerName = "ocm/" + resourcetypes.MVN_ARTIFACT + +type artifactHandler struct { + spec *Config +} + +func NewArtifactHandler(repospec *Config) cpi.BlobHandler { + return &artifactHandler{repospec} +} + +var log = logging.DynamicLogger(identity.REALM) + +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hint string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { + // check conditions + if b.spec == nil { + return nil, nil + } + mimeType := blob.MimeType() + if resourcetypes.MVN_ARTIFACT != resourceType { + log.Debug("not a MVN artifact", "resourceType", resourceType) + return nil, nil + } + if mime.MIME_TGZ != mimeType { + log.Debug("not a tarball, can't be a complete mvn GAV", "mimeType", mimeType) + return nil, nil + } + if b.spec.Url == "" { + return nil, errors.New("MVN repository url not provided") + } + + // setup logger + log := log.WithValues("repository", b.spec.Url) + // identify artifact + artifact, err := mvn.Parse(hint) + if err != nil { + return nil, err + } + log = log.WithValues("groupId", artifact.GroupId, "artifactId", artifact.ArtifactId, "version", artifact.Version) + log.Debug("identified") + + blobReader, err := blob.Reader() + if err != nil { + return nil, err + } + defer blobReader.Close() + tempFs, err := tarutils.ExtractTgzToTempFs(blobReader) + if err != nil { + return nil, err + } + defer vfs.Cleanup(tempFs) + files, err := tarutils.ListSortedFilesInDir(tempFs, "", false) + if err != nil { + return nil, err + } + for _, file := range files { + e := func() (err error) { + log.Debug("uploading", "file", file) + err = artifact.SetClassifierExtensionBy(file) + if err != nil { + return + } + readHash, err := tempFs.Open(file) + if err != nil { + return + } + defer readHash.Close() + // MD5 + SHA1 are still the most used ones in the mvn context + hr := iotools.NewHashReader(readHash, crypto.SHA256, crypto.SHA1, crypto.MD5) + _, err = hr.CalcHashes() + if err != nil { + return + } + reader, err := tempFs.Open(file) + if err != nil { + return + } + defer reader.Close() + err = deploy(artifact, b.spec.Url, reader, ctx.GetContext(), hr) + return + }() + if e != nil { + return nil, e + } + } + + log.Debug("done", "artifact", artifact) + return mvn.New(b.spec.Url, artifact.GroupId, artifact.ArtifactId, artifact.Version, mvn.WithClassifier(artifact.Classifier), mvn.WithExtension(artifact.Extension)), nil +} + +// deploy an artifact to the specified destination. See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact +func deploy(artifact *mvn.Coordinates, url string, reader io.ReadCloser, ctx accspeccpi.Context, hashes *iotools.HashReader) (err error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, artifact.Url(url), reader) + if err != nil { + return + } + err = identity.BasicAuth(req, ctx, url, artifact.GroupPath()) + if err != nil { + return + } + // give the remote server a chance to decide based upon the checksum policy + for k, v := range hashes.HttpHeader() { + req.Header.Set(k, v) + } + + // Execute the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return + } + defer resp.Body.Close() + + // Check the response + if resp.StatusCode != http.StatusCreated { + all, e := io.ReadAll(resp.Body) + if e != nil { + return e + } + return fmt.Errorf("http (%d) - failed to upload artifact: %s", resp.StatusCode, string(all)) + } + log.Debug("uploaded", "artifact", artifact, "extension", artifact.Extension, "classifier", artifact.Classifier) + + // Validate the response - especially the hash values with the ones we've tried to send + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var artifactBody Body + err = json.Unmarshal(respBody, &artifactBody) + if err != nil { + return + } + + // let's check only SHA256 for now + digest := hashes.GetString(crypto.SHA256) + remoteDigest := artifactBody.Checksums[strings.ReplaceAll(strings.ToLower(crypto.SHA256.String()), "-", "")] + if remoteDigest == "" { + log.Warn("no checksum found for algorithm, we can't guarantee that the artifact has been uploaded correctly", "algorithm", crypto.SHA256) + } else if remoteDigest != digest { + return errors.New("failed to upload artifact: checksums do not match") + } + log.Debug("digests are ok", "remoteDigest", remoteDigest, "digest", digest) + return +} + +// Body is the response struct of a deployment from the MVN repository (JFrog Artifactory). +type Body struct { + Repo string `json:"repo"` + Path string `json:"path"` + DownloadUri string `json:"downloadUri"` + Uri string `json:"uri"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + Checksums map[string]string `json:"checksums"` +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go new file mode 100644 index 0000000000..a632ace444 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/blobhandler_test.go @@ -0,0 +1,43 @@ +package mvn_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" +) + +var _ = Describe("blobhandler generic mvn tests", func() { + + It("Unmarshal deploy response Body", func() { + resp := `{ "repo" : "ocm-mvn-test", + "path" : "/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "created" : "2024-04-11T15:09:28.920Z", + "createdBy" : "john.doe", + "downloadUri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar", + "mimeType" : "application/java-archive", + "size" : "1792", + "checksums" : { + "sha1" : "99d9acac1ff93ac3d52229edec910091af1bc40a", + "md5" : "6cb7520b65d820b3b35773a8daa8368e", + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "originalChecksums" : { + "sha256" : "b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30" }, + "uri" : "https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar" }` + var body mvn.Body + err := json.Unmarshal([]byte(resp), &body) + Expect(err).To(BeNil()) + Expect(body.Repo).To(Equal("ocm-mvn-test")) + Expect(body.DownloadUri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.Uri).To(Equal("https://ocm.sofware/repository/ocm-mvn-test/open-component-model/hello-ocm/0.0.2/hello-ocm-0.0.2.jar")) + Expect(body.MimeType).To(Equal("application/java-archive")) + Expect(body.Size).To(Equal("1792")) + Expect(body.Checksums["md5"]).To(Equal("6cb7520b65d820b3b35773a8daa8368e")) + Expect(body.Checksums["sha1"]).To(Equal("99d9acac1ff93ac3d52229edec910091af1bc40a")) + Expect(body.Checksums["sha256"]).To(Equal("b19dcd275f72a0cbdead1e5abacb0ef25a0cb55ff36252ef44b1178eeedf9c30")) + Expect(body.Checksums["sha512"]).To(Equal("")) + }) + +}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go new file mode 100644 index 0000000000..7fc5ffb8f9 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration.go @@ -0,0 +1,74 @@ +package mvn + +import ( + "encoding/json" + "fmt" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/registrations" +) + +type Config struct { + Url string `json:"url"` +} + +type rawConfig Config + +func (c *Config) UnmarshalJSON(data []byte) error { + err := json.Unmarshal(data, &c.Url) + if err == nil { + return nil + } + var raw rawConfig + err = json.Unmarshal(data, &raw) + if err != nil { + return err + } + *c = Config(raw) + + return nil +} + +func init() { + cpi.RegisterBlobHandlerRegistrationHandler(BlobHandlerName, &RegistrationHandler{}) +} + +type RegistrationHandler struct{} + +var _ cpi.BlobHandlerRegistrationHandler = (*RegistrationHandler)(nil) + +func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, config cpi.BlobHandlerConfig, olist ...cpi.BlobHandlerOption) (bool, error) { + if handler != "" { + return true, fmt.Errorf("invalid %s handler %q", resourcetypes.MVN_ARTIFACT, handler) + } + if config == nil { + return true, fmt.Errorf("mvn target specification required") + } + cfg, err := registrations.DecodeConfig[Config](config) + if err != nil { + return true, errors.Wrapf(err, "blob handler configuration") + } + + ctx.BlobHandlers().Register(NewArtifactHandler(cfg), + cpi.ForArtifactType(resourcetypes.MVN_ARTIFACT), + cpi.ForMimeType(mime.MIME_TGZ), + cpi.NewBlobHandlerOptions(olist...), + ) + + return true, nil +} + +func (r *RegistrationHandler) GetHandlers(_ cpi.Context) registrations.HandlerInfos { + return registrations.NewLeafHandlerInfo("uploading mvn artifacts", ` +The `+BlobHandlerName+` uploader is able to upload mvn artifacts (whole GAV only!) +as artifact archive according to the mvn artifact spec. +If registered the default mime type is: `+mime.MIME_TGZ+` + +It accepts a plain string for the URL or a config with the following field: +'url': the URL of the mvn repository. +`, + ) +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go new file mode 100644 index 0000000000..c64b3e13bb --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/registration_test.go @@ -0,0 +1,23 @@ +package mvn_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" + "github.com/open-component-model/ocm/pkg/registrations" +) + +var _ = Describe("Config deserialization Test Environment", func() { + + It("deserializes string", func() { + cfg := Must(registrations.DecodeConfig[mvn.Config]("test")) + Expect(cfg).To(Equal(&mvn.Config{Url: "test"})) + }) + + It("deserializes struct", func() { + cfg := Must(registrations.DecodeConfig[mvn.Config](`{"Url":"test"}`)) + Expect(cfg).To(Equal(&mvn.Config{Url: "test"})) + }) +}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go new file mode 100644 index 0000000000..c62cadc7b0 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/mvn/suite_test.go @@ -0,0 +1,13 @@ +package mvn_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Maven (mvn) Test Suite") +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/init.go b/pkg/contexts/ocm/blobhandler/handlers/init.go index d19447156c..b318fe2c9e 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/init.go +++ b/pkg/contexts/ocm/blobhandler/handlers/init.go @@ -1,6 +1,7 @@ package handlers import ( + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/mvn" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo" diff --git a/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go b/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go new file mode 100644 index 0000000000..6c90b396a9 --- /dev/null +++ b/pkg/contexts/ocm/elements/artifactaccess/mvnaccess/resource.go @@ -0,0 +1,30 @@ +package mvnaccess + +import ( + "github.com/open-component-model/ocm/pkg/contexts/ocm" + access "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/mvn" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/elements/artifactaccess/genericaccess" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" +) + +const TYPE = resourcetypes.MVN_ARTIFACT + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, repository, groupId, artifactId, version string) cpi.ArtifactAccess[M] { + if meta.GetType() == "" { + meta.SetType(TYPE) + } + + spec := access.New(repository, groupId, artifactId, version) + // is global access, must work, otherwise there is an error in the lib. + return genericaccess.MustAccess(ctx, meta, spec) +} + +func ResourceAccess(ctx ocm.Context, meta *cpi.ResourceMeta, repository, groupId, artifactId, version string) cpi.ResourceAccess { + return Access(ctx, meta, repository, groupId, artifactId, version) +} + +func SourceAccess(ctx ocm.Context, meta *cpi.SourceMeta, repository, groupId, artifactId, version string) cpi.SourceAccess { + return Access(ctx, meta, repository, groupId, artifactId, version) +} diff --git a/pkg/contexts/ocm/internal/blobhandler.go b/pkg/contexts/ocm/internal/blobhandler.go index 4023e79a82..e12aed3eb0 100644 --- a/pkg/contexts/ocm/internal/blobhandler.go +++ b/pkg/contexts/ocm/internal/blobhandler.go @@ -33,9 +33,9 @@ type StorageContext interface { GetImplementationRepositoryType() ImplementationRepositoryType } -// BlobHandler s the interface for a dedicated handling of storing blobs +// BlobHandler is the interface for a dedicated handling of storing blobs // for the LocalBlob access method in a dedicated kind of repository. -// with the possibility of access by an external distribution spec. +// With the possibility of access by an external distribution spec // (besides of the blob storage as part of a component version). // The technical repository to use should be derivable from the chosen // component directory or passed together with the storage context. diff --git a/pkg/contexts/ocm/resourcetypes/const.go b/pkg/contexts/ocm/resourcetypes/const.go index 4c1be2b37b..6c0dc78650 100644 --- a/pkg/contexts/ocm/resourcetypes/const.go +++ b/pkg/contexts/ocm/resourcetypes/const.go @@ -14,16 +14,19 @@ const ( // HELM_CHART describes a helm chart, either stored as OCI artifact or as tar // blob (tar media type). HELM_CHART = "helmChart" - // NPM_PACKAGE describes an NPM package. + // NPM_PACKAGE describes a Node.js (npm) package. NPM_PACKAGE = "npmPackage" + // MVN_ARTIFACT describes a Maven artifact (jar). + MVN_ARTIFACT = "mvnArtifact" // BLUEPRINT describes a Gardener Landscaper blueprint which is an artifact used in its installations describing // how to deploy a software component. BLUEPRINT = "landscaper.gardener.cloud/blueprint" BLUEPRINT_LEGACY = "blueprint" // BLOB describes any anonymous untyped blob data. BLOB = "blob" + // DIRECTORY_TREE describes a directory structure. + DIRECTORY_TREE = "directoryTree" // FILESYSTEM describes a directory structure stored as archive (tar, tgz). - DIRECTORY_TREE = "directoryTree" FILESYSTEM = DIRECTORY_TREE FILESYSTEM_LEGACY = "filesystem" // EXECUTABLE describes an OS executable. diff --git a/pkg/iotools/hashReaderWriter.go b/pkg/iotools/hashReaderWriter.go new file mode 100644 index 0000000000..675cdb68c0 --- /dev/null +++ b/pkg/iotools/hashReaderWriter.go @@ -0,0 +1,152 @@ +package iotools + +import ( + "crypto" + "fmt" + "hash" + "io" + "strings" + + "github.com/open-component-model/ocm/pkg/errors" +) + +type HashReader struct { + reader io.Reader + hashMap map[crypto.Hash]hash.Hash +} + +func NewHashReader(delegate io.Reader, algorithms ...crypto.Hash) *HashReader { + newInstance := HashReader{ + reader: delegate, + hashMap: initMap(algorithms), + } + return &newInstance +} + +func (h *HashReader) Read(buf []byte) (int, error) { + c, err := h.reader.Read(buf) + return write(h, c, buf, err) +} + +func (h *HashReader) GetString(algorithm crypto.Hash) string { + return getString(h, algorithm) +} + +func (h *HashReader) GetBytes(algorithm crypto.Hash) []byte { + return getBytes(h, algorithm) +} + +func (h *HashReader) HttpHeader() map[string]string { + return httpHeader(h) +} + +func (h *HashReader) hashes() map[crypto.Hash]hash.Hash { + return h.hashMap +} + +func (h *HashReader) ReadAll() ([]byte, error) { + return io.ReadAll(h.reader) +} + +// CalcHashes returns the total number of bytes read and an error if any besides EOF. +func (h *HashReader) CalcHashes() (int64, error) { + b := make([]byte, 0, 512) + cnt := int64(0) + for { + n, err := h.Read(b[0:cap(b)]) // read a chunk, always from the beginning + b = b[:n] // reset slice to the actual read bytes + cnt += int64(n) + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + return cnt, err + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +type HashWriter struct { + writer io.Writer + hashMap map[crypto.Hash]hash.Hash +} + +func NewHashWriter(w io.Writer, algorithms ...crypto.Hash) *HashWriter { + newInstance := HashWriter{ + writer: w, + hashMap: initMap(algorithms), + } + return &newInstance +} + +func (h *HashWriter) Write(buf []byte) (int, error) { + c, err := h.writer.Write(buf) + return write(h, c, buf, err) +} + +func (h *HashWriter) GetString(algorithm crypto.Hash) string { + return getString(h, algorithm) +} + +func (h *HashWriter) GetBytes(algorithm crypto.Hash) []byte { + return getBytes(h, algorithm) +} + +func (h *HashWriter) HttpHeader() map[string]string { + return httpHeader(h) +} + +func (h *HashWriter) hashes() map[crypto.Hash]hash.Hash { + return h.hashMap +} + +//////////////////////////////////////////////////////////////////////////////// + +type hashes interface { + hashes() map[crypto.Hash]hash.Hash +} + +func getString(h hashes, algorithm crypto.Hash) string { + return fmt.Sprintf("%x", getBytes(h, algorithm)) +} + +func getBytes(h hashes, algorithm crypto.Hash) []byte { + hash := h.hashes()[algorithm] + if hash != nil { + return hash.Sum(nil) + } + return nil +} + +func httpHeader(h hashes) map[string]string { + headers := make(map[string]string, len(h.hashes())) + for algorithm := range h.hashes() { + headers[headerName(algorithm)] = getString(h, algorithm) + } + return headers +} + +func initMap(algorithms []crypto.Hash) map[crypto.Hash]hash.Hash { + hashMap := make(map[crypto.Hash]hash.Hash, len(algorithms)) + for _, algorithm := range algorithms { + hashMap[algorithm] = algorithm.New() + } + return hashMap +} + +func write(h hashes, c int, buf []byte, err error) (int, error) { + if err == nil && c > 0 { + for _, hash := range h.hashes() { + hash.Write(buf[:c]) + } + } + return c, err +} + +//////////////////////////////////////////////////////////////////////////////// + +func headerName(hash crypto.Hash) string { + a := strings.ReplaceAll(hash.String(), "-", "") + return "X-Checksum-" + a[:1] + strings.ToLower(a[1:]) +} diff --git a/pkg/iotools/hashReaderWriter_test.go b/pkg/iotools/hashReaderWriter_test.go new file mode 100644 index 0000000000..bda2a6fc3d --- /dev/null +++ b/pkg/iotools/hashReaderWriter_test.go @@ -0,0 +1,79 @@ +package iotools_test + +import ( + "bytes" + "crypto" + "io" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/iotools" +) + +var _ = Describe("Hash Reader Writer tests", func() { + It("Ensure interface implementation", func() { + var _ io.Reader = &iotools.HashReader{} + var _ io.Reader = (*iotools.HashReader)(nil) + var _ io.Reader = new(iotools.HashReader) + + var _ io.Writer = &iotools.HashWriter{} + var _ io.Writer = (*iotools.HashWriter)(nil) + var _ io.Writer = new(iotools.HashWriter) + }) + + It("test HashWriter", func() { + s := "Hello Hash!" + var b bytes.Buffer + hr := iotools.NewHashWriter(io.Writer(&b)) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + b.Reset() + + w := io.Writer(&b) + hr = iotools.NewHashWriter(w, crypto.SHA1) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + b.Reset() + + hr = iotools.NewHashWriter(io.Writer(&b), crypto.SHA1, crypto.MD5) + hr.Write([]byte(s)) + Expect(b.String()).To(Equal(s)) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + }) + + It("test HashReader", func() { + s := "Hello Hash!" + hr := iotools.NewHashReader(strings.NewReader(s)) + buf := make([]byte, len(s)) + hr.Read(buf) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(string(buf)).To(Equal(s)) + + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1) + hr.Read(buf) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1) + cnt, err := hr.CalcHashes() + Expect(err).To(BeNil()) + Expect(cnt).To(Equal(int64(len(s)))) + Expect(hr.GetBytes(0)).To(BeNil()) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + + hr = iotools.NewHashReader(strings.NewReader(s), crypto.SHA1, crypto.MD5) + hr.Read(buf) + Expect(hr.GetBytes(crypto.SHA256)).To(BeNil()) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.MD5)).To(Equal("c10e8df2e378a1584359b0e546cf0149")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + Expect(hr.GetString(crypto.SHA1)).To(Equal("5c075ed604db0adc524edd3516e8f0258ca6e58d")) + }) +}) diff --git a/pkg/iotools/suite_test.go b/pkg/iotools/suite_test.go new file mode 100644 index 0000000000..f2c3e11709 --- /dev/null +++ b/pkg/iotools/suite_test.go @@ -0,0 +1,13 @@ +package iotools_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "iotools Test Suite") +} diff --git a/pkg/mime/types.go b/pkg/mime/types.go index 020968835e..715f6845b2 100644 --- a/pkg/mime/types.go +++ b/pkg/mime/types.go @@ -1,5 +1,11 @@ package mime +import ( + "mime" + + "github.com/open-component-model/ocm/pkg/logging" +) + const ( MIME_TEXT = "text/plain" MIME_OCTET = "application/octet-stream" @@ -7,6 +13,7 @@ const ( MIME_JSON = "application/x-json" MIME_JSON_ALT = "text/json" // no utf8 MIME_JSON_OFFICIAL = "application/json" + MIME_XML = "application/xml" MIME_YAML = "application/x-yaml" MIME_YAML_ALT = "text/yaml" // no utf8 MIME_YAML_OFFICIAL = "application/yaml" @@ -15,4 +22,29 @@ const ( MIME_TAR = "application/x-tar" MIME_TGZ = "application/x-tgz" MIME_TGZ_ALT = MIME_TAR + "+gzip" + + MIME_JAR = "application/x-jar" ) + +func init() { + ocmTypes := map[string]string{ + // added entries + ".txt": MIME_TEXT, + ".yaml": MIME_YAML_OFFICIAL, + ".gzip": MIME_GZIP, + ".tar": MIME_TAR, + ".tgz": MIME_TGZ, + ".tar.gz": MIME_TGZ, + ".pom": MIME_XML, + ".zip": MIME_GZIP, + ".jar": MIME_JAR, + ".module": MIME_JSON, // gradle module metadata + } + + for k, v := range ocmTypes { + err := mime.AddExtensionType(k, v) + if err != nil { + logging.DynamicLogger(logging.DefineSubRealm("mimeutils")).Error("failed to add extension type", "extension", k, "type", v, "error", err) + } + } +} diff --git a/pkg/mimeutils/type.go b/pkg/mimeutils/type.go deleted file mode 100644 index e7c4b97463..0000000000 --- a/pkg/mimeutils/type.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Modifications Copyright 2024 SAP SE or an SAP affiliate company. - -// Package mimeutils implements parts of the MIME spec. -// Copied from mime to achieve a platform independent reproducible list of -// known mime types. -package mimeutils - -import ( - "fmt" - "mime" - "sort" - "strings" - "sync" - - ocmmime "github.com/open-component-model/ocm/pkg/mime" -) - -var ( - mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress" - mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress" - - // extensions maps from MIME type to list of lowercase file - // extensions: "image/jpeg" => [".jpg", ".jpeg"] - extensionsMu sync.Mutex // Guards stores (but not loads) on extensions. - extensions sync.Map // map[string][]string; slice values are append-only. -) - -func clearSyncMap(m *sync.Map) { - m.Range(func(k, _ any) bool { - m.Delete(k) - return true - }) -} - -// setMimeTypes is used by initMime's non-test path, and by tests. -func setMimeTypes(lowerExt, mixExt map[string]string) { - clearSyncMap(&mimeTypes) - clearSyncMap(&mimeTypesLower) - clearSyncMap(&extensions) - - for k, v := range lowerExt { - mimeTypesLower.Store(k, v) - } - for k, v := range mixExt { - mimeTypes.Store(k, v) - } - - extensionsMu.Lock() - defer extensionsMu.Unlock() - for k, v := range lowerExt { - justType, _, err := mime.ParseMediaType(v) - if err != nil { - panic(err) - } - var exts []string - if ei, ok := extensions.Load(justType); ok { - exts = ei.([]string) - } - extensions.Store(justType, append(exts, k)) - } -} - -var builtinTypesLower = map[string]string{ - // default list - ".avif": "image/avif", - ".css": "text/css; charset=utf-8", - ".gif": "image/gif", - ".htm": "text/html; charset=utf-8", - ".html": "text/html; charset=utf-8", - ".jpeg": "image/jpeg", - ".jpg": "image/jpeg", - ".js": "text/javascript; charset=utf-8", - ".json": "application/json", - ".mjs": "text/javascript; charset=utf-8", - ".pdf": "application/pdf", - ".png": "image/png", - ".svg": "image/svg+xml", - ".wasm": "application/wasm", - ".webp": "image/webp", - ".xml": "text/xml; charset=utf-8", - // added entries - ".txt": ocmmime.MIME_TEXT, - ".yaml": ocmmime.MIME_YAML_OFFICIAL, - ".gzip": ocmmime.MIME_GZIP, - ".tar": ocmmime.MIME_TAR, - ".tgz": ocmmime.MIME_TGZ, -} - -var once sync.Once // guards initMime - -var testInitMime func() - -func initMime() { - if fn := testInitMime; fn != nil { - fn() - } else { - setMimeTypes(builtinTypesLower, builtinTypesLower) - } -} - -// TypeByExtension returns the MIME type associated with the file extension ext. -// The extension ext should begin with a leading dot, as in ".html". -// When ext has no associated type, TypeByExtension returns "". -// -// Extensions are looked up first case-sensitively, then case-insensitively. -// -// The built-in table is small but on unix it is augmented by the local -// system's MIME-info database or mime.types file(s) if available under one or -// more of these names: -// -// /usr/local/share/mime/globs2 -// /usr/share/mime/globs2 -// /etc/mime.types -// /etc/apache2/mime.types -// /etc/apache/mime.types -// -// On Windows, MIME types are extracted from the registry. -// -// Text types have the charset parameter set to "utf-8" by default. -func TypeByExtension(ext string) string { - once.Do(initMime) - - // Case-sensitive lookup. - if v, ok := mimeTypes.Load(ext); ok { - return v.(string) - } - - // Case-insensitive lookup. - // Optimistically assume a short ASCII extension and be - // allocation-free in that case. - var buf [10]byte - lower := buf[:0] - const utf8RuneSelf = 0x80 // from utf8 package, but not importing it. - for i := 0; i < len(ext); i++ { - c := ext[i] - if c >= utf8RuneSelf { - // Slow path. - si, _ := mimeTypesLower.Load(strings.ToLower(ext)) - s, _ := si.(string) // handle nil pointer!!!! - return s - } - if 'A' <= c && c <= 'Z' { - lower = append(lower, c+('a'-'A')) - } else { - lower = append(lower, c) - } - } - si, _ := mimeTypesLower.Load(string(lower)) - s, _ := si.(string) // handle nil pointer!!!! - return s -} - -// ExtensionsByType returns the extensions known to be associated with the MIME -// type typ. The returned extensions will each begin with a leading dot, as in -// ".html". When typ has no associated extensions, ExtensionsByType returns an -// nil slice. -func ExtensionsByType(typ string) ([]string, error) { - justType, _, err := mime.ParseMediaType(typ) - if err != nil { - return nil, err - } - - once.Do(initMime) - s, ok := extensions.Load(justType) - if !ok { - return nil, nil - } - ret := append([]string(nil), s.([]string)...) - sort.Strings(ret) - return ret, nil -} - -// AddExtensionType sets the MIME type associated with -// the extension ext to typ. The extension should begin with -// a leading dot, as in ".html". -func AddExtensionType(ext, typ string) error { - if !strings.HasPrefix(ext, ".") { - return fmt.Errorf("mime: extension %q missing leading dot", ext) - } - once.Do(initMime) - return setExtensionType(ext, typ) -} - -func setExtensionType(extension, mimeType string) error { - justType, param, err := mime.ParseMediaType(mimeType) - if err != nil { - return err - } - if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" { - param["charset"] = "utf-8" - mimeType = mime.FormatMediaType(mimeType, param) - } - extLower := strings.ToLower(extension) - - mimeTypes.Store(extension, mimeType) - mimeTypesLower.Store(extLower, mimeType) - - extensionsMu.Lock() - defer extensionsMu.Unlock() - var exts []string - if ei, ok := extensions.Load(justType); ok { - exts = ei.([]string) - } - for _, v := range exts { - if v == extLower { - return nil - } - } - extensions.Store(justType, append(exts, extLower)) - return nil -} diff --git a/pkg/utils/tarutils/extract.go b/pkg/utils/tarutils/extract.go index b310685d8c..4a6b300a22 100644 --- a/pkg/utils/tarutils/extract.go +++ b/pkg/utils/tarutils/extract.go @@ -29,7 +29,7 @@ func ExtractArchiveToFs(fs vfs.FileSystem, path string, fss ...vfs.FileSystem) e return ExtractTarToFs(fs, r) } -// ExtractArchiveToFsWithInfo wunpacks an archive to a filesystem. +// ExtractArchiveToFsWithInfo unpacks an archive to a filesystem. func ExtractArchiveToFsWithInfo(fs vfs.FileSystem, path string, fss ...vfs.FileSystem) (int64, int64, error) { sfs := utils.OptionalDefaulted(osfs.New(), fss...) @@ -51,6 +51,31 @@ func ExtractTarToFs(fs vfs.FileSystem, in io.Reader) error { return err } +// UnzipTarToFs tries to decompress the input stream and then writes the tar stream to a filesystem. +func UnzipTarToFs(fs vfs.FileSystem, in io.Reader) error { + r, _, err := compression.AutoDecompress(in) + if err != nil { + return err + } + defer r.Close() + err = ExtractTarToFs(fs, r) + if err != nil { + return err + } + return err +} + +// ExtractTgzToTempFs extracts a tar.gz archive to a temporary filesystem. +// You should call vfs.Cleanup on the returned filesystem to clean up the temporary files. +func ExtractTgzToTempFs(in io.Reader) (vfs.FileSystem, error) { + fs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + + return fs, UnzipTarToFs(fs, in) +} + func ExtractTarToFsWithInfo(fs vfs.FileSystem, in io.Reader) (fcnt int64, bcnt int64, err error) { tr := tar.NewReader(in) for { diff --git a/pkg/utils/tarutils/pack.go b/pkg/utils/tarutils/pack.go index 584308a434..c65ce25a78 100644 --- a/pkg/utils/tarutils/pack.go +++ b/pkg/utils/tarutils/pack.go @@ -2,10 +2,14 @@ package tarutils import ( "archive/tar" + "compress/gzip" "fmt" "io" + "io/fs" pathutil "path" + "sort" "strings" + "time" "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/goutils/finalizer" @@ -184,3 +188,97 @@ func addFileToTar(fs vfs.FileSystem, tw *tar.Writer, path string, realPath strin return fmt.Errorf("unsupported file type %s in %s", info.Mode().String(), path) } } + +func Epoch() time.Time { + return time.Unix(0, 0) +} + +func SimpleTarHeader(fs vfs.FileSystem, filepath string) (*tar.Header, error) { + info, err := fs.Lstat(filepath) + if err != nil { + return nil, err + } + return RegularFileInfoHeader(info), nil +} + +// RegularFileInfoHeader creates a tar header for a regular file (`tar.TypeReg`). +// Besides name and size, the other header fields are set to default values (`fs.ModePerm`, 0, "", `time.Unix(0,0)`). +func RegularFileInfoHeader(fi fs.FileInfo) *tar.Header { + h := &tar.Header{ + Typeflag: tar.TypeReg, + Name: fi.Name(), + Size: fi.Size(), + Mode: int64(fs.ModePerm), + Uid: 0, + Gid: 0, + Uname: "", + Gname: "", + ModTime: Epoch(), + AccessTime: Epoch(), + ChangeTime: Epoch(), + } + return h +} + +// ListSortedFilesInDir returns a list of files in a directory sorted by name. +// Attention: If 'flat == true', files with same name but in different sub-paths, will be listed only once!!! +func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string, error) { + var files []string + err := vfs.Walk(fs, root, func(path string, info vfs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + pathOrName := path + if flat { + pathOrName = info.Name() + } + files = append(files, pathOrName) + } + return nil + }) + sort.Strings(files) + return files, err +} + +// TgzFs creates a tar.gz archive from a filesystem with all files being in the root of the zipped archive. +// The writer is closed after the archive is written. The TAR-headers are normalized, see RegularFileInfoHeader. +func TgzFs(fs vfs.FileSystem, writer io.Writer) error { + zip := gzip.NewWriter(writer) + err := TarFlatFs(fs, zip) + if err != nil { + return err + } + return zip.Close() +} + +func TarFlatFs(fs vfs.FileSystem, writer io.Writer) error { + tw := tar.NewWriter(writer) + files, err := ListSortedFilesInDir(fs, "", true) + if err != nil { + return err + } + + for _, fileName := range files { + header, err := SimpleTarHeader(fs, fileName) + if err != nil { + return err + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("unable to write header for %q: %w", fileName, err) + } + file, err := fs.OpenFile(fileName, vfs.O_RDONLY, vfs.ModePerm) + if err != nil { + return fmt.Errorf("unable to open file %q: %w", fileName, err) + } + if _, err := io.Copy(tw, file); err != nil { + _ = file.Close() + return fmt.Errorf("unable to add file to tar %q: %w", fileName, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("unable to close file %q: %w", fileName, err) + } + } + + return tw.Close() +} diff --git a/pkg/utils/tarutils/pack_test.go b/pkg/utils/tarutils/pack_test.go index 0ff84d662d..2487913a61 100644 --- a/pkg/utils/tarutils/pack_test.go +++ b/pkg/utils/tarutils/pack_test.go @@ -1,14 +1,16 @@ package tarutils_test import ( + "io/fs" "os" + "runtime" + "github.com/mandelsoft/vfs/pkg/osfs" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/open-component-model/ocm/pkg/testutils" - - "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/open-component-model/ocm/pkg/errors" + . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/utils/tarutils" ) @@ -40,4 +42,17 @@ var _ = Describe("tar utils mapping", func() { list := Must(tarutils.ListArchiveContent(file.Name())) Expect(list).To(ConsistOf("dir", "dir/dirlink", "dir/link", "dir/regular", "dir/subdir", "dir/subdir/file", "dir2", "dir2/file2", "file", "dir/dirlink/file2")) }) + + It("test ListSortedFilesInDir with non existing path", func() { + files, err := tarutils.ListSortedFilesInDir(osfs.New(), "/path/doesn't/exist!", true) + Expect(err).To(HaveOccurred()) + Expect(files).To(BeNil()) + Expect(errors.Is(err, fs.ErrNotExist)).To(BeTrue()) + if runtime.GOOS == "windows" { + Expect(err.Error()).To(ContainSubstring("The system cannot find the path specified.")) + } else { + Expect(err.Error()).To(ContainSubstring("no such file or directory")) + } + }) + }) diff --git a/pkg/version/generate/release_generate.go b/pkg/version/generate/release_generate.go index 3e864c26a9..d93f2c6a6c 100644 --- a/pkg/version/generate/release_generate.go +++ b/pkg/version/generate/release_generate.go @@ -82,6 +82,8 @@ func main() { //nolint:forbidigo // Logger not needed for this command. switch cmd { + case "print-semver": + fmt.Print(nonpre) case "print-version": fmt.Print(v) case "print-rc-version": diff --git a/pkg/version/version.go b/pkg/version/version.go index c764b1a3b0..1c7ef18fcf 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -45,6 +45,16 @@ func (info Info) String() string { return info.GitVersion } +// String returns info as a short semantic version string (0.8.15). +func (info Info) SemVer() string { + return info.Major + "." + info.Minor + "." + info.Patch +} + +// String returns current Release version. +func Current() string { + return Get().SemVer() +} + // GetInterface returns the overall codebase version. It's for detecting // what code a binary was built from. // These variables typically come from -ldflags settings and in