diff --git a/cmds/ocm/app/app.go b/cmds/ocm/app/app.go index 71af0e4282..3f479b79c9 100644 --- a/cmds/ocm/app/app.go +++ b/cmds/ocm/app/app.go @@ -95,9 +95,9 @@ type CLIOptions struct { } var desc = ` -The Open Component Model command line client support the work with OCM +The Open Component Model command line client supports the work with OCM artifacts, like Component Archives, Common Transport Archive, -Component Repositories, and component versions. +Component Repositories, and Component Versions. Additionally it provides some limited support for the docker daemon, OCI artifacts and registries. diff --git a/cmds/ocm/app/app_test.go b/cmds/ocm/app/app_test.go index 1980b7faf9..4d19586ea8 100644 --- a/cmds/ocm/app/app_test.go +++ b/cmds/ocm/app/app_test.go @@ -72,7 +72,9 @@ var _ = Describe("Test Environment", func() { buf := bytes.NewBuffer(nil) Expect(env.CatchOutput(buf).ExecuteModified(addTestCommands, "logtest")).To(Succeed()) Expect(log.String()).To(StringEqualTrimmedWithContext(` +V[2] warn realm ocm realm test ERROR error realm ocm realm test +V[2] ctxwarn realm ocm realm test ERROR ctxerror realm ocm realm test `)) }) @@ -139,7 +141,7 @@ ERROR ctxerror realm ocm realm test fmt.Printf("%s\n", string(data)) // {"level":"error","msg":"error","realm":"test","time":"2024-03-27 09:54:19"} // {"level":"error","msg":"ctxerror","realm":"test","time":"2024-03-27 09:54:19"} - Expect(len(string(data))).To(Equal(155)) + Expect(len(string(data))).To(Equal(312)) }) It("sets attr from file", func() { diff --git a/cmds/ocm/topics/oci/refs/topic.go b/cmds/ocm/topics/oci/refs/topic.go index 74bc6ba8ca..3664c59110 100644 --- a/cmds/ocm/topics/oci/refs/topic.go +++ b/cmds/ocm/topics/oci/refs/topic.go @@ -18,7 +18,21 @@ func New(ctx clictx.Context) *cobra.Command { Use: "oci-references", Short: "notation for OCI references", Example: ` -ghcr.io/mandelsoft/cnudie:1.0.0 ++ctf+directory::./ocm/ctf//ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::{"baseUrl": "ghcr.io"}//open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::https://ghcr.io/open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 +oci::https://ghcr.io//open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::http://localhost:8080/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 +oci::http://localhost:8080//ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +ubuntu:24.04 +ubuntu + +tensorflow/tensorflow:2.15.0 +tensorflow/tensorflow `, Long: ` The command line client supports a special notation scheme for specifying @@ -29,26 +43,97 @@ images are possible: 
[+][<type>::][./][<file path>//<repository>[:<tag>][@<digest>]
- or - 
[<type>::][<json repo spec>//]<repository>[:<tag>][@<digest>]
- or - 
[<type>::][<scheme>:://]<domain>[:<port>/]<repository>[:<tag>][@<digest>]
- or +
+ +or + +
+ 
[+][<type>::][<json repo spec>//]<repository>[:<tag>][@<digest>]
+
+ +Notice that if you specify the <type> in the beginning of this +notation AND in the <json repo spec>, the types have to match +(but there is no reason to specify the type in both places). + +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/]/<repository>[:<tag>][@<digest>]
+
+ +Notice that this notation optionally also allows a double slash to +seperate <domain>[:<port>] and <repository>. While it is +not necessary for unambiguous parsing here, it is supported for +consistency with the other notations. + +or + +
+ 
[+][<type>::][<scheme>://]<host>:<port>/<repository>[:<tag>][@<digest>]
+
+ +Notice that <port> is required in this notation. Without <port>, +this notation would be ambiguous with the docker library notation +mentioned below. + +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>]//<repository>[:<tag>][@<digest>]
+
+ +Notice the double slash (//) before the <repository>. This serves as +a clear separator between <host>[:<port>] and <repository>. +Thus, with this notation, the port is optional and can therefore be +omitted without creating ambiguity with the docker library notation +mentioned below. + +or + +
<docker library>[:<tag>][@<digest>]
- or +
+ +or + +
<docker repository>/<docker image>[:<tag>][@<digest>]
+--- + Besides dedicated artifacts it is also possible to denote registries as a whole:
- 
[+][<type>::][<scheme>:://]<domain>[:<port>]
- or - 
[+][<type>::]<json repo spec>
- or - 
[+][<type>::][./]<file path>
+ 
[+][<type>::][./]<file path>
+
+ +or + +
+ 
[+][<type>::]<json repo spec>
+
+ +Notice that if you specify the <type> in the beginning of this +notation AND in the <json repo spec>, the types have to match +(but there is no reason to specify the type in both places). + +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>]
+ +Notice that <port> is optional in this notation since this cannot be +an image reference and therefore cannot be ambiguous with the docker +library notation. ` + FileBasedUsage(), } } diff --git a/cmds/ocm/topics/ocm/refs/topic.go b/cmds/ocm/topics/ocm/refs/topic.go index 5bb6f499ad..41cac75876 100644 --- a/cmds/ocm/topics/ocm/refs/topic.go +++ b/cmds/ocm/topics/ocm/refs/topic.go @@ -16,9 +16,27 @@ func New(ctx clictx.Context) *cobra.Command { Use: "ocm-references", Short: "notation for OCM references", Example: ` -ghcr.io/mandelsoft/cnudie//github.com/mandelsoft/pause:1.0.0 +Complete Component Reference Specifications (including all optional arguments): -ctf+tgz::./ctf ++ctf+directory::./ocm/ctf//ocm.software/ocmcli:0.7.0 + +oci::{"baseUrl":"ghcr.io","componentNameMapping":"urlPath","subPath":"open-component-model"}//ocm.software/ocmcli.0.7.0 + +oci::https://ghcr.io:443/open-component-model//ocm.software/ocmcli:0.7.0 + +oci::http://localhost:8080/local-component-repository//ocm.software/ocmcli:0.7.0 + +--- + +Short-Hand Component Reference Specifications (omitting optional arguments): + +./ocm/ctf//ocm.software/ocmcli:0.7.0 + +ghcr.io/open-component-model//ocm.software/ocmcli:0.7.0 + +localhost:8080/local-component-repository//ocm.software/ocmcli:0.7.0 (defaulting to https) + +http://localhost:8080/local-component-repository//ocm.software/ocmcli:0.7.0 `, Long: ` The command line client supports a special notation scheme for specifying @@ -27,24 +45,53 @@ references to any registry supported by the OCM toolset that can host OCM components:
- 
[+][<type>::][./][<file path>//<component id>[:<version>]
- or - 
[+][<type>::]<domain>[:<port>][/<repository prefix>]//<component id>[:<version]
- or - 
[<type>::][<json repo spec>//]<component id>[:<version>]
+ 
[+][<type>::][./]<file path>//<component id>[:<version>]
+
+ +or +
+ 
[+][<type>::][<json repo spec>//]<component id>[:<version>]
+or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/<repository prefix>]//<component id>[:<version]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>][/<repository prefix>]//<component id>[:<version]
+
+ +--- + Besides dedicated components it is also possible to denote repositories as a whole:
- 
[+][<type>::][<scheme>:://]<domain>[:<port>][/<repository prefix>]
- or - 
[+][<type>::]<json repo spec>
- or 
[+][<type>::][./]<file path>
+ +or + +
+ 
[+][<type>::]<json repo spec>
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/<repository prefix>]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>][/<repository prefix>]
+
` + topicocirefs.FileBasedUsage(), } } diff --git a/components/ocmcli/Makefile b/components/ocmcli/Makefile index 694f6b7269..30db9447fc 100644 --- a/components/ocmcli/Makefile +++ b/components/ocmcli/Makefile @@ -20,7 +20,7 @@ PLATFORM_ARCH := $(shell go env GOARCH) CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(CMD) -type f) Makefile OCMSRCS=$(shell find $(REPO_ROOT)/pkg -type f) $(REPO_ROOT)/go.* -ATTRIBUTES = VERSION="$(VERSION)" NAME="$(NAME)" COMMIT="$(COMMIT)" IMAGE="$(IMAGE):$(VERSION)" PLATFORMS="$(IMAGE_PLATFORMS)" GEN="$(GEN)" MULTI=$(MULTI) +ATTRIBUTES = VERSION="$(VERSION)" NAME="$(NAME)" COMMIT="$(COMMIT)" IMAGE="$(IMAGE):$(VERSION)" PLATFORMS="$(PLATFORMS)" IMAGE_PLATFORMS="$(IMAGE_PLATFORMS)" GEN="$(GEN)" MULTI=$(MULTI) ifeq ($(MULTI),true) FLAGSUF = .multi diff --git a/components/ocmcli/resources.yaml b/components/ocmcli/resources.yaml index 312ab25ff4..ac152b9b83 100644 --- a/components/ocmcli/resources.yaml +++ b/components/ocmcli/resources.yaml @@ -23,7 +23,7 @@ helper: input: type: (( bool(values.MULTI) ? "dockermulti" :"docker" )) repository: (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE )) - variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) + variants: (( bool(values.MULTI) ? map[split(" ", values.IMAGE_PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) path: (( !bool(values.MULTI) ? values.IMAGE :~~ )) diff --git a/docs/pluginreference/plugin_accessmethod_compose.md b/docs/pluginreference/plugin_accessmethod_compose.md index 58bd57eff4..5ffbea5834 100644 --- a/docs/pluginreference/plugin_accessmethod_compose.md +++ b/docs/pluginreference/plugin_accessmethod_compose.md @@ -40,28 +40,36 @@ The following predefined option types can be used: - accessRegistry: [*string*] registry base URL - accessRepository: [*string*] repository URL - accessVersion: [*string*] version for access specification + - body: [*string*] body of a http request - bucket: [*string*] bucket name - comment: [*string*] comment field value - commit: [*string*] git commit id - digest: [*string*] blob digest - globalAccess: [*map[string]YAML*] access specification for global access + - header: [*string:string,string*] http headers - hint: [*string*] (repository) hint for local artifacts - mediaType: [*string*] media type for artifact blob representation + - noredirect: [*bool*] http redirect behavior - reference: [*string*] reference name - region: [*string*] region name - size: [*int*] blob size + - url: [*string*] artifact or server url + - verb: [*string*] http request method The following predefined value types are supported: - YAML: JSON or YAML document string + - []byte: byte value - []string: list of string values - bool: boolean flag - int: integer value - map[string]YAML: JSON or YAML map - string: string value + - string:string,string: string map defined by dedicated assignment of comma separated strings - string=YAML: string map with arbitrary values defined by dedicated assignments - string=string: string map defined by dedicated assignments + - string=string,string: string map defined by dedicated assignment of comma separated strings ### SEE ALSO diff --git a/docs/pluginreference/plugin_descriptor.md b/docs/pluginreference/plugin_descriptor.md index 2700827888..f04a2169f4 100644 --- a/docs/pluginreference/plugin_descriptor.md +++ b/docs/pluginreference/plugin_descriptor.md @@ -125,28 +125,36 @@ The following predefined option types can be used: - accessRegistry: [*string*] registry base URL - accessRepository: [*string*] repository URL - accessVersion: [*string*] version for access specification + - body: [*string*] body of a http request - bucket: [*string*] bucket name - comment: [*string*] comment field value - commit: [*string*] git commit id - digest: [*string*] blob digest - globalAccess: [*map[string]YAML*] access specification for global access + - header: [*string:string,string*] http headers - hint: [*string*] (repository) hint for local artifacts - mediaType: [*string*] media type for artifact blob representation + - noredirect: [*bool*] http redirect behavior - reference: [*string*] reference name - region: [*string*] region name - size: [*int*] blob size + - url: [*string*] artifact or server url + - verb: [*string*] http request method The following predefined value types are supported: - YAML: JSON or YAML document string + - []byte: byte value - []string: list of string values - bool: boolean flag - int: integer value - map[string]YAML: JSON or YAML map - string: string value + - string:string,string: string map defined by dedicated assignment of comma separated strings - string=YAML: string map with arbitrary values defined by dedicated assignments - string=string: string map defined by dedicated assignments + - string=string,string: string map defined by dedicated assignment of comma separated strings #### Uploader Descriptor diff --git a/docs/pluginreference/plugin_valueset_compose.md b/docs/pluginreference/plugin_valueset_compose.md index d7466f9d9f..0ad1025ed4 100644 --- a/docs/pluginreference/plugin_valueset_compose.md +++ b/docs/pluginreference/plugin_valueset_compose.md @@ -40,28 +40,36 @@ The following predefined option types can be used: - accessRegistry: [*string*] registry base URL - accessRepository: [*string*] repository URL - accessVersion: [*string*] version for access specification + - body: [*string*] body of a http request - bucket: [*string*] bucket name - comment: [*string*] comment field value - commit: [*string*] git commit id - digest: [*string*] blob digest - globalAccess: [*map[string]YAML*] access specification for global access + - header: [*string:string,string*] http headers - hint: [*string*] (repository) hint for local artifacts - mediaType: [*string*] media type for artifact blob representation + - noredirect: [*bool*] http redirect behavior - reference: [*string*] reference name - region: [*string*] region name - size: [*int*] blob size + - url: [*string*] artifact or server url + - verb: [*string*] http request method The following predefined value types are supported: - YAML: JSON or YAML document string + - []byte: byte value - []string: list of string values - bool: boolean flag - int: integer value - map[string]YAML: JSON or YAML map - string: string value + - string:string,string: string map defined by dedicated assignment of comma separated strings - string=YAML: string map with arbitrary values defined by dedicated assignments - string=string: string map defined by dedicated assignments + - string=string,string: string map defined by dedicated assignment of comma separated strings ### SEE ALSO diff --git a/docs/reference/ocm.md b/docs/reference/ocm.md index 3046ee8c66..d2e3a12be2 100644 --- a/docs/reference/ocm.md +++ b/docs/reference/ocm.md @@ -29,9 +29,9 @@ ocm [] ... ### Description -The Open Component Model command line client support the work with OCM +The Open Component Model command line client supports the work with OCM artifacts, like Component Archives, Common Transport Archive, -Component Repositories, and component versions. +Component Repositories, and Component Versions. Additionally it provides some limited support for the docker daemon, OCI artifacts and registries. diff --git a/docs/reference/ocm_add_routingslips.md b/docs/reference/ocm_add_routingslips.md index ce8478ffa4..dd10010bf6 100644 --- a/docs/reference/ocm_add_routingslips.md +++ b/docs/reference/ocm_add_routingslips.md @@ -97,10 +97,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_configuration.md b/docs/reference/ocm_bootstrap_configuration.md index e7609ec4cb..142e8d2202 100644 --- a/docs/reference/ocm_bootstrap_configuration.md +++ b/docs/reference/ocm_bootstrap_configuration.md @@ -78,10 +78,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_package.md b/docs/reference/ocm_bootstrap_package.md index ed44d538b5..bef6e13a14 100644 --- a/docs/reference/ocm_bootstrap_package.md +++ b/docs/reference/ocm_bootstrap_package.md @@ -159,10 +159,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_check_componentversions.md b/docs/reference/ocm_check_componentversions.md index 66af32da98..db42cf0858 100644 --- a/docs/reference/ocm_check_componentversions.md +++ b/docs/reference/ocm_check_componentversions.md @@ -66,10 +66,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_artifacts.md b/docs/reference/ocm_describe_artifacts.md index 218a74ba85..5c5728682a 100644 --- a/docs/reference/ocm_describe_artifacts.md +++ b/docs/reference/ocm_describe_artifacts.md @@ -60,6 +60,7 @@ linked library can be used: - DockerDaemon: v1 - Empty: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_package.md b/docs/reference/ocm_describe_package.md index 60eb15f23a..1790bd2a77 100644 --- a/docs/reference/ocm_describe_package.md +++ b/docs/reference/ocm_describe_package.md @@ -69,10 +69,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_artifacts.md b/docs/reference/ocm_download_artifacts.md index c2a2e85abe..101dea7314 100644 --- a/docs/reference/ocm_download_artifacts.md +++ b/docs/reference/ocm_download_artifacts.md @@ -62,6 +62,7 @@ linked library can be used: - DockerDaemon: v1 - Empty: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_cli.md b/docs/reference/ocm_download_cli.md index fc7a0ee462..01dc462c08 100644 --- a/docs/reference/ocm_download_cli.md +++ b/docs/reference/ocm_download_cli.md @@ -76,10 +76,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_componentversions.md b/docs/reference/ocm_download_componentversions.md index 82487f63b6..de66681659 100644 --- a/docs/reference/ocm_download_componentversions.md +++ b/docs/reference/ocm_download_componentversions.md @@ -65,10 +65,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_resources.md b/docs/reference/ocm_download_resources.md index b48beb3071..2754454a4f 100644 --- a/docs/reference/ocm_download_resources.md +++ b/docs/reference/ocm_download_resources.md @@ -101,10 +101,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_artifacts.md b/docs/reference/ocm_get_artifacts.md index 2eb9447cb9..a1f370c2c9 100644 --- a/docs/reference/ocm_get_artifacts.md +++ b/docs/reference/ocm_get_artifacts.md @@ -60,6 +60,7 @@ linked library can be used: - DockerDaemon: v1 - Empty: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_componentversions.md b/docs/reference/ocm_get_componentversions.md index 3f33d7d644..8d225c75b9 100644 --- a/docs/reference/ocm_get_componentversions.md +++ b/docs/reference/ocm_get_componentversions.md @@ -75,10 +75,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_references.md b/docs/reference/ocm_get_references.md index aaf24e7728..0807414767 100644 --- a/docs/reference/ocm_get_references.md +++ b/docs/reference/ocm_get_references.md @@ -76,10 +76,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_resources.md b/docs/reference/ocm_get_resources.md index cf9ef134d3..8abddf2f05 100644 --- a/docs/reference/ocm_get_resources.md +++ b/docs/reference/ocm_get_resources.md @@ -76,10 +76,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_routingslips.md b/docs/reference/ocm_get_routingslips.md index 518d9d3c0c..d0adb0de32 100644 --- a/docs/reference/ocm_get_routingslips.md +++ b/docs/reference/ocm_get_routingslips.md @@ -75,10 +75,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_sources.md b/docs/reference/ocm_get_sources.md index f3bc6f3c1f..51b4bfe545 100644 --- a/docs/reference/ocm_get_sources.md +++ b/docs/reference/ocm_get_sources.md @@ -76,10 +76,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_hash_componentversions.md b/docs/reference/ocm_hash_componentversions.md index 7999825f90..f9fd49fc13 100644 --- a/docs/reference/ocm_hash_componentversions.md +++ b/docs/reference/ocm_hash_componentversions.md @@ -109,10 +109,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_install_plugins.md b/docs/reference/ocm_install_plugins.md index 5e27180a96..72777db8db 100644 --- a/docs/reference/ocm_install_plugins.md +++ b/docs/reference/ocm_install_plugins.md @@ -68,10 +68,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_oci-references.md b/docs/reference/ocm_oci-references.md index c29d9b2404..b317192bfa 100644 --- a/docs/reference/ocm_oci-references.md +++ b/docs/reference/ocm_oci-references.md @@ -11,27 +11,98 @@ images are possible: 
[+][<type>::][./][<file path>//<repository>[:<tag>][@<digest>]
- or - 
[<type>::][<json repo spec>//]<repository>[:<tag>][@<digest>]
- or - 
[<type>::][<scheme>:://]<domain>[:<port>/]<repository>[:<tag>][@<digest>]
- or +
+ +or + +
+ 
[+][<type>::][<json repo spec>//]<repository>[:<tag>][@<digest>]
+
+ +Notice that if you specify the <type> in the beginning of this +notation AND in the <json repo spec>, the types have to match +(but there is no reason to specify the type in both places). + +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/]/<repository>[:<tag>][@<digest>]
+
+ +Notice that this notation optionally also allows a double slash to +seperate <domain>[:<port>] and <repository>. While it is +not necessary for unambiguous parsing here, it is supported for +consistency with the other notations. + +or + +
+ 
[+][<type>::][<scheme>://]<host>:<port>/<repository>[:<tag>][@<digest>]
+
+ +Notice that <port> is required in this notation. Without <port>, +this notation would be ambiguous with the docker library notation +mentioned below. + +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>]//<repository>[:<tag>][@<digest>]
+
+ +Notice the double slash (//) before the <repository>. This serves as +a clear separator between <host>[:<port>] and <repository>. +Thus, with this notation, the port is optional and can therefore be +omitted without creating ambiguity with the docker library notation +mentioned below. + +or + +
<docker library>[:<tag>][@<digest>]
- or +
+ +or + +
<docker repository>/<docker image>[:<tag>][@<digest>]
+--- + Besides dedicated artifacts it is also possible to denote registries as a whole:
- 
[+][<type>::][<scheme>:://]<domain>[:<port>]
- or - 
[+][<type>::]<json repo spec>
- or - 
[+][<type>::][./]<file path>
+ 
[+][<type>::][./]<file path>
+
+ +or + +
+ 
[+][<type>::]<json repo spec>
+
+ +Notice that if you specify the <type> in the beginning of this +notation AND in the <json repo spec>, the types have to match +(but there is no reason to specify the type in both places). + +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>]
+Notice that <port> is optional in this notation since this cannot be +an image reference and therefore cannot be ambiguous with the docker +library notation. + The optional + is used for file based implementations (Common Transport Format) to indicate the creation of a not yet existing file. @@ -42,7 +113,21 @@ character. The following formats are supported: directory, ta ### Examples ``` -ghcr.io/mandelsoft/cnudie:1.0.0 ++ctf+directory::./ocm/ctf//ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::{"baseUrl": "ghcr.io"}//open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::https://ghcr.io/open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 +oci::https://ghcr.io//open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +oci::http://localhost:8080/ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 +oci::http://localhost:8080//ocm.software/ocmcli/ocmcli-image:0.7.0@sha256:29c842be1ef1da67f6a1c07a3a3a8eb101bbcc4c80f174b87d147b341bca9625 + +ubuntu:24.04 +ubuntu + +tensorflow/tensorflow:2.15.0 +tensorflow/tensorflow ``` ### SEE ALSO diff --git a/docs/reference/ocm_ocm-references.md b/docs/reference/ocm_ocm-references.md index 28e2e37c64..9b486b596e 100644 --- a/docs/reference/ocm_ocm-references.md +++ b/docs/reference/ocm_ocm-references.md @@ -9,25 +9,54 @@ references to any registry supported by the OCM toolset that can host OCM components:
- 
[+][<type>::][./][<file path>//<component id>[:<version>]
- or - 
[+][<type>::]<domain>[:<port>][/<repository prefix>]//<component id>[:<version]
- or - 
[<type>::][<json repo spec>//]<component id>[:<version>]
+ 
[+][<type>::][./]<file path>//<component id>[:<version>]
+
+ +or + +
+ 
[+][<type>::][<json repo spec>//]<component id>[:<version>]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/<repository prefix>]//<component id>[:<version]
+
+or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>][/<repository prefix>]//<component id>[:<version]
+--- + Besides dedicated components it is also possible to denote repositories as a whole:
- 
[+][<type>::][<scheme>:://]<domain>[:<port>][/<repository prefix>]
- or - 
[+][<type>::]<json repo spec>
- or 
[+][<type>::][./]<file path>
+or + +
+ 
[+][<type>::]<json repo spec>
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<domain>[:<port>][/<repository prefix>]
+
+ +or + +
+ 
[+][<type>::][<scheme>://]<host>[:<port>][/<repository prefix>]
+
+ The optional + is used for file based implementations (Common Transport Format) to indicate the creation of a not yet existing file. @@ -38,9 +67,13 @@ character. The following formats are supported: directory, ta ### Examples ``` -ghcr.io/mandelsoft/cnudie//github.com/mandelsoft/pause:1.0.0 ++ctf+directory::./ocm/ctf//ocm.software/ocmcli:0.7.0 + +oci::{"baseUrl":"ghcr.io","componentNameMapping":"urlPath","subPath":"open-component-model"}//ocm.software/ocmcli.0.7.0 + +oci::https://ghcr.io:443/open-component-model//ocm.software/ocmcli:0.7.0 -ctf+tgz::./ctf +oci::http://localhost:8080/local-component-repository//ocm.software/ocmcli:0.7.0 ``` ### SEE ALSO diff --git a/docs/reference/ocm_show_tags.md b/docs/reference/ocm_show_tags.md index 886ee7a49f..3fd0a36d29 100644 --- a/docs/reference/ocm_show_tags.md +++ b/docs/reference/ocm_show_tags.md @@ -52,6 +52,7 @@ linked library can be used: - DockerDaemon: v1 - Empty: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_show_versions.md b/docs/reference/ocm_show_versions.md index 344a20baff..309f09b1e0 100644 --- a/docs/reference/ocm_show_versions.md +++ b/docs/reference/ocm_show_versions.md @@ -62,10 +62,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_sign_componentversions.md b/docs/reference/ocm_sign_componentversions.md index f349615e29..a19b73c505 100644 --- a/docs/reference/ocm_sign_componentversions.md +++ b/docs/reference/ocm_sign_componentversions.md @@ -84,10 +84,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_artifacts.md b/docs/reference/ocm_transfer_artifacts.md index b650382015..3a57c7b4f2 100644 --- a/docs/reference/ocm_transfer_artifacts.md +++ b/docs/reference/ocm_transfer_artifacts.md @@ -71,6 +71,7 @@ linked library can be used: - DockerDaemon: v1 - Empty: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index cbbae63fe8..5e75f7db5b 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -87,10 +87,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_verify_componentversions.md b/docs/reference/ocm_verify_componentversions.md index e34ae26685..08997e38ef 100644 --- a/docs/reference/ocm_verify_componentversions.md +++ b/docs/reference/ocm_verify_componentversions.md @@ -81,10 +81,12 @@ linked library can be used: Dedicated OCM repository types: - ComponentArchive: v1 + - ca: v1 OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - OCIRegistry: v1 + - ctf: v1 - oci: v1 - ociRegistry diff --git a/pkg/cobrautils/logopts/options.go b/pkg/cobrautils/logopts/options.go index 85058baa07..5af9483f5f 100644 --- a/pkg/cobrautils/logopts/options.go +++ b/pkg/cobrautils/logopts/options.go @@ -87,7 +87,7 @@ func (o *Options) Configure(ctx ocm.Context, logctx logging.Context) error { } logctx.SetDefaultLevel(l) } else { - logctx.SetDefaultLevel(logging.ErrorLevel) + logctx.SetDefaultLevel(logging.WarnLevel) } logcfg := &config.Config{DefaultLevel: logging.LevelName(logctx.GetDefaultLevel())} diff --git a/pkg/contexts/clictx/internal/context.go b/pkg/contexts/clictx/internal/context.go index ef30d76158..d6a3cfa70c 100644 --- a/pkg/contexts/clictx/internal/context.go +++ b/pkg/contexts/clictx/internal/context.go @@ -24,6 +24,7 @@ import ( ctfocm "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/out" + "github.com/open-component-model/ocm/pkg/utils" ) const CONTEXT_TYPE = "ocm.cmd" + datacontext.OCM_CONTEXT_SUFFIX @@ -47,15 +48,18 @@ func (f *FileSystem) ApplyOption(options accessio.Options) error { return nil } +type ContextProvider interface { + CLIContext() Context +} + type Context interface { datacontext.Context - - AttributesContext() datacontext.AttributesContext - - ConfigContext() config.Context - CredentialsContext() credentials.Context - OCIContext() oci.Context - OCMContext() ocm.Context + ContextProvider + datacontext.ContextProvider + config.ContextProvider + credentials.ContextProvider + oci.ContextProvider + ocm.ContextProvider FileSystem() *FileSystem @@ -91,13 +95,14 @@ func DefinedForContext(ctx context.Context) (Context, bool) { //////////////////////////////////////////////////////////////////////////////// +type _InternalContext = datacontext.InternalContext + type _context struct { - datacontext.Context + _InternalContext updater cfgcpi.Updater sharedAttributes datacontext.AttributesContext - config config.Context credentials credentials.Context oci *_oci ocm *_ocm @@ -105,7 +110,29 @@ type _context struct { out out.Context } -var _ Context = &_context{} +var ( + _ Context = (*_context)(nil) + _ datacontext.ViewCreator[Context] = (*_context)(nil) +) + +// gcWrapper is used as garbage collectable +// wrapper for a context implementation +// to establish a runtime finalizer. +type gcWrapper struct { + datacontext.GCWrapper + *_context +} + +func newView(c *_context, ref ...bool) Context { + if utils.Optional(ref...) { + return datacontext.FinalizedContext[gcWrapper](c) + } + return c +} + +func (w *gcWrapper) SetContext(c *_context) { + w._context = c +} func newContext(shared datacontext.AttributesContext, ocmctx ocm.Context, outctx out.Context, fs vfs.FileSystem, delegates datacontext.Delegates) Context { if outctx == nil { @@ -115,19 +142,27 @@ func newContext(shared datacontext.AttributesContext, ocmctx ocm.Context, outctx shared = ocmctx.AttributesContext() } c := &_context{ - sharedAttributes: shared, - credentials: ocmctx.CredentialsContext(), - config: ocmctx.CredentialsContext().ConfigContext(), + sharedAttributes: datacontext.PersistentContextRef(shared), + credentials: datacontext.PersistentContextRef(ocmctx.CredentialsContext()), out: outctx, } - c.Context = datacontext.NewContextBase(c, CONTEXT_TYPE, key, ocmctx.GetAttributes(), delegates) - c.updater = cfgcpi.NewUpdater(ocmctx.CredentialsContext().ConfigContext(), c) + c._InternalContext = datacontext.NewContextBase(c, CONTEXT_TYPE, key, ocmctx.GetAttributes(), delegates) + c.updater = cfgcpi.NewUpdater(datacontext.PersistentContextRef(ocmctx.CredentialsContext().ConfigContext()), c) + ocmctx = datacontext.PersistentContextRef(ocmctx) c.oci = newOCI(c, ocmctx) c.ocm = newOCM(c, ocmctx) if fs != nil { vfsattr.Set(c.AttributesContext(), fs) } - return c + return newView(c, true) +} + +func (c *_context) CreateView() Context { + return newView(c, true) +} + +func (c *_context) CLIContext() Context { + return newView(c) } func (c *_context) Update() error { @@ -139,7 +174,7 @@ func (c *_context) AttributesContext() datacontext.AttributesContext { } func (c *_context) ConfigContext() config.Context { - return c.config + return c.updater.GetContext() } func (c *_context) CredentialsContext() credentials.Context { @@ -155,7 +190,7 @@ func (c *_context) OCMContext() ocm.Context { } func (c *_context) FileSystem() *FileSystem { - return &FileSystem{vfsattr.Get(c)} + return &FileSystem{vfsattr.Get(c.CLIContext())} } func (c *_context) OCI() OCI { @@ -185,7 +220,7 @@ func (c *_context) StdIn() io.Reader { func (c *_context) WithStdIO(r io.Reader, o io.Writer, e io.Writer) Context { return &_view{ - Context: c, + Context: c.CLIContext(), out: out.NewFor(out.WithStdIO(c.out, r, o, e)), } } @@ -211,7 +246,7 @@ func (c *_view) StdIn() io.Reader { func (c *_view) WithStdIO(r io.Reader, o io.Writer, e io.Writer) Context { return &_view{ - Context: c.Context, + Context: c.CLIContext(), out: out.NewFor(out.WithStdIO(c.out, r, o, e)), } } @@ -222,16 +257,16 @@ func (c *_view) WithStdIO(r io.Reader, o io.Writer, e io.Writer) Context { // _ocm uses ocm and ctfocm type _oci struct { - *_context + cli *_context ctx oci.Context repos map[string]oci.RepositorySpec } func newOCI(ctx *_context, ocmctx ocm.Context) *_oci { return &_oci{ - _context: ctx, - ctx: ocmctx.OCIContext(), - repos: map[string]oci.RepositorySpec{}, + cli: ctx, + ctx: ocmctx.OCIContext(), + repos: map[string]oci.RepositorySpec{}, } } @@ -240,29 +275,29 @@ func (c *_oci) Context() oci.Context { } func (c *_oci) OpenCTF(path string) (oci.Repository, error) { - ok, err := vfs.Exists(c.FileSystem(), path) + ok, err := vfs.Exists(c.cli.FileSystem(), path) if err != nil { return nil, err } if !ok { return nil, errors.ErrNotFound("file", path) } - return ctfoci.Open(c.ctx, accessobj.ACC_WRITABLE, path, 0, accessio.PathFileSystem(c.FileSystem())) + return ctfoci.Open(c.ctx, accessobj.ACC_WRITABLE, path, 0, accessio.PathFileSystem(c.cli.FileSystem())) } //////////////////////////////////////////////////////////////////////////////// type _ocm struct { - *_context + cli *_context ctx ocm.Context repos map[string]ocm.RepositorySpec } func newOCM(ctx *_context, ocmctx ocm.Context) *_ocm { return &_ocm{ - _context: ctx, - ctx: ocmctx, - repos: map[string]ocm.RepositorySpec{}, + cli: ctx, + ctx: ocmctx, + repos: map[string]ocm.RepositorySpec{}, } } @@ -271,12 +306,12 @@ func (c *_ocm) Context() ocm.Context { } func (c *_ocm) OpenCTF(path string) (ocm.Repository, error) { - ok, err := vfs.Exists(c.FileSystem(), path) + ok, err := vfs.Exists(c.cli.FileSystem(), path) if err != nil { return nil, err } if !ok { return nil, errors.ErrNotFound("file", path) } - return ctfocm.Open(c.ctx, accessobj.ACC_WRITABLE, path, 0, c) + return ctfocm.Open(c.ctx, accessobj.ACC_WRITABLE, path, 0, c.cli.FileSystem()) } diff --git a/pkg/contexts/config/config/context_test.go b/pkg/contexts/config/config/context_test.go index 161f87edaf..7fffc81209 100644 --- a/pkg/contexts/config/config/context_test.go +++ b/pkg/contexts/config/config/context_test.go @@ -7,17 +7,27 @@ package config_test import ( "os" "reflect" + "runtime" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" + "github.com/open-component-model/ocm/pkg/generics" "sigs.k8s.io/yaml" "github.com/open-component-model/ocm/pkg/contexts/config" local "github.com/open-component-model/ocm/pkg/contexts/config/config" - "github.com/open-component-model/ocm/pkg/testutils" + . "github.com/open-component-model/ocm/pkg/testutils" ) +func CheckRefs(ctx config.Context, n int) { + runtime.GC() + time.Sleep(time.Second) + Expect(datacontext.GetContextRefCount(ctx)).To(Equal(n)) // all temp refs have been finalized +} + var _ = Describe("generic config handling", func() { var scheme config.ConfigTypeScheme @@ -41,6 +51,8 @@ var _ = Describe("generic config handling", func() { Expect(err).To(Succeed()) Expect(config.IsGeneric(result)).To(BeFalse()) Expect(reflect.TypeOf(result).String()).To(Equal("*config.Config")) + + CheckRefs(cfgctx, 1) }) It("it applies to existing context", func() { @@ -57,6 +69,8 @@ var _ = Describe("generic config handling", func() { Expect(len(cfgs)).To(Equal(3)) Expect(d.getApplied()).To(Equal([]*Config{NewConfig("alice", ""), NewConfig("", "bob")})) + + CheckRefs(cfgctx, 1) }) It("it applies nested to existing context", func() { @@ -73,6 +87,8 @@ var _ = Describe("generic config handling", func() { Expect(len(cfgs)).To(Equal(4)) Expect(d.getApplied()).To(Equal([]*Config{NewConfig("alice", ""), NewConfig("", "bob")})) + + CheckRefs(cfgctx, 1) }) It("it applies unknown type to existing context", func() { @@ -82,7 +98,7 @@ var _ = Describe("generic config handling", func() { err = cfgctx.ApplyConfig(cfg, "testconfig") Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(testutils.StringEqualWithContext("testconfig: config apply errors: {config entry 0--testconfig: config type \"Dummy\" is unknown, config entry 1--testconfig: config type \"Dummy\" is unknown}")) + Expect(err.Error()).To(StringEqualWithContext("testconfig: config apply errors: {config entry 0--testconfig: config type \"Dummy\" is unknown, config entry 1--testconfig: config type \"Dummy\" is unknown}")) gen, cfgs := cfgctx.GetConfig(config.AllGenerations, nil) Expect(gen).To(Equal(int64(3))) Expect(len(cfgs)).To(Equal(3)) @@ -90,6 +106,8 @@ var _ = Describe("generic config handling", func() { RegisterAt(scheme) d := newDummy(cfgctx) Expect(d.getApplied()).To(Equal([]*Config{NewConfig("alice", ""), NewConfig("", "bob")})) + + CheckRefs(cfgctx, 1) }) It("it applies composed config to existing context", func() { @@ -110,6 +128,8 @@ var _ = Describe("generic config handling", func() { Expect(len(cfgs)).To(Equal(3)) Expect(d.getApplied()).To(Equal([]*Config{NewConfig("alice", ""), NewConfig("", "bob")})) + + CheckRefs(cfgctx, 1) }) It("it applies composed config set to existing context", func() { @@ -138,5 +158,27 @@ var _ = Describe("generic config handling", func() { Expect(gen).To(Equal(int64(3))) Expect(len(cfgs)).To(Equal(3)) Expect(d.getApplied()).To(Equal([]*Config{NewConfig("", "bob"), NewConfig("alice", "")})) + + CheckRefs(cfgctx, 1) + }) + + It("it applies compig to storing target", func() { + RegisterAt(scheme) + d := newDummy(cfgctx) + + cfg := NewConfig("alice", "") + + err := cfgctx.ApplyConfig(cfg, "testconfig") + Expect(err).To(Succeed()) + + Expect(d.getApplied()).To(Equal([]*Config{NewConfig("alice", "")})) + + target := dummyTarget{} + MustBeSuccessful(cfgctx.ApplyTo(0, &target)) + Expect(target.used).NotTo(BeNil()) + Expect(target.used.GetId()).To(Equal(cfgctx.GetId())) + + CheckRefs(cfgctx, generics.Conditional(datacontext.MULTI_REF, 2, 1)) // config context stored in target with separate ref + target.used.GetId() }) }) diff --git a/pkg/contexts/config/config/dummy_test.go b/pkg/contexts/config/config/dummy_test.go index 26d30fd226..117056a0e6 100644 --- a/pkg/contexts/config/config/dummy_test.go +++ b/pkg/contexts/config/config/dummy_test.go @@ -50,11 +50,22 @@ func (a *Config) ApplyTo(ctx config.Context, target interface{}) error { d.applied = append(d.applied, a) return nil } + c, ok := target.(*dummyTarget) + if ok { + c.used = ctx + return nil + } return cpi.ErrNoContext(DummyType) } //////////////////////////////////////////////////////////////////////////////// +type dummyTarget struct { + used config.Context +} + +//////////////////////////////////////////////////////////////////////////////// + func newDummy(ctx config.Context) *dummyContext { d := &dummyContext{ config: ctx, diff --git a/pkg/contexts/config/context_test.go b/pkg/contexts/config/context_test.go index 4292864c48..b1ee3fcb2a 100644 --- a/pkg/contexts/config/context_test.go +++ b/pkg/contexts/config/context_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/contexts/config" "github.com/open-component-model/ocm/pkg/errors" @@ -23,6 +24,7 @@ var _ = Describe("config handling", func() { BeforeEach(func() { scheme = config.NewConfigTypeScheme() cfgctx = config.WithConfigTypeScheme(scheme).New() + Expect(cfgctx.AttributesContext().GetId()).NotTo(BeIdenticalTo(datacontext.DefaultContext.GetId())) }) It("can deserialize unknown", func() { diff --git a/pkg/contexts/config/cpi/interface.go b/pkg/contexts/config/cpi/interface.go index f5351a649d..d2ce4d6db2 100644 --- a/pkg/contexts/config/cpi/interface.go +++ b/pkg/contexts/config/cpi/interface.go @@ -62,6 +62,10 @@ func NewUpdater(ctx Context, target interface{}) Updater { return internal.NewUpdater(ctx, target) } +func NewUpdaterForFactory[T any](ctx Context, f func() T) Updater { + return internal.NewUpdaterForFactory(ctx, f) +} + //////////////////////////////////////////////////////////////////////////////// func ErrNoContext(name string) error { diff --git a/pkg/contexts/config/internal/context.go b/pkg/contexts/config/internal/context.go index a4bc566445..ea23e652ac 100644 --- a/pkg/contexts/config/internal/context.go +++ b/pkg/contexts/config/internal/context.go @@ -12,6 +12,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) // OCM_CONFIG_TYPE_SUFFIX is the standard suffix used for configuration @@ -130,7 +131,10 @@ type _context struct { description string } -var _ Context = &_context{} +var ( + _ Context = (*_context)(nil) + _ datacontext.ViewCreator[Context] = (*_context)(nil) +) // gcWrapper is used as garbage collectable // wrapper for a context implementation @@ -140,6 +144,13 @@ type gcWrapper struct { *_context } +func newView(c *_context, ref ...bool) Context { + if utils.Optional(ref...) { + return datacontext.FinalizedContext[gcWrapper](c) + } + return c +} + func (w *gcWrapper) SetContext(c *_context) { w._context = c } @@ -153,14 +164,18 @@ func newContext(shared datacontext.AttributesContext, reposcheme ConfigTypeSchem }, } c._InternalContext = datacontext.NewContextBase(c, CONTEXT_TYPE, key, shared.GetAttributes(), delegates) - c.updater = NewUpdater(c, c) - datacontext.AssureUpdater(shared, NewUpdater(c, shared)) + c.updater = NewUpdaterForFactory(c, c.ConfigContext) // provide target as new view to internal context + datacontext.AssureUpdater(shared, NewUpdater(c, datacontext.PersistentContextRef(shared))) - return datacontext.FinalizedContext[gcWrapper](c) + return newView(c, true) +} + +func (c *_context) CreateView() Context { + return newView(c, true) } func (c *_context) ConfigContext() Context { - return c + return newView(c) } func (c *_context) Update() error { @@ -177,7 +192,7 @@ func (c *_context) WithInfo(desc string) Context { if c.description != "" { desc = desc + "--" + c.description } - return &_context{c.coreContext, desc} + return newView(&_context{c.coreContext, desc}) } func (c *_context) AttributesContext() datacontext.AttributesContext { @@ -203,7 +218,9 @@ func (c *_context) GetConfigForData(data []byte, unmarshaler runtime.Unmarshaler func (c *_context) ApplyConfig(spec Config, desc string) error { var unknown error - spec = (&AppliedConfig{config: spec}).eval(c) + + // use temporary view for outbound calls + spec = (&AppliedConfig{config: spec}).eval(newView(c)) if IsGeneric(spec) { unknown = errors.ErrUnknown(KIND_CONFIGTYPE, spec.GetType()) } diff --git a/pkg/contexts/config/internal/updater.go b/pkg/contexts/config/internal/updater.go index ce76a0024e..170710ca5a 100644 --- a/pkg/contexts/config/internal/updater.go +++ b/pkg/contexts/config/internal/updater.go @@ -26,29 +26,52 @@ type Updater interface { RUnlock() } +// updater implements the Updater interface. +// It must be prepared to work with an internal config context +// representation. Therefore, it must pass a self reference +// of the context to outbound calls. type updater struct { sync.RWMutex ctx Context - target interface{} + targetFunc func() interface{} lastGeneration int64 inupdate bool } +// TargetFunction can be used to map any type specific factory function +// to a target function returning a formal interface{} type. +func TargetFunction[T any](f func() T) func() interface{} { + return func() interface{} { return f() } +} + // NewUpdater create a configuration updater for a configuration target // based on a dedicated configuration context. func NewUpdater(ctx Context, target interface{}) Updater { + var targetFunc func() interface{} + if f, ok := target.(func() interface{}); ok { + targetFunc = f + } else { + targetFunc = func() interface{} { return target } + } + return &updater{ + ctx: ctx, + targetFunc: targetFunc, + } +} + +func NewUpdaterForFactory[T any](ctx Context, t func() T) Updater { return &updater{ - ctx: ctx, - target: target, + ctx: ctx, + targetFunc: TargetFunction(t), } } func (u *updater) GetContext() Context { - return u.ctx + return u.ctx.ConfigContext() } func (u *updater) GetTarget() interface{} { - return u.target + return u.targetFunc() } func (u *updater) State() (int64, bool) { @@ -66,7 +89,7 @@ func (u *updater) Update() error { u.inupdate = true u.Unlock() - gen, err := u.ctx.ApplyTo(u.lastGeneration, u.target) + gen, err := u.ctx.ApplyTo(u.lastGeneration, u.GetTarget()) u.Lock() defer u.Unlock() diff --git a/pkg/contexts/credentials/internal/builder_test.go b/pkg/contexts/credentials/internal/builder_test.go index f748bdebe7..b128be5a93 100644 --- a/pkg/contexts/credentials/internal/builder_test.go +++ b/pkg/contexts/credentials/internal/builder_test.go @@ -21,7 +21,7 @@ var _ = Describe("builder test", func() { Expect(ctx).NotTo(BeIdenticalTo(local.DefaultContext)) Expect(ctx.RepositoryTypes()).To(BeIdenticalTo(local.DefaultRepositoryTypeScheme)) - Expect(ctx.ConfigContext()).To(BeIdenticalTo(config.DefaultContext())) + Expect(ctx.ConfigContext().GetId()).To(BeIdenticalTo(config.DefaultContext().GetId())) }) It("creates defaulted", func() { @@ -31,7 +31,7 @@ var _ = Describe("builder test", func() { Expect(ctx).NotTo(BeIdenticalTo(local.DefaultContext)) Expect(ctx.RepositoryTypes()).To(BeIdenticalTo(local.DefaultRepositoryTypeScheme)) - Expect(ctx.ConfigContext()).NotTo(BeIdenticalTo(config.DefaultContext())) + Expect(ctx.ConfigContext().GetId()).NotTo(BeIdenticalTo(config.DefaultContext().GetType())) Expect(ctx.ConfigContext().ConfigTypes()).To(BeIdenticalTo(config.DefaultContext().ConfigTypes())) }) @@ -43,7 +43,7 @@ var _ = Describe("builder test", func() { Expect(ctx.RepositoryTypes()).NotTo(BeIdenticalTo(local.DefaultRepositoryTypeScheme)) Expect(ctx.RepositoryTypes().KnownTypeNames()).To(Equal(local.DefaultRepositoryTypeScheme.KnownTypeNames())) - Expect(ctx.ConfigContext()).NotTo(BeIdenticalTo(config.DefaultContext())) + Expect(ctx.ConfigContext().GetId()).NotTo(BeIdenticalTo(config.DefaultContext().GetId())) Expect(ctx.ConfigContext().ConfigTypes()).NotTo(BeIdenticalTo(config.DefaultContext().ConfigTypes())) Expect(ctx.ConfigContext().ConfigTypes().KnownTypeNames()).To(Equal(config.DefaultContext().ConfigTypes().KnownTypeNames())) }) diff --git a/pkg/contexts/credentials/internal/context.go b/pkg/contexts/credentials/internal/context.go index e6d3e952c0..a2d1074437 100644 --- a/pkg/contexts/credentials/internal/context.go +++ b/pkg/contexts/credentials/internal/context.go @@ -14,6 +14,7 @@ import ( "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/finalizer" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) // CONTEXT_TYPE is the global type for a credential context. @@ -102,7 +103,10 @@ type _context struct { consumerProviders *consumerProviderRegistry } -var _ Context = &_context{} +var ( + _ Context = (*_context)(nil) + _ datacontext.ViewCreator[Context] = (*_context)(nil) +) // gcWrapper is used as garbage collectable // wrapper for a context implementation @@ -112,24 +116,35 @@ type gcWrapper struct { *_context } +func newView(c *_context, ref ...bool) Context { + if utils.Optional(ref...) { + return datacontext.FinalizedContext[gcWrapper](c) + } + return c +} + func (w *gcWrapper) SetContext(c *_context) { w._context = c } func newContext(configctx config.Context, reposcheme RepositoryTypeScheme, consumerMatchers IdentityMatcherRegistry, delegates datacontext.Delegates) Context { c := &_context{ - sharedattributes: configctx.AttributesContext(), + sharedattributes: datacontext.PersistentContextRef(configctx.AttributesContext()), knownRepositoryTypes: reposcheme, consumerIdentityMatchers: consumerMatchers, consumerProviders: newConsumerProviderRegistry(), } c._InternalContext = datacontext.NewContextBase(c, CONTEXT_TYPE, key, configctx.GetAttributes(), delegates) - c.updater = cfgcpi.NewUpdater(configctx, c) - return datacontext.FinalizedContext[gcWrapper](c) + c.updater = cfgcpi.NewUpdaterForFactory(datacontext.PersistentContextRef(configctx), c.CredentialsContext) + return newView(c, true) +} + +func (c *_context) CreateView() Context { + return newView(c, true) } func (c *_context) CredentialsContext() Context { - return c + return newView(c) } func (c *_context) Update() error { @@ -157,12 +172,13 @@ func (c *_context) RepositorySpecForConfig(data []byte, unmarshaler runtime.Unma } func (c *_context) RepositoryForSpec(spec RepositorySpec, creds ...CredentialsSource) (Repository, error) { - cred, err := CredentialsChain(creds).Credentials(c) + out := newView(c) + cred, err := CredentialsChain(creds).Credentials(out) if err != nil { return nil, err } c.Update() - return spec.Repository(c, cred) + return spec.Repository(out, cred) } func (c *_context) RepositoryForConfig(data []byte, unmarshaler runtime.Unmarshaler, creds ...CredentialsSource) (Repository, error) { @@ -174,7 +190,8 @@ func (c *_context) RepositoryForConfig(data []byte, unmarshaler runtime.Unmarsha } func (c *_context) CredentialsForSpec(spec CredentialsSpec, creds ...CredentialsSource) (Credentials, error) { - repospec := spec.GetRepositorySpec(c) + out := newView(c) + repospec := spec.GetRepositorySpec(out) repo, err := c.RepositoryForSpec(repospec, creds...) if err != nil { return nil, err diff --git a/pkg/contexts/credentials/repositories/dockerconfig/repo_test.go b/pkg/contexts/credentials/repositories/dockerconfig/repo_test.go index d4cd772560..7a42e69f0d 100644 --- a/pkg/contexts/credentials/repositories/dockerconfig/repo_test.go +++ b/pkg/contexts/credentials/repositories/dockerconfig/repo_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" . "github.com/open-component-model/ocm/pkg/testutils" "github.com/open-component-model/ocm/pkg/common" @@ -165,6 +166,7 @@ var _ = Describe("docker config", func() { ctx.GetType() Expect(r.Get()).To(BeNil()) + Expect(datacontext.GetContextRefCount(ctx)).To(Equal(1)) ctx = nil runtime.GC() time.Sleep(time.Second) diff --git a/pkg/contexts/credentials/repositories/dockerconfig/repository.go b/pkg/contexts/credentials/repositories/dockerconfig/repository.go index 756df304c4..79d2aea161 100644 --- a/pkg/contexts/credentials/repositories/dockerconfig/repository.go +++ b/pkg/contexts/credentials/repositories/dockerconfig/repository.go @@ -17,6 +17,7 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/finalizer" "github.com/open-component-model/ocm/pkg/utils" @@ -33,7 +34,7 @@ type Repository struct { func NewRepository(ctx cpi.Context, path string, data []byte, propagate bool) (*Repository, error) { r := &Repository{ - ctx: ctx, + ctx: datacontext.InternalContextRef(ctx), propagate: propagate, path: path, data: data, diff --git a/pkg/contexts/datacontext/attrs/rootcertsattr/config.go b/pkg/contexts/datacontext/attrs/rootcertsattr/config.go index 9fcf7d1238..d562553964 100644 --- a/pkg/contexts/datacontext/attrs/rootcertsattr/config.go +++ b/pkg/contexts/datacontext/attrs/rootcertsattr/config.go @@ -62,7 +62,7 @@ func (a *Config) AddRootCertificate(chain signutils.GenericCertificateChain) err func (a *Config) ApplyTo(ctx cfgcpi.Context, target interface{}) error { if t, ok := target.(Context); ok { - if t.AttributesContext() == t { // apply only to root context + if t.AttributesContext().IsAttributesContext() { // apply only to root context return errors.Wrapf(a.ApplyToAttribute(Get(t)), "applying config to certattr failed") } } diff --git a/pkg/contexts/datacontext/context-refcount-model.png b/pkg/contexts/datacontext/context-refcount-model.png new file mode 100755 index 0000000000..89d79e12c8 Binary files /dev/null and b/pkg/contexts/datacontext/context-refcount-model.png differ diff --git a/pkg/contexts/datacontext/context.go b/pkg/contexts/datacontext/context.go index 1cf0a7fe19..2af9fb34d9 100644 --- a/pkg/contexts/datacontext/context.go +++ b/pkg/contexts/datacontext/context.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "reflect" - runtime2 "runtime" "sync" "github.com/mandelsoft/logging" @@ -19,6 +18,7 @@ import ( "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/finalizer" ocmlog "github.com/open-component-model/ocm/pkg/logging" + "github.com/open-component-model/ocm/pkg/refmgmt" "github.com/open-component-model/ocm/pkg/runtime" "github.com/open-component-model/ocm/pkg/utils" ) @@ -47,6 +47,8 @@ const ( MODE_INITIAL ) +const MULTI_REF = false + func (m BuilderMode) String() string { switch m { case MODE_SHARED: @@ -123,6 +125,8 @@ type Context interface { ContextProvider Delegates + IsIdenticalTo(Context) bool + // GetType returns the context type GetType() string GetId() ContextIdentity @@ -137,7 +141,7 @@ type InternalContext interface { Context finalizer.RecorderProvider GetKey() interface{} - Cleanup() error + GetAllocatable() refmgmt.Allocatable } //////////////////////////////////////////////////////////////////////////////// @@ -216,40 +220,21 @@ type gcWrapper struct { *_context } -func (w *gcWrapper) SetContext(c *_context) { - w._context = c -} - -// var _ Context = gcWrapper{} - -// AttributesContext must be defined at wrapper as special -// case for a root context. -// Unfortunately GO generics do not accept this these for -// FinalizedContext anymore because of this -// pointer receiver, therefore we have to copy and specialize -// the complete stuff. -func (w *gcWrapper) AttributesContext() AttributesContext { - if w.updater != nil { - w.updater.Update() +func newView(c *_context, ref ...bool) AttributesContext { + if utils.Optional(ref...) { + return FinalizedContext[gcWrapper](c) } - return w + return c } -func finalizedContext(c *_context) AttributesContext { - var v gcWrapper - p := &v - p.SetContext(c) - p.setSelf(p, c.GetKey()) // prepare for generic bind operation - runtime2.SetFinalizer(&v, lfi) - Debug(p, "create context", "id", c.GetId()) - return p +func (w *gcWrapper) SetContext(c *_context) { + w._context = c } -func lfi(c *gcWrapper) { - err := c.Cleanup() - c.GetRecorder().Record(c.GetId()) - Debug(c, "cleanup context", "error", err) -} +var ( + _ Context = (*_context)(nil) + _ ViewCreator[AttributesContext] = (*_context)(nil) +) // New provides a root attribute context. func New(parentAttrs ...Attributes) AttributesContext { @@ -265,8 +250,18 @@ func newWithActions(mode BuilderMode, parentAttrs Attributes, actions handlers.R c.contextBase = newContextBase(c, CONTEXT_TYPE, key, parentAttrs, &c.updater, ComposeDelegates(logging.NewWithBase(ocmlog.Context()), handlers.NewRegistry(nil, actions)), ) - // return SetupContext(mode, FinalizedContext[gcWrapper](c)) // see above - return SetupContext(mode, finalizedContext(c)) + return SetupContext(mode, c.CreateView()) // see above +} + +func (c *_context) CreateView() AttributesContext { + return newView(c, true) +} + +func (c *_context) AttributesContext() AttributesContext { + if c.updater != nil { + c.updater.Update() + } + return newView(c) } func (c *_context) IsAttributesContext() bool { diff --git a/pkg/contexts/datacontext/context_test.go b/pkg/contexts/datacontext/context_test.go new file mode 100644 index 0000000000..ecb1593e1d --- /dev/null +++ b/pkg/contexts/datacontext/context_test.go @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package datacontext_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + me "github.com/open-component-model/ocm/pkg/contexts/datacontext" +) + +var _ = Describe("area test", func() { + It("can be garbage collected", func() { + // ocmlog.Context().AddRule(logging.NewConditionRule(logging.DebugLevel, me.Realm)) + + ctx := me.New() + Expect(ctx.IsIdenticalTo(ctx)).To(BeTrue()) + + ctx2 := ctx.AttributesContext() + Expect(ctx.IsIdenticalTo(ctx2)).To(BeTrue()) + + ctx3 := me.New() + Expect(ctx.IsIdenticalTo(ctx3)).To(BeFalse()) + }) +}) diff --git a/pkg/contexts/datacontext/cpi.go b/pkg/contexts/datacontext/cpi.go index 3bb83318be..4d372330ad 100644 --- a/pkg/contexts/datacontext/cpi.go +++ b/pkg/contexts/datacontext/cpi.go @@ -7,13 +7,15 @@ package datacontext import ( "context" "fmt" - runtime2 "runtime" "github.com/mandelsoft/logging" + "github.com/modern-go/reflect2" "github.com/open-component-model/ocm/pkg/contexts/datacontext/action/handlers" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/refmgmt" + "github.com/open-component-model/ocm/pkg/refmgmt/finalized" ) // NewContextBase creates a context base implementation supporting @@ -29,22 +31,38 @@ func NewContextBase(eff Context, typ string, key interface{}, parentAttrs Attrib // for a context wrapper handling garbage collection. // It also handles the BindTo interface for a context. type GCWrapper struct { - self Context + ref *finalized.FinalizedRef + self Context // reference to wrapper + ctx Context // reference to internal context key interface{} } +var _ provider = (*GCWrapper)(nil) // wrapper provides access to internal context ref + // setSelf is not public to enforce // the usage of this GCWrapper type in context // specific garbage collection wrappers. // It is enforced by the // finalizableContextWrapper interface. -func (w *GCWrapper) setSelf(self Context, key interface{}) { +func (w *GCWrapper) setSelf(a refmgmt.Allocatable, self Context, ictx Context, key interface{}) { + if a != nil { + w.ref, _ = finalized.NewPlainFinalizedView(a) + } w.self = self + w.ctx = ictx w.key = key } +func (w *GCWrapper) IsPersistent() bool { + return true +} + +func (w *GCWrapper) GetInternalContext() Context { + return w.ctx +} + func init() { // linter complains about unused method. - (&GCWrapper{}).setSelf(nil, nil) + (&GCWrapper{}).setSelf(nil, nil, nil, nil) } // BindTo makes the Context reachable via the resulting context.Context. @@ -53,6 +71,88 @@ func (b GCWrapper) BindTo(ctx context.Context) context.Context { return context.WithValue(ctx, b.key, b.self) } +func (w GCWrapper) getAllocatable() refmgmt.Allocatable { + return w.ref.GetAllocatable() +} + +type view interface { + getAllocatable() refmgmt.Allocatable +} + +type viewI interface { + GetAllocatable() refmgmt.Allocatable +} + +func GetContextRefCount(ctx Context) int { + switch a := ctx.(type) { + case view: + if m, ok := a.getAllocatable().(refmgmt.RefMgmt); ok { + return m.RefCount() + } + case viewI: + if m, ok := a.GetAllocatable().(refmgmt.RefMgmt); ok { + return m.RefCount() + } + } + return -1 +} + +type persistent interface { + IsPersistent() bool +} + +type provider interface { + GetInternalContext() Context +} + +type ViewCreator[C Context] interface { + CreateView() C +} + +func IsPersistentContextRef(ctx Context) bool { + if p, ok := ctx.(persistent); ok { + return p.IsPersistent() + } + return false +} + +// PersistentContextRef ensures a persistent context ref to the given +// context to avoid an automatic cleanup of the context, which is +// executed if all persistent refs are gone. +// If you want to keep context related objects longer than your used +// context reference, you should keep a persistent ref. This +// could be the one provided by context creation, or by retrieving +// an explicit one using this function. +func PersistentContextRef[C Context](ctx C) C { + if IsPersistentContextRef(ctx) { + return ctx + } + var c interface{} = ctx + for { + if p, ok := c.(provider); ok { + c = p.GetInternalContext() + } else { + break + } + } + return c.(ViewCreator[C]).CreateView() +} + +func InternalContextRef[C Context](ctx C) C { + if IsPersistentContextRef(ctx) { + var c interface{} = ctx + for { + if p, ok := c.(provider); ok { + c = p.GetInternalContext() + } else { + break + } + } + return c.(C) + } + return ctx +} + // finalizableContextWrapper is the interface for // a context wrapper used to establish a garbage collectable // runtime finalizer. @@ -62,7 +162,8 @@ type finalizableContextWrapper[C InternalContext, P any] interface { InternalContext SetContext(C) - setSelf(Context, interface{}) + provider + setSelf(refmgmt.Allocatable, Context, Context, interface{}) *P } @@ -81,24 +182,17 @@ func FinalizedContext[W Context, C InternalContext, P finalizableContextWrapper[ var v W p := (P)(&v) p.SetContext(c) - p.setSelf(p, c.GetKey()) // prepare for generic bind operation - runtime2.SetFinalizer(p, fi[W, C, P]) - Debug(p, "create context", "id", c.GetId()) + p.setSelf(c.GetAllocatable(), p, c, c.GetKey()) // prepare for generic bind operation return p } -func fi[W Context, C InternalContext, P finalizableContextWrapper[C, W]](c P) { - err := c.Cleanup() - c.GetRecorder().Record(c.GetId()) - Debug(c, "cleanup context", "error", err) -} - type contextBase struct { - ctxtype string - id ContextIdentity - key interface{} - effective Context - attributes *_attributes + ctxtype string + allocatable refmgmt.Allocatable + id ContextIdentity + key interface{} + effective Context + attributes *_attributes delegates finalizer *finalizer.Finalizer @@ -120,9 +214,22 @@ func newContextBase(eff Context, typ string, key interface{}, parentAttrs Attrib delegates: delegates, recorder: recorder, } + c.allocatable = refmgmt.NewAllocatable(c.cleanup, true) + Debug(c, "create context", "id", c.GetId()) return c } +func (c *contextBase) IsIdenticalTo(ctx Context) bool { + if reflect2.IsNil(ctx) { + return false + } + return c.GetId() == ctx.GetId() +} + +func (c *contextBase) GetAllocatable() refmgmt.Allocatable { + return c.allocatable +} + func (c *contextBase) BindTo(ctx context.Context) context.Context { panic("should never be called") } @@ -151,6 +258,13 @@ func (c *contextBase) GetRecorder() *finalizer.RuntimeFinalizationRecoder { return c.recorder } +func (c *contextBase) cleanup() error { + if c.recorder != nil { + c.recorder.Record(c.id) + } + return c.Cleanup() +} + func (c *contextBase) Cleanup() error { list := errors.ErrListf("cleanup %s", c.id) list.Addf(nil, c.finalizer.Finalize(), "finalizers") diff --git a/pkg/contexts/datacontext/gc_test.go b/pkg/contexts/datacontext/gc_test.go index ffae3b33c3..a3c833961f 100644 --- a/pkg/contexts/datacontext/gc_test.go +++ b/pkg/contexts/datacontext/gc_test.go @@ -10,16 +10,34 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/open-component-model/ocm/pkg/generics" me "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/finalizer" ) var _ = Describe("area test", func() { - It("can be garbage collected", func() { + It("can be garbage collectede", func() { + ctx := me.New() + r := finalizer.GetRuntimeFinalizationRecorder(ctx) + id := ctx.GetId() + Expect(me.GetContextRefCount(ctx)).To(Equal(1)) + ctx = nil + runtime.GC() + time.Sleep(time.Second) + Expect(r.Get()).To(ConsistOf(id)) + + }) + + It("provides second reference", func() { // ocmlog.Context().AddRule(logging.NewConditionRule(logging.DebugLevel, me.Realm)) + multiRefs := generics.Conditional(me.MULTI_REF, 2, 1) ctx := me.New() + Expect(me.GetContextRefCount(ctx)).To(Equal(1)) + + actx := ctx.AttributesContext() + Expect(me.GetContextRefCount(ctx)).To(Equal(multiRefs)) r := finalizer.GetRuntimeFinalizationRecorder(ctx) Expect(r).NotTo(BeNil()) @@ -29,6 +47,14 @@ var _ = Describe("area test", func() { ctx.GetType() Expect(r.Get()).To(BeNil()) + actx.GetType() + actx = nil + runtime.GC() + time.Sleep(time.Second) + ctx.GetType() + Expect(r.Get()).To(BeNil()) + Expect(me.GetContextRefCount(ctx)).To(Equal(1)) + ctx = nil for i := 0; i < 100; i++ { runtime.GC() @@ -37,4 +63,34 @@ var _ = Describe("area test", func() { Expect(r.Get()).To(ContainElement(ContainSubstring(me.CONTEXT_TYPE))) }) + + It("creates views", func() { + ctx := me.New() + r := finalizer.GetRuntimeFinalizationRecorder(ctx) + + Expect(me.GetContextRefCount(ctx)).To(Equal(1)) + Expect(me.IsPersistentContextRef(ctx)).To(BeTrue()) + + view := me.PersistentContextRef(ctx) + Expect(me.GetContextRefCount(view)).To(Equal(1)) // reuse persistent ref + Expect(me.IsPersistentContextRef(view)).To(BeTrue()) + + non := view.AttributesContext() + Expect(me.IsPersistentContextRef(non)).To(BeFalse()) + + view2 := me.PersistentContextRef(non) + Expect(me.GetContextRefCount(view2)).To(Equal(2)) // create new view + Expect(me.IsPersistentContextRef(view2)).To(BeTrue()) + + Expect(ctx.IsIdenticalTo(view)).To(BeTrue()) + Expect(ctx.IsIdenticalTo(view2)).To(BeTrue()) + + ctx = nil + view = nil + view2 = nil + + runtime.GC() + time.Sleep(time.Second) + Expect(len(r.Get())).To(Equal(1)) // ref non is not persistent + }) }) diff --git a/pkg/contexts/oci/grammar/grammar.go b/pkg/contexts/oci/grammar/grammar.go index e949574f11..09990802b6 100644 --- a/pkg/contexts/oci/grammar/grammar.go +++ b/pkg/contexts/oci/grammar/grammar.go @@ -50,14 +50,14 @@ var ( // repository name components. RepositorySeparatorRegexp = Literal(RepositorySeparator) - // alphaNumericRegexp defines the alpha numeric atom, typically a + // AlphaNumericRegexp defines the alpha numeric atom, typically a // component of names. This only allows lower case characters and digits. AlphaNumericRegexp = Match(`[a-z0-9]+`) - // separatorRegexp defines the separators allowed to be embedded in name + // SeparatorRegexp defines the separators allowed to be embedded in name // components. This allow one period, one or two underscore and multiple // dashes. - separatorRegexp = Match(`(?:[._]|__|[-]*)`) + SeparatorRegexp = Match(`(?:[._]|__|[-]*)`) // dockerOrgSeparatorRegexp defines the separators allowed to be // embedded in a docker organization name. @@ -76,7 +76,7 @@ var ( // separated by one period, one or two underscore and multiple dashes. NameComponentRegexp = Sequence( AlphaNumericRegexp, - Optional(Repeated(separatorRegexp, AlphaNumericRegexp))) + Optional(Repeated(SeparatorRegexp, AlphaNumericRegexp))) // DomainComponentRegexp restricts the registry domain component of a // repository name to start with a component as defined by DomainPortRegexp @@ -85,6 +85,10 @@ var ( IPRegexp = Sequence(Match("[0-9]+"), Literal(`.`), Match("[0-9]+"), Literal(`.`), Match("[0-9]+"), Literal(`.`), Match("[0-9]+")) + SchemeRegexp = Sequence(Capture(Match("[a-z]+")), Literal(`://`)) + + AnchoredSchemedRegexp = Anchored(Optional(SchemeRegexp), Capture(Match(".*"))) + // DomainRegexp defines the structure of potential domain components // that may be part of image names. This is purposely a subset of what is // allowed by DNS to ensure backwards compatibility with Docker image @@ -101,11 +105,48 @@ var ( DomainRegexp, Optional(Literal(`:`), Match(`[0-9]+`))) + // SchemeDomainPortRegexp defines the structure of potential domain components + // that may be part of image names. This is purposely a subset of what is + // allowed by DNS to ensure backwards compatibility with Docker image + // names followed by an optional port part. + SchemeDomainPortRegexp = Sequence( + Optional(SchemeRegexp), + Capture(DomainPortRegexp)) + // HostPortRegexp describes a non-DNS simple hostname like localhost. HostPortRegexp = Sequence( Or(DomainComponentRegexp, IPRegexp), Optional(Literal(`:`), Match(`[0-9]+`))) + // SchemedHostPortRegexp describes a non-DNS simple hostname with scheme like https://localhost. + SchemedHostPortRegexp = Sequence( + SchemeRegexp, + Capture(HostPortRegexp)) + + SchemeHostPortRegexp = Sequence( + Optional(SchemeRegexp), + Capture(ReqHostPortRegexp)) + + // SchemedHostPortArtifactRegexp describes a non-DNS simple hostname with scheme like https://localhost/repository:1.0.0 with the scheme being required. + AnchoredTypedSchemedHostPortArtifactRegexp = Anchored(Sequence( + Optional(Capture(TypeRegexp), Literal("::")), + Optional(SchemeRegexp), + Capture(Or(HostPortRegexp, DomainPortRegexp)), + Literal("/"), + Literal("/"), + CapturedArtifactVersionRegexp)) + + ReqHostPortRegexp = Sequence( + Or(DomainComponentRegexp, IPRegexp), + Literal(`:`), Match(`[0-9]+`)) + + AnchoredTypedOptSchemedReqHostReqPortArtifactRegexp = Anchored( + Optional(Capture(TypeRegexp), Literal("::")), + Optional(SchemeRegexp), + Capture(ReqHostPortRegexp), + Match(RepositorySeparator), + Optional(CapturedArtifactVersionRegexp)) + PathRegexp = Sequence( Optional(Literal("/")), Match(`[a-zA-Z0-9-_.]+(?:/[a-zA-Z0-9-_.]+)+`)) @@ -124,7 +165,7 @@ var ( // DigestRegexp matches valid digests. DigestRegexp = Match(`[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`) - // RepositoryRegexp is the format of a repository ppart of references. + // RepositoryRegexp is the format of a repository part of references. RepositoryRegexp = Sequence( NameComponentRegexp, Optional(Repeated(RepositorySeparatorRegexp, NameComponentRegexp))) diff --git a/pkg/contexts/oci/internal/builder_test.go b/pkg/contexts/oci/internal/builder_test.go index 3b4cffc107..2240f5bd48 100644 --- a/pkg/contexts/oci/internal/builder_test.go +++ b/pkg/contexts/oci/internal/builder_test.go @@ -22,7 +22,7 @@ var _ = Describe("builder test", func() { Expect(ctx).NotTo(BeIdenticalTo(local.DefaultContext)) Expect(ctx.RepositoryTypes()).To(BeIdenticalTo(local.DefaultRepositoryTypeScheme)) - Expect(ctx.ConfigContext()).To(BeIdenticalTo(config.DefaultContext())) + Expect(ctx.ConfigContext().GetId()).To(BeIdenticalTo(config.DefaultContext().GetId())) Expect(ctx.CredentialsContext()).To(BeIdenticalTo(credentials.DefaultContext())) }) diff --git a/pkg/contexts/oci/internal/context.go b/pkg/contexts/oci/internal/context.go index da67e49200..2fd458db76 100644 --- a/pkg/contexts/oci/internal/context.go +++ b/pkg/contexts/oci/internal/context.go @@ -14,6 +14,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) const CONTEXT_TYPE = "oci" + datacontext.OCM_CONTEXT_SUFFIX @@ -37,6 +38,7 @@ type Context interface { RepositoryForSpec(spec RepositorySpec, creds ...credentials.CredentialsSource) (Repository, error) RepositoryForConfig(data []byte, unmarshaler runtime.Unmarshaler, creds ...credentials.CredentialsSource) (Repository, error) + RepositorySpecForConfig(data []byte, unmarshaler runtime.Unmarshaler) (RepositorySpec, error) GetAlias(name string) RepositorySpec SetAlias(name string, spec RepositorySpec) @@ -77,15 +79,17 @@ type _context struct { _InternalContext updater cfgcpi.Updater - sharedattributes datacontext.AttributesContext - credentials credentials.Context + credentials credentials.Context knownRepositoryTypes RepositoryTypeScheme specHandlers RepositorySpecHandlers aliases map[string]RepositorySpec } -var _ Context = &_context{} +var ( + _ Context = (*_context)(nil) + _ datacontext.ViewCreator[Context] = (*_context)(nil) +) // gcWrapper is used as garbage collectable // wrapper for a context implementation @@ -95,25 +99,35 @@ type gcWrapper struct { *_context } +func newView(c *_context, ref ...bool) Context { + if utils.Optional(ref...) { + return datacontext.FinalizedContext[gcWrapper](c) + } + return c +} + func (w *gcWrapper) SetContext(c *_context) { w._context = c } func newContext(credctx credentials.Context, reposcheme RepositoryTypeScheme, specHandlers RepositorySpecHandlers, delegates datacontext.Delegates) Context { c := &_context{ - sharedattributes: credctx.AttributesContext(), - credentials: credctx, + credentials: datacontext.PersistentContextRef(credctx), knownRepositoryTypes: reposcheme, specHandlers: specHandlers, aliases: map[string]RepositorySpec{}, } c._InternalContext = datacontext.NewContextBase(c, CONTEXT_TYPE, key, credctx.ConfigContext().GetAttributes(), delegates) - c.updater = cfgcpi.NewUpdater(credctx.ConfigContext(), c) - return datacontext.FinalizedContext[gcWrapper](c) + c.updater = cfgcpi.NewUpdaterForFactory(credctx.ConfigContext(), c.OCIContext) + return newView(c, true) +} + +func (c *_context) CreateView() Context { + return newView(c, true) } func (c *_context) OCIContext() Context { - return c + return newView(c) } func (c *_context) Update() error { @@ -121,7 +135,7 @@ func (c *_context) Update() error { } func (c *_context) AttributesContext() datacontext.AttributesContext { - return c.sharedattributes + return c.credentials.AttributesContext() } func (c *_context) ConfigContext() config.Context { @@ -141,7 +155,7 @@ func (c *_context) RepositorySpecHandlers() RepositorySpecHandlers { } func (c *_context) MapUniformRepositorySpec(u *UniformRepositorySpec) (RepositorySpec, error) { - return c.specHandlers.MapUniformRepositorySpec(c, u) + return c.specHandlers.MapUniformRepositorySpec(c.OCIContext(), u) } func (c *_context) RepositorySpecForConfig(data []byte, unmarshaler runtime.Unmarshaler) (RepositorySpec, error) { @@ -153,7 +167,7 @@ func (c *_context) RepositoryForSpec(spec RepositorySpec, creds ...credentials.C if err != nil { return nil, err } - return spec.Repository(c, cred) + return spec.Repository(c.OCIContext(), cred) } func (c *_context) RepositoryForConfig(data []byte, unmarshaler runtime.Unmarshaler, creds ...credentials.CredentialsSource) (Repository, error) { diff --git a/pkg/contexts/oci/internal/uniform.go b/pkg/contexts/oci/internal/uniform.go index 24a8f39ad8..ad4a204bb5 100644 --- a/pkg/contexts/oci/internal/uniform.go +++ b/pkg/contexts/oci/internal/uniform.go @@ -11,9 +11,11 @@ import ( "strings" "sync" + "github.com/containerd/containerd/reference" "github.com/sirupsen/logrus" "golang.org/x/exp/slices" + "github.com/open-component-model/ocm/pkg/contexts/oci/grammar" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/runtime" ) @@ -84,21 +86,34 @@ func (u *UniformRepositorySpec) RepositoryRef() string { return fmt.Sprintf("%s%s://%s", t, u.Scheme, u.Host) } +func (u *UniformRepositorySpec) SetType(typ string) { + t, _ := grammar.SplitTypeSpec(typ) + u.Type = t + u.TypeHint = typ +} + func (u *UniformRepositorySpec) String() string { return u.RepositoryRef() } func UniformRepositorySpecForHostURL(typ string, host string) *UniformRepositorySpec { - scheme := "" - parsed, err := url.Parse(host) + s := "" + h := host + var parsed *url.URL + ref, err := reference.Parse(host) + if err == nil { + parsed, err = url.Parse("https://" + ref.Locator) + } else { + parsed, err = url.Parse(host) + } if err == nil { - host = parsed.Host - scheme = parsed.Scheme + s = parsed.Scheme + h = parsed.Host } u := &UniformRepositorySpec{ Type: typ, - Scheme: scheme, - Host: host, + Scheme: s, + Host: h, } return u } @@ -168,6 +183,14 @@ func (s *specHandlers) MapUniformRepositorySpec(ctx Context, u *UniformRepositor defer s.lock.RUnlock() deferr := errors.ErrNotSupported("uniform repository ref", u.String()) + if u.Info != "" && string(u.Info[0]) == "{" && u.Host == "" && u.Scheme == "" { + data, err := runtime.CompleteSpecWithType(u.Type, []byte(u.Info)) + if err != nil { + return nil, err + } + return ctx.RepositorySpecForConfig(data, runtime.DefaultJSONEncoding) + } + if u.Type == "" { if u.Info != "" { spec := ctx.GetAlias(u.Info) diff --git a/pkg/contexts/oci/ref.go b/pkg/contexts/oci/ref.go index 24500d5faf..764058884a 100644 --- a/pkg/contexts/oci/ref.go +++ b/pkg/contexts/oci/ref.go @@ -23,37 +23,30 @@ const ( KIND_ARETEFACT_REFERENCE = "artifact reference" ) -// ParseRepo parses a standard oci repository reference into a internal representation. +// ParseRepo parses a standard oci repository reference into an internal representation. func ParseRepo(ref string) (UniformRepositorySpec, error) { create := false if strings.HasPrefix(ref, "+") { create = true ref = ref[1:] } + uspec := UniformRepositorySpec{} match := grammar.AnchoredRegistryRegexp.FindSubmatch([]byte(ref)) if match == nil { match = grammar.AnchoredGenericRegistryRegexp.FindSubmatch([]byte(ref)) if match == nil { - return UniformRepositorySpec{}, errors.ErrInvalid(KIND_OCI_REFERENCE, ref) + return uspec, errors.ErrInvalid(KIND_OCI_REFERENCE, ref) } - h := string(match[1]) - t, _ := grammar.SplitTypeSpec(h) - return UniformRepositorySpec{ - Type: t, - TypeHint: h, - Info: string(match[2]), - CreateIfMissing: create, - }, nil + uspec.SetType(string(match[1])) + uspec.Info = string(match[2]) + uspec.CreateIfMissing = create + return uspec, nil } - h := string(match[1]) - t, _ := grammar.SplitTypeSpec(h) - return UniformRepositorySpec{ - Type: t, - TypeHint: h, - Scheme: string(match[2]), - Host: string(match[3]), - CreateIfMissing: create, - }, nil + uspec.SetType(string(match[1])) + uspec.Scheme = string(match[2]) + uspec.Host = string(match[3]) + uspec.CreateIfMissing = create + return uspec, nil } // RefSpec is a go internal representation of an oci reference. @@ -87,10 +80,30 @@ func ParseRef(ref string) (RefSpec, error) { } spec := RefSpec{UniformRepositorySpec: UniformRepositorySpec{CreateIfMissing: create}} + match := grammar.AnchoredTypedSchemedHostPortArtifactRegexp.FindSubmatch([]byte(ref)) + if match != nil { + spec.SetType(string(match[1])) + spec.Scheme = string(match[2]) + spec.Host = string(match[3]) + spec.Repository = string(match[4]) + spec.Tag = pointer(match[5]) + spec.Digest = dig(match[6]) + return spec, nil + } - match := grammar.FileReferenceRegexp.FindSubmatch([]byte(ref)) + match = grammar.AnchoredTypedOptSchemedReqHostReqPortArtifactRegexp.FindSubmatch([]byte(ref)) + if match != nil { + spec.SetType(string(match[1])) + spec.Scheme = string(match[2]) + spec.Host = string(match[3]) + spec.Repository = string(match[4]) + spec.Tag = pointer(match[5]) + spec.Digest = dig(match[6]) + return spec, nil + } + match = grammar.FileReferenceRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Info = string(match[2]) spec.Repository = string(match[3]) spec.Tag = pointer(match[4]) @@ -124,7 +137,7 @@ func ParseRef(ref string) (RefSpec, error) { } match = grammar.TypedReferenceRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Scheme = string(match[2]) spec.Host = string(match[3]) spec.Repository = string(match[4]) @@ -134,7 +147,7 @@ func ParseRef(ref string) (RefSpec, error) { } match = grammar.TypedURIRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Scheme = string(match[2]) spec.Host = string(match[3]) spec.Repository = string(match[4]) @@ -144,7 +157,7 @@ func ParseRef(ref string) (RefSpec, error) { } match = grammar.TypedGenericReferenceRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Info = string(match[2]) spec.Repository = string(match[3]) spec.Tag = pointer(match[4]) @@ -153,7 +166,7 @@ func ParseRef(ref string) (RefSpec, error) { } match = grammar.AnchoredRegistryRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Info = string(match[2]) spec.Repository = string(match[3]) spec.Tag = pointer(match[4]) @@ -163,7 +176,7 @@ func ParseRef(ref string) (RefSpec, error) { match = grammar.AnchoredGenericRegistryRegexp.FindSubmatch([]byte(ref)) if match != nil { - spec.Type = string(match[1]) + spec.SetType(string(match[1])) spec.Info = string(match[2]) match = grammar.ErrorCheckRegexp.FindSubmatch([]byte(ref)) diff --git a/pkg/contexts/oci/ref_test.go b/pkg/contexts/oci/ref_test.go index 3dc52f38b2..b0c8d1977f 100644 --- a/pkg/contexts/oci/ref_test.go +++ b/pkg/contexts/oci/ref_test.go @@ -5,14 +5,85 @@ package oci_test import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/testutils" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ctf" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ocireg" + "github.com/open-component-model/ocm/pkg/runtime" ) +func Type(t string) string { + if t == "" { + return t + } + return t + "::" +} +func FileFormat(t, f string) string { + if t == "" { + return f + } + if f == "" { + return t + } + return t + "+" + f +} +func FileType(t, f string) string { + if t != "" { + return t + } else { + return f + } +} +func Scheme(s string) string { + if s == "" { + return s + } + return s + "://" +} +func Sub(t string) string { + if t == "" { + return t + } + return "/" + t +} +func Vers(t, d string) string { + if t == "" && d == "" { + return "" + } + if t == "" { + return "@" + d + } + if d == "" { + return ":" + t + } + return ":" + t + "@" + d +} + +func Dig(b []byte) *godigest.Digest { + if len(b) == 0 { + return nil + } + s := godigest.Digest(b) + return &s +} + +func Pointer(b []byte) *string { + if len(b) == 0 { + return nil + } + s := string(b) + return &s +} + func CheckRef(ref string, exp *oci.RefSpec) { spec, err := oci.ParseRef(ref) if exp == nil { @@ -34,12 +105,460 @@ func CheckRepo(ref string, exp *oci.UniformRepositorySpec) { } var _ = Describe("ref parsing", func() { - digest := digest.Digest("sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a") + digest := godigest.Digest("sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a") tag := "v1" ghcr := oci.UniformRepositorySpec{Host: "ghcr.io"} docker := oci.UniformRepositorySpec{Host: "docker.io"} + Context("parse file path refs", func() { + t := "ctf" + p := "file/path" + r := "github.com/mandelsoft/ocm" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + Context("[+][::][./][//[:]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, up := range []string{p, "./" + p} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := cm + Type(FileFormat(ut, uf)) + up + "//" + r + Vers(uv, ud) + ut, uf, uv, up, ud := ut, uf, uv, up, ud + + // tests parsing of all permutations of + // [+][::][./][//[:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: "", + Host: "", + Info: up, + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + } + } + } + } + }) + }) + + Context("parse domain refs", func() { + t := "oci" + h := "ghcr.io" + r := "github.com/mandelsoft/ocm" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + // Notice that the file formats (directory, tar, tgz) CAN BE PARSED in this notation, BUT for non file based + // implementations like oci, this information is not used. + Context("[+][::][://][:][/]/[:][@]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, ush := range []string{"", "http", "https"} { + for _, uh := range []string{h, h + ":3030"} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + for _, sep := range []string{"/", "//"} { + ref := cm + Type(FileFormat(ut, uf)) + Scheme(ush) + uh + sep + r + Vers(uv, ud) + ut, uf, ush, uh, uv, ud := ut, uf, ush, uh, uv, ud + + // tests parsing of all permutations of + // [::][://][:][/]/[:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: ush, + Host: uh, + Info: "", + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + } + } + } + } + } + } + }) + + It("repository creation from parsed repo", func() { + ctx := oci.New() + aliasreg := ocireg.NewRepositorySpec("http://ghcr.io") + ctx.SetAlias("myalias", aliasreg) + repo := Must(oci.ParseRef("myalias//repository:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&repo.UniformRepositorySpec)) + Expect(spec).To(Equal(aliasreg)) + }) + }) + + Context("parse host port refs", func() { + t := "oci" + h := "localhost" + r := "github.com/mandelsoft/ocm" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + // localhost (with and without port) (and other host names) are a special case since these are not formally + // valid domains + // the combination of this test and the test below test parsing of all permutations of + // [::][://]:/[:][@] + Context("[+][::][://]:/[:][@]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, ush := range []string{"", "http", "https"} { + for _, uh := range []string{h + ":3030"} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := cm + Type(FileFormat(ut, uf)) + Scheme(ush) + uh + "/" + r + Vers(uv, ud) + ut, uf, ush, uh, uv, ud := ut, uf, ush, uh, uv, ud + + // tests parsing of all permutations of + // [::][://]:/[:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: ush, + Host: uh, + Info: "", + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + } + } + } + } + } + }) + Context("[+][::][://][:]//[:][@]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, ush := range []string{"", "http", "https"} { + for _, uh := range []string{h, h + ":3030"} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := cm + Type(FileFormat(ut, uf)) + Scheme(ush) + uh + "//" + r + Vers(uv, ud) + ut, uf, ush, uh, uv, ud := ut, uf, ush, uh, uv, ud + + // tests parsing of all permutations of + // [::][://][:]//[:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: ush, + Host: uh, + Info: "", + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + } + } + } + } + } + }) + }) + + Context("parse json repo spec refs", func() { + t := "oci" + h := "ghcr.io" + r := "github.com/mandelsoft/ocm" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + repospec := ocireg.NewRepositorySpec(h) + jsonrepospec := string(Must(runtime.DefaultJSONEncoding.Marshal(repospec))) + + // Notice that the file formats (directory, tar, tgz) CAN BE PARSED in this notation, BUT for non file based + // implementations like oci, this information is not used. + Context("[+][::][//][:][@]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := cm + Type(FileFormat(ut, uf)) + jsonrepospec + "//" + r + Vers(uv, ud) + ut, uf, uv, ud := ut, uf, uv, ud + + // tests parsing of all permutations of + // [::][//][:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: "", + Host: "", + Info: jsonrepospec, + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + } + } + } + }) + }) + + Context("parse docker library refs", func() { + // h := "docker.io" + r := "ubuntu" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + Context("[:][@]", func() { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := r + Vers(uv, ud) + uv, ud := uv, ud + + // tests parsing of all permutations of + // [:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: "", + Scheme: "", + Host: "docker.io", + Info: "", + CreateIfMissing: false, + TypeHint: "", + }, + ArtSpec: oci.ArtSpec{ + Repository: "library/" + r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + }) + }) + + Context("parse docker repository refs", func() { + // h := "docker.io" + r := "docker-repo/ubuntu" + v := "v1" + d := "sha256:3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a" + + Context("/[:][@]", func() { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + ".1.2-rc.1"} { + for _, ud := range []string{"", d} { + ref := r + Vers(uv, ud) + uv, ud := uv, ud + + // tests parsing of all permutations of + // [:][@] + It("parses ref "+ref, func() { + CheckRef(ref, &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: "", + Scheme: "", + Host: "docker.io", + Info: "", + CreateIfMissing: false, + TypeHint: "", + }, + ArtSpec: oci.ArtSpec{ + Repository: r, + Tag: Pointer([]byte(uv)), + Digest: Dig([]byte(ud)), + }, + }) + }) + } + } + }) + }) + + Context("parse file path repos", func() { + t := "ctf" + p := "file/path" + + Context("[+][::][./][", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, up := range []string{p, "./" + p} { + ref := cm + Type(FileFormat(ut, uf)) + up + ut, uf, up := ut, uf, up + // tests parsing of all permutations of + // [+][::][./][//[:][@] + It("parses ref "+ref, func() { + CheckRepo(ref, &oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: "", + Host: "", + Info: up, + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }) + }) + } + } + } + } + }) + }) + + Context("parse domain repos", func() { + t := "oci" + h := "ghcr.io" + + Context("[+][::][://][:]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, ush := range []string{"", "http", "https"} { + for _, uh := range []string{h, h + ":3030", "localhost", "localhost:3030"} { + ref := cm + Type(FileFormat(ut, uf)) + Scheme(ush) + uh + ut, uf, ush, uh := ut, uf, ush, uh + + // tests parsing of all permutations of + // [+][::][://][:] + It("parses ref "+ref, func() { + // if you are coming from the ocm test and + // wondering why the corresponding tests if + // has an additional condition that the + // type has to be empty - this is because + // the corresponding parse method calls + // an intermediate handler based on the + // type that resolves the localhost in the + // info. + // For oci repositories, such this + // handling is done in the + // MapUniformRepositorySpec logic. + if strings.HasPrefix(uh, "localhost") { + CheckRepo(ref, &oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: "", + Host: "", + Info: Scheme(ush) + uh, + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }) + } else { + CheckRepo(ref, &oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: ush, + Host: uh, + Info: "", + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }) + } + }) + } + } + } + } + } + }) + It("repository creation from parsed repo with localhost", func() { + ctx := oci.New() + repo := Must(oci.ParseRepo("http://localhost")) + spec := Must(ctx.MapUniformRepositorySpec(&repo)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("http://localhost"))) + }) + It("repository creation from parsed repo with localhost", func() { + ctx := oci.New() + + aliasreg := ocireg.NewRepositorySpec("http://ghcr.io") + ctx.SetAlias("myalias", aliasreg) + repo := Must(oci.ParseRepo("myalias")) + spec := Must(ctx.MapUniformRepositorySpec(&repo)) + Expect(spec).To(Equal(aliasreg)) + }) + }) + + Context("parse json repo spec refs", func() { + t := "oci" + h := "ghcr.io" + + repospec := ocireg.NewRepositorySpec(h) + jsonrepospec := string(Must(runtime.DefaultJSONEncoding.Marshal(repospec))) + + // Notice that the file formats (directory, tar, tgz) CAN BE PARSED in this notation, BUT for non file based + // implementations like oci, this information is not used. + Context("[+][::]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + ref := cm + Type(FileFormat(ut, uf)) + jsonrepospec + ut, uf := ut, uf + + // tests parsing of all permutations of + // [::] + It("parses ref "+ref, func() { + CheckRepo(ref, &oci.UniformRepositorySpec{ + Type: FileType(ut, uf), + Scheme: "", + Host: "", + Info: jsonrepospec, + CreateIfMissing: ref[0] == '+', + TypeHint: FileFormat(ut, uf), + }) + }) + } + } + } + }) + }) + It("succeeds for repository", func() { CheckRef("::ghcr.io/", &oci.RefSpec{UniformRepositorySpec: ghcr}) }) @@ -67,10 +586,11 @@ var _ = Describe("ref parsing", func() { }) CheckRef("type::https://ghcr.io/repo/repo:v1@"+digest.String(), &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "type", - Scheme: "https", - Host: "ghcr.io", - Info: "", + Type: "type", + Scheme: "https", + Host: "ghcr.io", + Info: "", + TypeHint: "type", }, ArtSpec: oci.ArtSpec{ Repository: "repo/repo", @@ -93,10 +613,11 @@ var _ = Describe("ref parsing", func() { }) CheckRef("directory::a/b", &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "directory", - Scheme: "", - Host: "", - Info: "a/b", + Type: "directory", + Scheme: "", + Host: "", + Info: "a/b", + TypeHint: "directory", }, ArtSpec: oci.ArtSpec{ Repository: "", @@ -104,10 +625,11 @@ var _ = Describe("ref parsing", func() { }) CheckRef("ctf+directory::a/b", &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "ctf+directory", - Scheme: "", - Host: "", - Info: "a/b", + Type: "ctf", + Scheme: "", + Host: "", + Info: "a/b", + TypeHint: "ctf+directory", }, ArtSpec: oci.ArtSpec{ Repository: "", @@ -115,11 +637,12 @@ var _ = Describe("ref parsing", func() { }) CheckRef("+ctf+directory::a/b", &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "ctf+directory", + Type: "ctf", Scheme: "", Host: "", Info: "a/b", CreateIfMissing: true, + TypeHint: "ctf+directory", }, ArtSpec: oci.ArtSpec{ Repository: "", @@ -140,10 +663,11 @@ var _ = Describe("ref parsing", func() { CheckRef("directory::a/b//c/d", &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "directory", - Scheme: "", - Host: "", - Info: "a/b", + Type: "directory", + Scheme: "", + Host: "", + Info: "a/b", + TypeHint: "directory", }, ArtSpec: oci.ArtSpec{ Repository: "c/d", @@ -152,10 +676,11 @@ var _ = Describe("ref parsing", func() { CheckRef("oci::ghcr.io", &oci.RefSpec{ UniformRepositorySpec: oci.UniformRepositorySpec{ - Type: "oci", - Scheme: "", - Host: "ghcr.io", - Info: "", + Type: "oci", + Scheme: "", + Host: "ghcr.io", + Info: "", + TypeHint: "oci", }, ArtSpec: oci.ArtSpec{ Repository: "", @@ -184,6 +709,52 @@ var _ = Describe("ref parsing", func() { }) + It("json spec", func() { + ctx := oci.New() + + tag := "1.0.0" + CheckRef("OCIRegistry::{\"baseUrl\": \"test.com\"}//repo:1.0.0", &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: "OCIRegistry", + Scheme: "", + Host: "", + Info: "{\"baseUrl\": \"test.com\"}", + TypeHint: "OCIRegistry", + }, + ArtSpec: oci.ArtSpec{ + Repository: "repo", + Tag: &tag, + }, + }) + ref := Must(oci.ParseRef("OCIRegistry::{\"type\":\"OCIRegistry\", \"baseUrl\": \"test.com\"}//repo:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + repo := Must(spec.Repository(ctx, nil)) + _ = repo + }) + + It("fail for json spec with type mismatch", func() { + ctx := oci.New() + + tag := "1.0.0" + CheckRef("oci::{\"type\":\"OCIRegistry\", \"baseUrl\": \"test.com\"}//repo:1.0.0", &oci.RefSpec{ + UniformRepositorySpec: oci.UniformRepositorySpec{ + Type: "oci", + Scheme: "", + Host: "", + Info: "{\"type\":\"OCIRegistry\", \"baseUrl\": \"test.com\"}", + TypeHint: "oci", + }, + ArtSpec: oci.ArtSpec{ + Repository: "repo", + Tag: &tag, + }, + }) + ref := Must(oci.ParseRef("oci::{\"type\":\"OCIRegistry\", \"baseUrl\": \"test.com\"}//repo:1.0.0")) + spec, err := ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec) + Expect(spec).To(BeNil()) + Expect(err).ToNot(BeNil()) + }) + It("fails", func() { CheckRef("https://ubuntu", nil) CheckRef("ubuntu@4711", nil) @@ -212,5 +783,49 @@ var _ = Describe("ref parsing", func() { Info: "a/b.tar", }) }) + It("localhost", func() { + ctx := oci.New() + // port is necessary here, otherwise it is ambiguous with dockerhub reference (localhost/test:1.0.0 could be + // an artifact stored on duckerhub) + ref := Must(oci.ParseRef("localhost:80/test:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("localhost:80"))) + }) + It("localhost with unambiguous separator and without port", func() { + ctx := oci.New() + ref := Must(oci.ParseRef("localhost//test:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("localhost"))) + }) + It("localhost with unambiguous separator", func() { + ctx := oci.New() + ref := Must(oci.ParseRef("localhost:80//test:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("localhost:80"))) + }) + It("scheme://localhost:port//repository:version", func() { + ctx := oci.New() + ref := Must(oci.ParseRef("http://localhost:80//test:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("http://localhost:80"))) + }) + It("scheme://localhost:port/repository:version", func() { + ctx := oci.New() + ref := Must(oci.ParseRef("http://localhost:80/test:1.0.0")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(ocireg.NewRepositorySpec("http://localhost:80"))) + }) + It("ctf with create", func() { + ctx := oci.New() + ref := Must(oci.ParseRef("+ctf+directory::./file/path//github.com/mandelsoft/ocm")) + spec := Must(ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec)) + Expect(spec).To(Equal(Must(ctf.NewRepositorySpec(accessobj.ACC_CREATE, "./file/path", accessio.FormatDirectory)))) + }) + It("ctf without create", func() { + ctx := oci.New() + ref := Must(oci.ParseRepo("ctf+directory::./file/path")) + spec := Must(ctx.MapUniformRepositorySpec(&ref)) + Expect(spec).To(Equal(Must(ctf.NewRepositorySpec(accessobj.ACC_WRITABLE, "./file/path")))) + }) }) diff --git a/pkg/contexts/oci/repositories/ctf/uniform.go b/pkg/contexts/oci/repositories/ctf/uniform.go index 2ecc3e462c..2b668addb9 100644 --- a/pkg/contexts/oci/repositories/ctf/uniform.go +++ b/pkg/contexts/oci/repositories/ctf/uniform.go @@ -17,6 +17,7 @@ func init() { h := &repospechandler{} cpi.RegisterRepositorySpecHandler(h, "") cpi.RegisterRepositorySpecHandler(h, Type) + cpi.RegisterRepositorySpecHandler(h, AltType) for _, f := range SupportedFormats() { cpi.RegisterRepositorySpecHandler(h, string(f)) } diff --git a/pkg/contexts/oci/repositories/ocireg/repository.go b/pkg/contexts/oci/repositories/ocireg/repository.go index 376e7851f2..ed474c4ffa 100644 --- a/pkg/contexts/oci/repositories/ocireg/repository.go +++ b/pkg/contexts/oci/repositories/ocireg/repository.go @@ -62,6 +62,9 @@ var ( func NewRepository(ctx cpi.Context, spec *RepositorySpec, info *RepositoryInfo) (cpi.Repository, error) { urs := spec.UniformRepositorySpec() + if urs.Scheme == "http" { + ocmlog.Logger(REALM).Warn("using insecure http for oci registry {{host}}", "host", urs.Host) + } i := &RepositoryImpl{ RepositoryImplBase: cpi.NewRepositoryImplBase(ctx), logger: logging.DynamicLogger(ctx, REALM, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), @@ -145,23 +148,28 @@ func (r *RepositoryImpl) getResolver(comp string) (resolve.Resolver, error) { }, DefaultScheme: r.info.Scheme, //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. - DefaultTLS: &tls.Config{ - // MinVersion: tls.VersionTLS13, - RootCAs: func() *x509.CertPool { - var rootCAs *x509.CertPool - if creds != nil { - c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) - if c != "" { - rootCAs = x509.NewCertPool() - rootCAs.AppendCertsFromPEM([]byte(c)) + DefaultTLS: func() *tls.Config { + if r.info.Scheme == "http" { + return nil + } + return &tls.Config{ + // MinVersion: tls.VersionTLS13, + RootCAs: func() *x509.CertPool { + var rootCAs *x509.CertPool + if creds != nil { + c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) + if c != "" { + rootCAs = x509.NewCertPool() + rootCAs.AppendCertsFromPEM([]byte(c)) + } } - } - if rootCAs == nil { - rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) - } - return rootCAs - }(), - }, + if rootCAs == nil { + rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) + } + return rootCAs + }(), + } + }(), })), } diff --git a/pkg/contexts/oci/repositories/ocireg/uniform.go b/pkg/contexts/oci/repositories/ocireg/uniform.go index 4cf9233581..761a772a60 100644 --- a/pkg/contexts/oci/repositories/ocireg/uniform.go +++ b/pkg/contexts/oci/repositories/ocireg/uniform.go @@ -6,6 +6,8 @@ package ocireg import ( "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" + "github.com/open-component-model/ocm/pkg/contexts/oci/grammar" + "github.com/open-component-model/ocm/pkg/regex" ) func init() { @@ -15,9 +17,24 @@ func init() { type repospechandler struct{} func (h *repospechandler) MapReference(ctx cpi.Context, u *cpi.UniformRepositorySpec) (cpi.RepositorySpec, error) { - if u.Info != "" || u.Host == "" { + scheme := u.Scheme + host := u.Host + if u.Host == "" && u.Scheme == "" && u.Info != "" { + host = u.Info + match := grammar.AnchoredSchemedRegexp.FindStringSubmatch(host) + if match != nil { + scheme = match[1] + host = match[2] + } + if !(regex.Anchored(grammar.HostPortRegexp).MatchString(host) || regex.Anchored(grammar.DomainPortRegexp).MatchString(host)) { + return nil, nil + } + } else if u.Info != "" || u.Host == "" { return nil, nil } - return NewRepositorySpec(u.Host), nil + if scheme != "" { + host = scheme + "://" + host + } + return NewRepositorySpec(host), nil } diff --git a/pkg/contexts/ocm/attrs/signingattr/config.go b/pkg/contexts/ocm/attrs/signingattr/config.go index e531e5cb5b..8eff58ab8b 100644 --- a/pkg/contexts/ocm/attrs/signingattr/config.go +++ b/pkg/contexts/ocm/attrs/signingattr/config.go @@ -210,10 +210,7 @@ func (a *Config) ApplyTo(ctx cfgcpi.Context, target interface{}) error { t, ok := target.(Context) if !ok { if t, ok := target.(datacontext.AttributesContext); ok { - // datacontext.Context is implemented by all context types. - // Therefore, we have to check for the root context, this is the one - // identical to the attributes context of a context. - if t.AttributesContext() == t { + if t.AttributesContext().IsAttributesContext() { return errors.Wrapf(a.ApplyToRootCertsAttr(rootcertsattr.Get(t)), "applying config to certattr failed") } } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo/blobhandler.go index ca073b5987..6dd9bc9641 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo/blobhandler.go @@ -61,6 +61,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g return nil, err } + // this section is for logging, only target, err := json.Marshal(repo.GetSpecification()) if err != nil { return nil, errors.Wrapf(err, "cannot marshal target specification") diff --git a/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo/blobhandler.go index 14bf533573..5ce47f72e7 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo/blobhandler.go @@ -17,6 +17,7 @@ import ( "github.com/open-component-model/ocm/pkg/common/accessobj" "github.com/open-component-model/ocm/pkg/contexts/oci" "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" + "github.com/open-component-model/ocm/pkg/contexts/oci/grammar" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ocireg" "github.com/open-component-model/ocm/pkg/contexts/oci/transfer" @@ -329,8 +330,16 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if err != nil { return nil, wrap(err, errhint, "transfer artifact") } - - ref := path.Join(base, namespace.GetNamespace()) + version + match := grammar.AnchoredSchemedRegexp.FindStringSubmatch(base) + scheme := "" + if match != nil { + scheme = match[1] + base = match[2] + } + if scheme != "" { + scheme += "://" + } + ref := scheme + path.Join(base, namespace.GetNamespace()) + version return ociartifact.New(ref), nil } diff --git a/pkg/contexts/ocm/grammar/grammar.go b/pkg/contexts/ocm/grammar/grammar.go index 96aa3ef385..d84ac83cf8 100644 --- a/pkg/contexts/ocm/grammar/grammar.go +++ b/pkg/contexts/ocm/grammar/grammar.go @@ -30,7 +30,19 @@ var ( // AnchoredRepositoryRegexp parses a uniform repository spec. AnchoredRepositoryRegexp = Anchored( Optional(Capture(TypeRegexp), Literal("::")), - Capture(grammar.DomainPortRegexp), Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + grammar.SchemeDomainPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + ) + + // AnchoredSchemedHostPortRepositoryRegexp parses a uniform repository spec. + AnchoredSchemedHostPortRepositoryRegexp = Anchored( + Optional(Capture(TypeRegexp), Literal("::")), + grammar.SchemedHostPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + ) + + // AnchoredHostWithPortRepositoryRegexp parses a uniform repository spec. + AnchoredHostWithPortRepositoryRegexp = Anchored( + Optional(Capture(TypeRegexp), Literal("::")), + grammar.SchemeHostPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), ) // AnchoredGenericRepositoryRegexp describes a CTF reference. @@ -54,7 +66,29 @@ var ( // It provides 5 captures: type, repository host port, sub path, component and version. AnchoredReferenceRegexp = Anchored( Optional(Capture(TypeRegexp), Literal("::")), - Capture(grammar.DomainPortRegexp), Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + grammar.SchemeDomainPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + Literal("//"), Capture(ComponentRegexp), + Optional(Literal(VersionSeparator), Capture(VersionRegexp)), + ) + + // AnchoredSchemedHostPortReferenceRegexp parses a complete string representation for default component references + // including the repository part. Since the type is optional, the scheme is required to allow for a distinction + // from filepaths. + // It provides 6 captures: type, scheme, repository host port, sub path, component and version. + AnchoredSchemedHostPortReferenceRegexp = Anchored( + Optional(Capture(TypeRegexp), Literal("::")), + grammar.SchemedHostPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), + Literal("//"), Capture(ComponentRegexp), + Optional(Literal(VersionSeparator), Capture(VersionRegexp)), + ) + + // AnchoredHostWithPortReferenceRegexp parses a complete string representation for default component references + // including the repository part. Since the type is optional, the scheme is required to allow for a distinction + // from filepaths. + // It provides 6 captures: type, scheme, repository host port, sub path, component and version. + AnchoredHostWithPortReferenceRegexp = Anchored( + Optional(Capture(TypeRegexp), Literal("::")), + grammar.SchemeHostPortRegexp, Optional(grammar.RepositorySeparatorRegexp, Capture(grammar.RepositoryRegexp)), Literal("//"), Capture(ComponentRegexp), Optional(Literal(VersionSeparator), Capture(VersionRegexp)), ) diff --git a/pkg/contexts/ocm/grammar/grammar_test.go b/pkg/contexts/ocm/grammar/grammar_test.go index 507b38ed04..1878e60256 100644 --- a/pkg/contexts/ocm/grammar/grammar_test.go +++ b/pkg/contexts/ocm/grammar/grammar_test.go @@ -50,6 +50,12 @@ func Type(t string) string { } return t + "::" } +func Scheme(sc string) string { + if sc == "" { + return sc + } + return sc + "://" +} func Sub(t string) string { if t == "" { return t @@ -90,6 +96,7 @@ var _ = Describe("ref matching", func() { Context("complete refs", func() { t := "OCIRepository" + sc := "http" s := "mandelsoft/cnudie" v := "v1" @@ -98,10 +105,12 @@ var _ = Describe("ref matching", func() { It("succeeds", func() { for _, ut := range []string{"", t} { - for _, us := range []string{"", s} { - for _, uv := range []string{"", v} { - ref := Type(ut) + h + Sub(us) + "//" + c + Vers(uv) - CheckRef(ref, ut, h, us, c, uv) + for _, usc := range []string{"", sc} { + for _, us := range []string{"", s} { + for _, uv := range []string{"", v} { + ref := Type(ut) + Scheme(usc) + h + Sub(us) + "//" + c + Vers(uv) + CheckRef(ref, ut, usc, h, us, c, uv) + } } } } @@ -129,10 +138,10 @@ var _ = Describe("ref matching", func() { Context("repo", func() { It("succeeds", func() { - Check("directory::ghcr.io/sub/path", AnchoredRepositoryRegexp, "directory", "ghcr.io", "sub/path") - Check("ghcr.io/sub/path", AnchoredRepositoryRegexp, "", "ghcr.io", "sub/path") - Check("ghcr.io", AnchoredRepositoryRegexp, "", "ghcr.io", "") - Check("ghcr.io/sub/path", AnchoredRepositoryRegexp, "", "ghcr.io", "sub/path") + Check("directory::ghcr.io/sub/path", AnchoredRepositoryRegexp, "directory", "", "ghcr.io", "sub/path") + Check("ghcr.io/sub/path", AnchoredRepositoryRegexp, "", "", "ghcr.io", "sub/path") + Check("ghcr.io", AnchoredRepositoryRegexp, "", "", "ghcr.io", "") + Check("ghcr.io/sub/path", AnchoredRepositoryRegexp, "", "", "ghcr.io", "sub/path") }) It("fails", func() { Check("/ghcr.io/sub/path", AnchoredRepositoryRegexp) diff --git a/pkg/contexts/ocm/internal/builder_test.go b/pkg/contexts/ocm/internal/builder_test.go index 5350affe3c..98b133c2c2 100644 --- a/pkg/contexts/ocm/internal/builder_test.go +++ b/pkg/contexts/ocm/internal/builder_test.go @@ -29,11 +29,9 @@ var _ = Describe("builder test", func() { Expect(ctx.BlobHandlers()).To(BeIdenticalTo(local.DefaultBlobHandlerRegistry)) Expect(ctx.BlobDigesters()).To(BeIdenticalTo(local.DefaultBlobDigesterRegistry)) - Expect(ctx.ConfigContext()).To(BeIdenticalTo(config.DefaultContext())) - - Expect(ctx.CredentialsContext()).To(BeIdenticalTo(credentials.DefaultContext())) - - Expect(ctx.OCIContext()).To(BeIdenticalTo(oci.DefaultContext())) + Expect(ctx.ConfigContext().GetId()).To(BeIdenticalTo(config.DefaultContext().GetId())) + Expect(ctx.CredentialsContext().GetId()).To(BeIdenticalTo(credentials.DefaultContext().GetId())) + Expect(ctx.OCIContext().GetId()).To(BeIdenticalTo(oci.DefaultContext().GetId())) }) It("creates defaulted", func() { diff --git a/pkg/contexts/ocm/internal/context.go b/pkg/contexts/ocm/internal/context.go index 5e395afad2..f09de688b8 100644 --- a/pkg/contexts/ocm/internal/context.go +++ b/pkg/contexts/ocm/internal/context.go @@ -21,6 +21,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ctf" "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) const CONTEXT_TYPE = "ocm" + datacontext.OCM_CONTEXT_SUFFIX @@ -127,9 +128,8 @@ type _context struct { _InternalContext updater cfgcpi.Updater - sharedattributes datacontext.AttributesContext - credctx credentials.Context - ocictx oci.Context + credctx credentials.Context + ocictx oci.Context knownRepositoryTypes RepositoryTypeScheme knownAccessTypes AccessTypeScheme @@ -141,7 +141,10 @@ type _context struct { resolver *resolver } -var _ Context = &_context{} +var ( + _ Context = (*_context)(nil) + _ datacontext.ViewCreator[Context] = (*_context)(nil) +) // gcWrapper is used as garbage collectable // wrapper for a context implementation @@ -151,15 +154,21 @@ type gcWrapper struct { *_context } +func newView(c *_context, ref ...bool) Context { + if utils.Optional(ref...) { + return datacontext.FinalizedContext[gcWrapper](c) + } + return c +} + func (w *gcWrapper) SetContext(c *_context) { w._context = c } func newContext(credctx credentials.Context, ocictx oci.Context, reposcheme RepositoryTypeScheme, accessscheme AccessTypeScheme, specHandlers RepositorySpecHandlers, blobHandlers BlobHandlerRegistry, blobDigesters BlobDigesterRegistry, repodel RepositoryDelegationRegistry, delegates datacontext.Delegates) Context { c := &_context{ - sharedattributes: credctx.AttributesContext(), - credctx: credctx, - ocictx: ocictx, + credctx: datacontext.PersistentContextRef(credctx), + ocictx: datacontext.PersistentContextRef(ocictx), specHandlers: specHandlers, blobHandlers: blobHandlers, blobDigesters: blobDigesters, @@ -172,17 +181,21 @@ func newContext(credctx credentials.Context, ocictx oci.Context, reposcheme Repo c.knownRepositoryTypes = NewRepositoryTypeScheme(&delegatingDecoder{ctx: c, delegate: repodel}, reposcheme) } c._InternalContext = datacontext.NewContextBase(c, CONTEXT_TYPE, key, credctx.GetAttributes(), delegates) - c.updater = cfgcpi.NewUpdater(credctx.ConfigContext(), c) + c.updater = cfgcpi.NewUpdaterForFactory(credctx.ConfigContext(), c.OCMContext) c.resolver = &resolver{ ctx: c, MatchingResolver: NewMatchingResolver(c), } c.Finalizer().With(c.resolver.Finalize) - return datacontext.FinalizedContext[gcWrapper](c) + return newView(c, true) +} + +func (c *_context) CreateView() Context { + return newView(c, true) } func (c *_context) OCMContext() Context { - return c + return newView(c) } func (c *_context) Update() error { @@ -190,7 +203,7 @@ func (c *_context) Update() error { } func (c *_context) AttributesContext() datacontext.AttributesContext { - return c.sharedattributes + return c.credctx.AttributesContext() } func (c *_context) ConfigContext() config.Context { diff --git a/pkg/contexts/ocm/internal/resolver.go b/pkg/contexts/ocm/internal/resolver.go index ad32729535..85665b2855 100644 --- a/pkg/contexts/ocm/internal/resolver.go +++ b/pkg/contexts/ocm/internal/resolver.go @@ -55,6 +55,13 @@ func NewRepositoryCache() *RepositoryCache { } } +func (c *RepositoryCache) Reset() { + c.lock.Lock() + defer c.lock.Unlock() + + c.repositories = map[datacontext.ObjectKey]Repository{} +} + func (c *RepositoryCache) LookupRepository(ctx Context, spec RepositorySpec) (Repository, bool, error) { spec, err := ctx.RepositoryTypes().Convert(spec) if err != nil { @@ -104,9 +111,17 @@ func (r *ResolverRule) Match(name string) bool { return r.prefix == "" || r.prefix == name || strings.HasPrefix(name, r.prefix+"/") } +// MatchingResolver hosts rule to match component version names. +// Matched names will be mapped to a specification for repository +// which should be used to look up the component version. +// Therefore, it keeps a reference to the context to use. +// +// ATTENTION: Because such an object is used by the context +// implementation, the context must be kept as ContextProvider +// to provide context views to outbound calls. type MatchingResolver struct { lock sync.Mutex - ctx Context + ctx ContextProvider finalize finalizer.Finalizer cache *RepositoryCache rules []*ResolverRule @@ -115,17 +130,20 @@ type MatchingResolver struct { func NewMatchingResolver(ctx ContextProvider, rules ...*ResolverRule) *MatchingResolver { return &MatchingResolver{ lock: sync.Mutex{}, - ctx: ctx.OCMContext(), + ctx: ctx, cache: NewRepositoryCache(), rules: nil, } } func (r *MatchingResolver) OCMContext() Context { - return r.ctx + return r.ctx.OCMContext() } func (r *MatchingResolver) Finalize() error { + r.lock.Lock() + defer r.lock.Unlock() + defer r.cache.Reset() return r.finalize.Finalize() } @@ -154,9 +172,10 @@ func (r *MatchingResolver) LookupComponentVersion(name string, version string) ( r.lock.Lock() defer r.lock.Unlock() + ctx := r.ctx.OCMContext() for _, rule := range r.rules { if rule.Match(name) { - repo, cached, err := r.cache.LookupRepository(r.ctx, rule.spec) + repo, cached, err := r.cache.LookupRepository(ctx, rule.spec) if err != nil { return nil, err } diff --git a/pkg/contexts/ocm/internal/uniform.go b/pkg/contexts/ocm/internal/uniform.go index 6460d11284..91bcbbd3b2 100644 --- a/pkg/contexts/ocm/internal/uniform.go +++ b/pkg/contexts/ocm/internal/uniform.go @@ -27,6 +27,8 @@ const ( type UniformRepositorySpec struct { // Type Type string `json:"type,omitempty"` + // Scheme + Scheme string `json:"scheme,omitempty"` // Host is the hostname of an ocm ref. Host string `json:"host,omitempty"` // SubPath is the sub path spec used to host component versions @@ -36,7 +38,7 @@ type UniformRepositorySpec struct { // CreateIfMissing indicates whether a file based or dynamic repo should be created if it does not exist CreateIfMissing bool `json:"createIfMissing,omitempty"` - // TypeHintshould be set if CreateIfMissing is true to help to decide what kind of repo to create + // TypeHint should be set if CreateIfMissing is true to help to decide what kind of repo to create TypeHint string `json:"typeHint,omitempty"` } @@ -151,6 +153,14 @@ func (s *specHandlers) MapUniformRepositorySpec(ctx Context, u *UniformRepositor s.lock.RLock() defer s.lock.RUnlock() + if u.Info != "" && string(u.Info[0]) == "{" && u.Host == "" && u.Scheme == "" && u.SubPath == "" { + data, err := runtime.CompleteSpecWithType(u.Type, []byte(u.Info)) + if err != nil { + return nil, err + } + return ctx.RepositorySpecForConfig(data, runtime.DefaultJSONEncoding) + } + deferr := errors.ErrNotSupported("uniform repository ref", u.String()) if u.Type == "" { if u.Info != "" { diff --git a/pkg/contexts/ocm/ref.go b/pkg/contexts/ocm/ref.go index 366027188f..c630a621a0 100644 --- a/pkg/contexts/ocm/ref.go +++ b/pkg/contexts/ocm/ref.go @@ -33,27 +33,57 @@ func ParseRepo(ref string) (UniformRepositorySpec, error) { }) } match := grammar.AnchoredRepositoryRegexp.FindSubmatch([]byte(ref)) - if match == nil { - match = grammar.AnchoredGenericRepositoryRegexp.FindSubmatch([]byte(ref)) - if match == nil { - return UniformRepositorySpec{}, errors.ErrInvalid(KIND_OCM_REFERENCE, ref) - } + if match != nil { + h := string(match[1]) + t, _ := grammar.SplitTypeSpec(h) + return cpi.HandleRef(UniformRepositorySpec{ + Type: t, + TypeHint: h, + Scheme: string(match[2]), + Host: string(match[3]), + SubPath: string(match[4]), + CreateIfMissing: create, + }) + } + + match = grammar.AnchoredSchemedHostPortRepositoryRegexp.FindSubmatch([]byte(ref)) + if match != nil { + h := string(match[1]) + t, _ := grammar.SplitTypeSpec(h) + return cpi.HandleRef(UniformRepositorySpec{ + Type: t, + TypeHint: h, + Scheme: string(match[2]), + Host: string(match[3]), + SubPath: string(match[4]), + CreateIfMissing: create, + }) + } + + match = grammar.AnchoredHostWithPortRepositoryRegexp.FindSubmatch([]byte(ref)) + if match != nil { h := string(match[1]) t, _ := grammar.SplitTypeSpec(h) return cpi.HandleRef(UniformRepositorySpec{ Type: t, TypeHint: h, - Info: string(match[2]), + Scheme: string(match[2]), + Host: string(match[3]), + SubPath: string(match[4]), CreateIfMissing: create, }) } + + match = grammar.AnchoredGenericRepositoryRegexp.FindSubmatch([]byte(ref)) + if match == nil { + return UniformRepositorySpec{}, errors.ErrInvalid(KIND_OCM_REFERENCE, ref) + } h := string(match[1]) t, _ := grammar.SplitTypeSpec(h) return cpi.HandleRef(UniformRepositorySpec{ Type: t, TypeHint: h, - Host: string(match[2]), - SubPath: string(match[3]), + Info: string(match[2]), CreateIfMissing: create, }) } @@ -90,49 +120,104 @@ func ParseRef(ref string) (RefSpec, error) { var spec RefSpec v := "" match := grammar.AnchoredReferenceRegexp.FindSubmatch([]byte(ref)) - if match == nil { - match = grammar.AnchoredGenericReferenceRegexp.FindSubmatch([]byte(ref)) - if match == nil { - return RefSpec{}, errors.ErrInvalid(KIND_OCM_REFERENCE, ref) - } - v = string(match[4]) + if match != nil { + v = string(match[6]) + s := string(match[2]) h := string(match[1]) t, _ := grammar.SplitTypeSpec(h) spec = RefSpec{ UniformRepositorySpec{ Type: t, TypeHint: h, - Info: string(match[2]), + Scheme: s, + Host: string(match[3]), + SubPath: string(match[4]), CreateIfMissing: create, }, CompSpec{ - Component: string(match[3]), + Component: string(match[5]), Version: nil, }, } - } else { - v = string(match[5]) + } + + if match == nil { + match = grammar.AnchoredSchemedHostPortReferenceRegexp.FindSubmatch([]byte(ref)) + if match != nil { + v = string(match[6]) + s := string(match[2]) + h := string(match[1]) + t, _ := grammar.SplitTypeSpec(h) + spec = RefSpec{ + UniformRepositorySpec{ + Type: t, + TypeHint: h, + Scheme: s, + Host: string(match[3]), + SubPath: string(match[4]), + CreateIfMissing: create, + }, + CompSpec{ + Component: string(match[5]), + Version: nil, + }, + } + } + } + + if match == nil { + match = grammar.AnchoredHostWithPortReferenceRegexp.FindSubmatch([]byte(ref)) + if match != nil { + v = string(match[6]) + s := string(match[2]) + h := string(match[1]) + t, _ := grammar.SplitTypeSpec(h) + spec = RefSpec{ + UniformRepositorySpec{ + Type: t, + TypeHint: h, + Scheme: s, + Host: string(match[3]), + SubPath: string(match[4]), + CreateIfMissing: create, + }, + CompSpec{ + Component: string(match[5]), + Version: nil, + }, + } + } + } + + if match == nil { + match = grammar.AnchoredGenericReferenceRegexp.FindSubmatch([]byte(ref)) + if match == nil { + return RefSpec{}, errors.ErrInvalid(KIND_OCM_REFERENCE, ref) + } + v = string(match[4]) h := string(match[1]) t, _ := grammar.SplitTypeSpec(h) spec = RefSpec{ UniformRepositorySpec{ Type: t, TypeHint: h, - Host: string(match[2]), - SubPath: string(match[3]), + Info: string(match[2]), CreateIfMissing: create, }, CompSpec{ - Component: string(match[4]), + Component: string(match[3]), Version: nil, }, } } + if v != "" { spec.Version = &v } var err error - spec.UniformRepositorySpec, err = cpi.HandleRef(spec.UniformRepositorySpec) + if spec.Info == "" || !(string(spec.Info[0]) == "{") { + spec.UniformRepositorySpec, err = cpi.HandleRef(spec.UniformRepositorySpec) + } return spec, err } diff --git a/pkg/contexts/ocm/ref_test.go b/pkg/contexts/ocm/ref_test.go index 4fad348779..3501bc4905 100644 --- a/pkg/contexts/ocm/ref_test.go +++ b/pkg/contexts/ocm/ref_test.go @@ -5,8 +5,6 @@ package ocm_test import ( - "strings" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/open-component-model/ocm/pkg/testutils" @@ -22,6 +20,28 @@ func Type(t string) string { } return t + "::" } +func FileFormat(t, f string) string { + if t == "" { + return f + } + if f == "" { + return t + } + return t + "+" + f +} +func FileType(t, f string) string { + if t != "" { + return t + } else { + return f + } +} +func Scheme(s string) string { + if s == "" { + return s + } + return s + "://" +} func Sub(t string) string { if t == "" { return t @@ -35,7 +55,7 @@ func Vers(t string) string { return ":" + t } -func CheckRef(ref, ut, h, us, c, uv, i string, th ...string) { +func CheckRef(ref, ut, scheme, h, us, c, uv, i string, th ...string) { var v *string if uv != "" { v = &uv @@ -48,6 +68,7 @@ func CheckRef(ref, ut, h, us, c, uv, i string, th ...string) { Expect(spec).WithOffset(1).To(Equal(ocm.RefSpec{ UniformRepositorySpec: ocm.UniformRepositorySpec{ Type: ut, + Scheme: scheme, Host: h, SubPath: us, Info: i, @@ -62,7 +83,80 @@ func CheckRef(ref, ut, h, us, c, uv, i string, th ...string) { } var _ = Describe("ref parsing", func() { - Context("complete refs", func() { + Context("file path refs", func() { + t := "ctf" + p := "file/path" + c := "github.com/mandelsoft/ocm" + v := "v1" + + Context("[+][::][./][//[:]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{"", t} { + for _, uf := range []string{"", "directory", "tar", "tgz"} { + for _, up := range []string{p, "./" + p} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + "+65", v + ".1.2-rc.1", v + ".1.2+65"} { + ref := cm + Type(FileFormat(ut, uf)) + up + "//" + c + Vers(uv) + ut, uf, uv, up := ut, uf, uv, up + + // tests parsing of all permutations of + // [+][::][./]//[:] + It("parses ref "+ref, func() { + if ut != "" || uf != "" { + CheckRef(ref, FileType(ut, uf), "", "", "", c, uv, up, FileFormat(ut, uf)) + } else { + CheckRef(ref, FileType(ut, uf), "", "", "", c, uv, up) + } + }) + } + } + } + } + } + }) + }) + + Context("json repo spec refs", func() { + t := ocireg.Type + s := "mandelsoft/cnudie" + v := "v1" + + h := "ghcr.io" + c := "github.com/mandelsoft/ocm" + + repospec := ocireg.NewRepositorySpec(h, &ocireg.ComponentRepositoryMeta{ + ComponentNameMapping: "", + SubPath: s, + }) + jsonrepospec := string(Must(repospec.MarshalJSON())) + + Context("[::][//][:]", func() { + for _, cm := range []string{"", "+"} { + for _, ut := range []string{t, ""} { + for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + "+65", v + ".1.2-rc.1", v + ".1.2+65"} { + ref := cm + Type(ut) + jsonrepospec + "//" + c + Vers(uv) + ut, uv := ut, uv + + // tests parsing of all permutations of + // [::][//][:] + It("parses ref "+ref, func() { + CheckRef(ref, ut, "", "", "", c, uv, jsonrepospec) + }) + } + } + } + }) + + It("fail if mismatch between type in ref (here, ctf) and type in json repo spec (here, OCIRegistry)", func() { + ctx := ocm.New() + + ref := Must(ocm.ParseRef("ctf::{\"baseUrl\":\"ghcr.io\",\"subPath\":\"mandelsoft/cnudie\",\"type\":\"OCIRegistry\"}//github.com/mandelsoft/ocm:v1")) + spec, err := ctx.MapUniformRepositorySpec(&ref.UniformRepositorySpec) + Expect(spec).To(BeNil()) + Expect(err).ToNot(BeNil()) + }) + }) + + Context("domain refs", func() { t := ocireg.Type s := "mandelsoft/cnudie" v := "v1" @@ -70,48 +164,110 @@ var _ = Describe("ref parsing", func() { h := "ghcr.io" c := "github.com/mandelsoft/ocm" - Context("without info", func() { - for _, ut := range []string{t, ""} { - for _, uh := range []string{h, h + ":3030", "localhost", "localhost:3030"} { - for _, us := range []string{"", s} { - for _, uv := range []string{"", v, v + ".1.1", v + "-rc.1", v + "+65", v + ".1.2-rc.1", v + ".1.2+65"} { - ref := Type(ut) + uh + Sub(us) + "//" + c + Vers(uv) - ut, uh, us, uv := ut, uh, us, uv - - It("parses ref "+ref, func() { - if ut == "" && strings.HasPrefix(uh, "localhost") { - CheckRef(ref, ut, "", "", c, uv, uh+Sub(us)) - } else { - CheckRef(ref, ut, uh, us, c, uv, "") + Context("[+][::][://][:][/]//[:::][scheme://][:][/]//[:::]://[:][/]//[:::]scheme://[:][/]//[:::][://]:[/]//[:::][scheme://]:[/]//[: 0 { - host = u.Info[:idx] - subp = u.Info[idx+1:] + host = info[:idx] + subp = info[idx+1:] } else { - host = u.Info + host = info } if grammar.HostPortRegexp.MatchString(host) || grammar.DomainPortRegexp.MatchString(host) { + u.Scheme = scheme u.Host = host u.SubPath = subp u.Info = "" diff --git a/pkg/contexts/ocm/repositories/virtual/repository.go b/pkg/contexts/ocm/repositories/virtual/repository.go index fe3cc2eaa0..dafbe5457d 100644 --- a/pkg/contexts/ocm/repositories/virtual/repository.go +++ b/pkg/contexts/ocm/repositories/virtual/repository.go @@ -5,6 +5,7 @@ package virtual import ( + "github.com/open-component-model/ocm/pkg/contexts/datacontext" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi/repocpi" ) @@ -18,9 +19,9 @@ type RepositoryImpl struct { var _ repocpi.RepositoryImpl = (*RepositoryImpl)(nil) -func NewRepository(ctx cpi.Context, acc Access) cpi.Repository { +func NewRepository(ctxp cpi.ContextProvider, acc Access) cpi.Repository { impl := &RepositoryImpl{ - ctx: ctx, + ctx: datacontext.InternalContextRef(ctxp.OCMContext()), access: acc, } return repocpi.NewRepository(impl, "OCM repo[Simple]") diff --git a/pkg/finalizer/object.go b/pkg/finalizer/object.go index 0cd95cfc46..d092914916 100644 --- a/pkg/finalizer/object.go +++ b/pkg/finalizer/object.go @@ -94,7 +94,7 @@ func fi(o *RuntimeFinalizer) { o.finalize() } -func NewRuntimeFinalizer(id ObjectIdentity, r *RuntimeFinalizationRecoder) *RuntimeFinalizer { +func NewRuntimeFinalizer(id ObjectIdentity, r *RuntimeFinalizationRecoder, cleanup ...func() error) *RuntimeFinalizer { f := &RuntimeFinalizer{ id: id, recorder: r, diff --git a/pkg/refmgmt/finalized/doc.go b/pkg/refmgmt/finalized/doc.go new file mode 100644 index 0000000000..13b1e31d54 --- /dev/null +++ b/pkg/refmgmt/finalized/doc.go @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +// Package finalized provided a view management for a backend object, +// which is based on Go Garbage Collection and runtime finalizers. +// Finalization is not possible in Go, if an object is involved in +// a reference cycle. In such a case the complete cycle is not +// garbage collected at all. +// +// If some kind of finalization is required together with cyclic +// object dependencies, the cleanup of the object can therefore not +// be done with runtime finalizers. +// We separate a garbage collectable view object, which holds a +// reference to a backend object featuring cycles. +// The view object uses reference counting for its backend +// together with runtime finalization. Therefore, it does not require +// a Close method. If the view is garbage collected it releases its +// reference to the backend object. If the last view vanished +// the cleanup method for the backend object is called. +// +// The object functionality is exposed via an interface, only, which +// is also implemented by the vies by embedding a pointer to the backend +// object. +// +// If the backend object requires a cycle by holding local objects +// requiring a reference to the object, this can be done +// by NOT using view objects for this cycle, but the backend object +// itself. If the local object wants to pass the backend object to some +// outgoing call, it MUST wrap the backend object again into a new view. +// Therefore, objects involved in the cycle MUST be prepared to handle +// such outgoing references. +package finalized diff --git a/pkg/refmgmt/finalized/finalized_test.go b/pkg/refmgmt/finalized/finalized_test.go new file mode 100644 index 0000000000..09d9dce5a9 --- /dev/null +++ b/pkg/refmgmt/finalized/finalized_test.go @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package finalized_test + +import ( + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/refmgmt" + "github.com/open-component-model/ocm/pkg/refmgmt/finalized" +) + +type Interface interface { + GetId() finalizer.ObjectIdentity + GetSelf() Interface + + GetRefId() finalizer.ObjectIdentity +} + +type object struct { + refmgmt.Allocatable + recorder *finalizer.RuntimeFinalizationRecoder + name finalizer.ObjectIdentity +} + +func (o *object) GetId() finalizer.ObjectIdentity { + return o.name +} + +func (o *object) GetSelf() Interface { + v, _ := newView(o) + return v +} + +func (o *object) cleanup() error { + o.recorder.Record(finalizer.ObjectIdentity(o.name)) + return nil +} + +type view struct { + ref *finalized.FinalizedRef + *object +} + +var _ Interface = (*view)(nil) + +func newView(o *object) (Interface, error) { + ref, err := finalized.NewFinalizedView(o.Allocatable, finalizer.NewObjectIdentity("test"), o.recorder) + if err != nil { + return nil, err + } + return &view{ + ref: ref, + object: o, + }, nil +} + +func (v *view) GetRefId() finalizer.ObjectIdentity { + return v.ref.GetRefId() +} + +func New(name string, rec *finalizer.RuntimeFinalizationRecoder) Interface { + o := &object{ + name: finalizer.ObjectIdentity(name), + recorder: rec, + } + o.Allocatable = refmgmt.NewAllocatable(o.cleanup, true) + + v, _ := newView(o) + return v +} + +//////////////////////////////////////////////////////////////////////////////// + +var _ = Describe("finalized ref", func() { + var rec *finalizer.RuntimeFinalizationRecoder + + BeforeEach(func() { + rec = &finalizer.RuntimeFinalizationRecoder{} + }) + + It("cleanup ref and object", func() { + o := New("test", rec) + orefid := o.GetRefId() + id := o.GetId() + + o = nil + + runtime.GC() + time.Sleep(time.Second) + + Expect(rec.Get()).To(ConsistOf( + id, + orefid, + )) + }) + + It("cleanup ref after ref and then object", func() { + o := New("test", rec) + o2 := o + orefoid := o.GetRefId() + Expect(o2.GetRefId()).To(Equal(orefoid)) + + id := o.GetId() + r := o.GetSelf() + rrefid := r.GetRefId() + + Expect(r.GetId()).To(Equal(id)) + Expect(orefoid).NotTo(Equal(rrefid)) + + o.GetId() + o = nil + runtime.GC() + time.Sleep(time.Second) + + Expect(rec.Get()).To(ConsistOf()) + + r.GetId() + r = nil + runtime.GC() + time.Sleep(time.Second) + + Expect(rec.Get()).To(ConsistOf( + rrefid, + )) + + o2.GetId() + o2 = nil + runtime.GC() + time.Sleep(time.Second) + + Expect(rec.Get()).To(ContainElements( + orefoid, + rrefid, + id, + )) + + }) + +}) diff --git a/pkg/refmgmt/finalized/finalizedref.go b/pkg/refmgmt/finalized/finalizedref.go new file mode 100644 index 0000000000..c6262fb157 --- /dev/null +++ b/pkg/refmgmt/finalized/finalizedref.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package finalized + +import ( + "runtime" + + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/refmgmt" +) + +type FinalizedRef struct { + allocatable refmgmt.Allocatable + id finalizer.ObjectIdentity + recorder *finalizer.RuntimeFinalizationRecoder +} + +func NewPlainFinalizedView(allocatable refmgmt.Allocatable) (*FinalizedRef, error) { + return NewFinalizedView(allocatable, "", nil) +} + +func NewFinalizedView(allocatable refmgmt.Allocatable, id finalizer.ObjectIdentity, rec *finalizer.RuntimeFinalizationRecoder) (*FinalizedRef, error) { + err := allocatable.Ref() + if err != nil { + return nil, err + } + v := &FinalizedRef{allocatable, id, rec} + + runtime.SetFinalizer(v, cleanup) + return v, nil +} + +func (v *FinalizedRef) GetAllocatable() refmgmt.Allocatable { + return v.allocatable +} + +func (v *FinalizedRef) GetRefId() finalizer.ObjectIdentity { + return v.id +} + +func cleanup(v *FinalizedRef) { + v.allocatable.Unref() + if v.recorder != nil { + v.recorder.Record(v.id) + } +} diff --git a/pkg/refmgmt/finalized/suite_test.go b/pkg/refmgmt/finalized/suite_test.go new file mode 100644 index 0000000000..46c70a1d28 --- /dev/null +++ b/pkg/refmgmt/finalized/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package finalized_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Finalized Object Test Suite") +} diff --git a/pkg/runtime/utils.go b/pkg/runtime/utils.go index 2f8ec1a6d5..cbd1360537 100644 --- a/pkg/runtime/utils.go +++ b/pkg/runtime/utils.go @@ -122,3 +122,21 @@ func CheckSpecification(data []byte) error { } // --- end check --- + +func CompleteSpecWithType(typ string, data []byte) ([]byte, error) { + var m map[string]interface{} + err := DefaultJSONEncoding.Unmarshal(data, &m) + if err != nil { + return nil, err + } + if typ != "" { + if m["type"] != nil && m["type"] != typ { + return nil, fmt.Errorf("type mismatch between type in reference \"%s\" and type in json spec \"%s\"", typ, m["type"]) + } + m["type"] = typ + return DefaultJSONEncoding.Marshal(m) + } else if m["type"] == nil { + return nil, fmt.Errorf("type missing") + } + return data, nil +}