From 2e7a689f23c92a28733a02cc3d6b85fa91db03a3 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Fri, 3 Nov 2023 14:48:49 +0200 Subject: [PATCH] feat(sync): use regclient for sync extension replaced containers/image package with regclient/regclient package Signed-off-by: Petu Eusebiu --- go.mod | 1 - go.sum | 8 +- pkg/api/controller.go | 1 - pkg/api/routes.go | 4 +- pkg/extensions/extension_sync.go | 13 +- .../constants/{consts.go => constants.go} | 0 ...ntent_internal_test.go => content_test.go} | 2 +- pkg/extensions/sync/destination.go | 140 +++--- pkg/extensions/sync/features/features.go | 53 --- pkg/extensions/sync/httpclient/cache.go | 58 --- .../sync/httpclient/client_internal_test.go | 167 ------- pkg/extensions/sync/oci_layout.go | 39 +- pkg/extensions/sync/on_demand.go | 92 +--- pkg/extensions/sync/references/oci.go | 237 ---------- pkg/extensions/sync/references/references.go | 234 ---------- .../references/references_internal_test.go | 306 ------------ .../sync/references/referrers_tag.go | 172 ------- pkg/extensions/sync/referrers.go | 48 ++ pkg/extensions/sync/remote.go | 362 +++++++++++---- pkg/extensions/sync/service.go | 437 +++++++++--------- pkg/extensions/sync/sync.go | 44 +- pkg/extensions/sync/sync_internal_test.go | 362 +-------------- pkg/extensions/sync/sync_test.go | 162 ++----- pkg/extensions/sync/utils.go | 277 +++-------- pkg/test/auth/bearer.go | 26 +- 25 files changed, 806 insertions(+), 2439 deletions(-) rename pkg/extensions/sync/constants/{consts.go => constants.go} (100%) rename pkg/extensions/sync/{content_internal_test.go => content_test.go} (99%) delete mode 100644 pkg/extensions/sync/features/features.go delete mode 100644 pkg/extensions/sync/httpclient/cache.go delete mode 100644 pkg/extensions/sync/httpclient/client_internal_test.go delete mode 100644 pkg/extensions/sync/references/oci.go delete mode 100644 pkg/extensions/sync/references/references.go delete mode 100644 pkg/extensions/sync/references/references_internal_test.go delete mode 100644 pkg/extensions/sync/references/referrers_tag.go create mode 100644 pkg/extensions/sync/referrers.go diff --git a/go.mod b/go.mod index 20cb3f535..d9d118e7e 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,6 @@ require ( github.com/ProtonMail/go-crypto v1.1.0-alpha.5-proton // indirect github.com/ThalesIgnite/crypto11 v1.2.5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect diff --git a/go.sum b/go.sum index 8d1b662aa..af1a9ab7d 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,6 @@ github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= @@ -1209,6 +1207,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olareg/olareg v0.1.0 h1:1dXBOgPrig5N7zoXyIZVQqU0QBo6sD9pbL6UYjY75CA= +github.com/olareg/olareg v0.1.0/go.mod h1:RBuU7JW7SoIIxZKzLRhq8sVtQeAHzCAtRrXEBx2KlM4= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1273,8 +1273,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= -github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a h1:525aNEKSyDcqJcawiGtA2NPNApJMta8bUe9SoYuhQ+o= github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a/go.mod h1:ltIE6ZO/czh/g4xdNQlFGkl7DAfaLLFYmitB4taA5ys= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1452,8 +1450,6 @@ github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= diff --git a/pkg/api/controller.go b/pkg/api/controller.go index d9446ba75..60a1e709b 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -520,5 +520,4 @@ func (c *Controller) StartBackgroundTasks() { type SyncOnDemand interface { SyncImage(ctx context.Context, repo, reference string) error - SyncReference(ctx context.Context, repo string, subjectDigestStr string, referenceType string) error } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 97774058f..3e2f743e0 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -41,7 +41,6 @@ import ( "zotregistry.dev/zot/pkg/debug/pprof" debug "zotregistry.dev/zot/pkg/debug/swagger" ext "zotregistry.dev/zot/pkg/extensions" - syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta" mTypes "zotregistry.dev/zot/pkg/meta/types" @@ -568,8 +567,7 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler, routeHandler.c.Log.Info().Str("repository", name).Str("reference", digest.String()). Msg("referrers not found, trying to get reference by syncing on demand") - if errSync := routeHandler.c.SyncOnDemand.SyncReference(ctx, name, digest.String(), - syncConstants.OCI); errSync != nil { + if errSync := routeHandler.c.SyncOnDemand.SyncImage(ctx, name, digest.String()); errSync != nil { routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", digest.String()). Msg("failed to sync OCI reference for image") } diff --git a/pkg/extensions/extension_sync.go b/pkg/extensions/extension_sync.go index 3e96902d3..b3b17da97 100644 --- a/pkg/extensions/extension_sync.go +++ b/pkg/extensions/extension_sync.go @@ -102,17 +102,15 @@ func getLocalIPs() ([]string, error) { return localIPs, nil } -func getIPFromHostName(host string) ([]string, error) { +func getIPFromHostName(host string) ([]net.IP, error) { addrs, err := net.LookupIP(host) if err != nil { - return []string{}, err + return []net.IP{}, err } - ips := make([]string, 0, len(addrs)) + ips := make([]net.IP, 0, len(addrs)) - for _, ip := range addrs { - ips = append(ips, ip.String()) - } + ips = append(ips, addrs...) return ips, nil } @@ -163,7 +161,8 @@ func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConf for _, localIP := range localIPs { // if ip resolved from hostname/dns is equal with any local ip for _, ip := range ips { - if net.JoinHostPort(ip, url.Port()) == net.JoinHostPort(localIP, port) { + if (ip.IsLoopback() && (url.Port() == port)) || + (net.JoinHostPort(ip.String(), url.Port()) == net.JoinHostPort(localIP, port)) { registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...) removed = true diff --git a/pkg/extensions/sync/constants/consts.go b/pkg/extensions/sync/constants/constants.go similarity index 100% rename from pkg/extensions/sync/constants/consts.go rename to pkg/extensions/sync/constants/constants.go diff --git a/pkg/extensions/sync/content_internal_test.go b/pkg/extensions/sync/content_test.go similarity index 99% rename from pkg/extensions/sync/content_internal_test.go rename to pkg/extensions/sync/content_test.go index 78b03b6c6..864c3f5dc 100644 --- a/pkg/extensions/sync/content_internal_test.go +++ b/pkg/extensions/sync/content_test.go @@ -1,7 +1,7 @@ //go:build sync // +build sync -package sync +package sync //nolint: testpackage import ( "testing" diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 3384e6270..1b8714590 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -13,9 +13,9 @@ import ( "strings" "time" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" @@ -52,7 +52,8 @@ func NewDestinationRegistry( } } -func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error) { +// Check if image is already synced. +func (registry *DestinationRegistry) CanSkipImage(repo, tag string, digest godigest.Digest) (bool, error) { // check image already synced imageStore := registry.storeController.GetImageStore(repo) @@ -68,10 +69,10 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest return false, err } - if localImageManifestDigest != imageDigest { + if localImageManifestDigest != digest { registry.log.Info().Str("repo", repo).Str("reference", tag). Str("localDigest", localImageManifestDigest.String()). - Str("remoteDigest", imageDigest.String()). + Str("remoteDigest", digest.String()). Msg("remote image digest changed, syncing again") return false, nil @@ -80,116 +81,121 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest return true, nil } -func (registry *DestinationRegistry) GetContext() *types.SystemContext { - return registry.tempStorage.GetContext() -} - -func (registry *DestinationRegistry) GetImageReference(repo, reference string) (types.ImageReference, error) { +func (registry *DestinationRegistry) GetImageReference(repo, reference string) (ref.Ref, error) { return registry.tempStorage.GetImageReference(repo, reference) } // finalize a syncing image. -func (registry *DestinationRegistry) CommitImage(imageReference types.ImageReference, repo, reference string) error { +func (registry *DestinationRegistry) CommitAll(repo string, imageReference ref.Ref) error { imageStore := registry.storeController.GetImageStore(repo) tempImageStore := getImageStoreFromImageReference(imageReference, repo, reference, registry.log) defer os.RemoveAll(tempImageStore.RootDir()) - registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("reference", reference). + registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("repository", repo). Msg("pushing synced local image to local registry") var lockLatency time.Time - manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference) + index, err := storageCommon.GetIndex(tempImageStore, repo, registry.log) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference). - Msg("couldn't find synced manifest in temporary sync dir") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo). + Msg("failed to get repo index from temp sync dir") return err } - // is image manifest - switch mediaType { - case ispec.MediaTypeImageManifest: - if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil { - if errors.Is(err, zerr.ErrImageLintAnnotations) { - registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't upload manifest because of missing annotations") + for _, desc := range index.Manifests { + reference := GetDescriptorReference(desc) - return nil - } + manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference) + if err != nil { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference). + Msg("failed to get manifest from temporary sync dir") return err } - case ispec.MediaTypeImageIndex: - // is image index - var indexManifest ispec.Index - if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)). - Msg("invalid JSON") + // is image manifest + switch mediaType { + case ispec.MediaTypeImageManifest: + if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil { + if errors.Is(err, zerr.ErrImageLintAnnotations) { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Msg("failed to upload manifest because of missing annotations") - return err - } + return nil + } - for _, manifest := range indexManifest.Manifests { - tempImageStore.RLock(&lockLatency) - manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest) - tempImageStore.RUnlock(&lockLatency) + return err + } + case ispec.MediaTypeImageIndex: + // is image index + var indexManifest ispec.Index - if err != nil { + if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()). - Msg("couldn't find manifest which is part of an image index") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)). + Msg("invalid JSON") return err } - if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), - tempImageStore); err != nil { - if errors.Is(err, zerr.ErrImageLintAnnotations) { + for _, manifest := range indexManifest.Manifests { + tempImageStore.RLock(&lockLatency) + manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest) + tempImageStore.RUnlock(&lockLatency) + + if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't upload manifest because of missing annotations") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()). + Msg("failed find manifest which is part of an image index") - return nil + return err } - return err - } - } + if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), + tempImageStore); err != nil { + if errors.Is(err, zerr.ErrImageLintAnnotations) { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Msg("failed to upload manifest because of missing annotations") - _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob) - if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). - Err(err).Msg("couldn't upload manifest") + return nil + } - return err - } + return err + } + } - if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, - manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) + _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob) if err != nil { - return fmt.Errorf("failed to set metadata for image '%s %s': %w", repo, reference, err) + registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). + Err(err).Msg("failed to upload manifest") + + return err } - registry.log.Debug().Str("repo", repo).Str("reference", reference).Str("component", "metadb"). - Msg("successfully set metadata for image") + if registry.metaDB != nil { + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, + manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) + if err != nil { + return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err) + } + + registry.log.Debug().Str("repo", repo).Str("reference", reference). + Msg("metaDB: successfully set metadata for image") + } } } - registry.log.Info().Str("image", fmt.Sprintf("%s:%s", repo, reference)).Msg("successfully synced image") - return nil } -func (registry *DestinationRegistry) CleanupImage(imageReference types.ImageReference, repo, reference string) error { - tmpDir := getTempRootDirFromImageReference(imageReference, repo, reference) - - return os.RemoveAll(tmpDir) +func (registry *DestinationRegistry) CleanupImage(imageReference ref.Ref, repo string) error { + return os.RemoveAll(strings.TrimSuffix(imageReference.Path, repo)) } func (registry *DestinationRegistry) copyManifest(repo string, manifestContent []byte, reference string, @@ -251,7 +257,7 @@ func (registry *DestinationRegistry) copyManifest(repo string, manifestContent [ } // Copy a blob from one image store to another image store. -func (registry *DestinationRegistry) copyBlob(repo string, blobDigest digest.Digest, blobMediaType string, +func (registry *DestinationRegistry) copyBlob(repo string, blobDigest godigest.Digest, blobMediaType string, tempImageStore storageTypes.ImageStore, ) error { imageStore := registry.storeController.GetImageStore(repo) diff --git a/pkg/extensions/sync/features/features.go b/pkg/extensions/sync/features/features.go deleted file mode 100644 index 0dd88a479..000000000 --- a/pkg/extensions/sync/features/features.go +++ /dev/null @@ -1,53 +0,0 @@ -package features - -import ( - "sync" - "time" -) - -const defaultExpireMinutes = 10 - -type featureKey struct { - kind string - repo string -} - -type featureVal struct { - enabled bool - expire time.Time -} - -type Map struct { - store map[featureKey]*featureVal - expireAfter time.Duration - mu *sync.Mutex -} - -func New() *Map { - return &Map{ - store: make(map[featureKey]*featureVal), - expireAfter: defaultExpireMinutes * time.Minute, - mu: new(sync.Mutex), - } -} - -// returns if registry supports this feature and if ok. -func (f *Map) Get(kind, repo string) (bool, bool) { - f.mu.Lock() - defer f.mu.Unlock() - - if feature, ok := f.store[featureKey{kind, repo}]; ok { - if time.Now().Before(feature.expire) { - return feature.enabled, true - } - } - - // feature expired or not found - return false, false -} - -func (f *Map) Set(kind, repo string, enabled bool) { - f.mu.Lock() - f.store[featureKey{kind: kind, repo: repo}] = &featureVal{enabled: enabled, expire: time.Now().Add(f.expireAfter)} - f.mu.Unlock() -} diff --git a/pkg/extensions/sync/httpclient/cache.go b/pkg/extensions/sync/httpclient/cache.go deleted file mode 100644 index e0aa56dee..000000000 --- a/pkg/extensions/sync/httpclient/cache.go +++ /dev/null @@ -1,58 +0,0 @@ -package client - -import ( - "sync" -) - -// Key:Value store for bearer tokens, key is namespace, value is token. -// We are storing only pull scoped tokens, the http client is for pulling only. -type TokenCache struct { - entries sync.Map -} - -func NewTokenCache() *TokenCache { - return &TokenCache{ - entries: sync.Map{}, - } -} - -func (c *TokenCache) Set(namespace string, token *bearerToken) { - if c == nil || token == nil { - return - } - - defer c.prune() - - c.entries.Store(namespace, token) -} - -func (c *TokenCache) Get(namespace string) *bearerToken { - if c == nil { - return nil - } - - val, ok := c.entries.Load(namespace) - if !ok { - return nil - } - - bearerToken, ok := val.(*bearerToken) - if !ok { - return nil - } - - return bearerToken -} - -func (c *TokenCache) prune() { - c.entries.Range(func(key, val any) bool { - bearerToken, ok := val.(*bearerToken) - if ok { - if bearerToken.isExpired() { - c.entries.Delete(key) - } - } - - return true - }) -} diff --git a/pkg/extensions/sync/httpclient/client_internal_test.go b/pkg/extensions/sync/httpclient/client_internal_test.go deleted file mode 100644 index 48ec04da5..000000000 --- a/pkg/extensions/sync/httpclient/client_internal_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - . "github.com/smartystreets/goconvey/convey" - - "zotregistry.dev/zot/pkg/log" -) - -func TestTokenCache(t *testing.T) { - Convey("Get/Set tokens", t, func() { - tokenCache := NewTokenCache() - token := &bearerToken{ - Token: "tokenA", - ExpiresIn: 3, - IssuedAt: time.Now(), - } - - token.expirationTime = token.IssuedAt.Add(time.Duration(token.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo", token) - cachedToken := tokenCache.Get("repo") - So(cachedToken.Token, ShouldEqual, token.Token) - - // add token which expires soon - token2 := &bearerToken{ - Token: "tokenB", - ExpiresIn: 1, - IssuedAt: time.Now(), - } - - token2.expirationTime = token2.IssuedAt.Add(time.Duration(token2.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo2", token2) - cachedToken = tokenCache.Get("repo2") - So(cachedToken.Token, ShouldEqual, token2.Token) - - time.Sleep(1 * time.Second) - - // token3 should be expired when adding a new one - token3 := &bearerToken{ - Token: "tokenC", - ExpiresIn: 3, - IssuedAt: time.Now(), - } - - token3.expirationTime = token3.IssuedAt.Add(time.Duration(token3.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo3", token3) - cachedToken = tokenCache.Get("repo3") - So(cachedToken.Token, ShouldEqual, token3.Token) - - // token2 should be expired - token = tokenCache.Get("repo2") - So(token, ShouldBeNil) - - time.Sleep(2 * time.Second) - - // the rest of them should also be expired - tokenCache.Set("repo4", &bearerToken{ - Token: "tokenD", - }) - - // token1 should be expired - token = tokenCache.Get("repo1") - So(token, ShouldBeNil) - }) - - Convey("Error paths", t, func() { - tokenCache := NewTokenCache() - token := tokenCache.Get("repo") - So(token, ShouldBeNil) - - tokenCache = nil - token = tokenCache.Get("repo") - So(token, ShouldBeNil) - - tokenCache = NewTokenCache() - tokenCache.Set("repo", nil) - token = tokenCache.Get("repo") - So(token, ShouldBeNil) - }) -} - -func TestNeedsRetryOnInsuficientScope(t *testing.T) { - resp := http.Response{ - Status: "401 Unauthorized", - StatusCode: http.StatusUnauthorized, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Header: map[string][]string{ - "Content-Length": {"145"}, - "Content-Type": {"application/json"}, - "Date": {"Fri, 26 Aug 2022 08:03:13 GMT"}, - "X-Content-Type-Options": {"nosniff"}, - }, - Request: nil, - } - - Convey("Test client retries on insufficient scope", t, func() { - resp.Header["Www-Authenticate"] = []string{ - `Bearer realm="https://registry.suse.com/auth",service="SUSE Linux Docker Registry"` + - `,scope="registry:catalog:*",error="insufficient_scope"`, - } - - expectedScope := "registry:catalog:*" - expectedRealm := "https://registry.suse.com/auth" - expectedService := "SUSE Linux Docker Registry" - - needsRetry, params := needsRetryWithUpdatedScope(nil, &resp) - - So(needsRetry, ShouldBeTrue) - So(params.scope, ShouldEqual, expectedScope) - So(params.realm, ShouldEqual, expectedRealm) - So(params.service, ShouldEqual, expectedService) - }) - - Convey("Test client fails on insufficient scope", t, func() { - resp.Header["Www-Authenticate"] = []string{ - `Bearer realm="https://registry.suse.com/auth=error"`, - } - - needsRetry, _ := needsRetryWithUpdatedScope(nil, &resp) - So(needsRetry, ShouldBeFalse) - }) -} - -func TestClient(t *testing.T) { - Convey("Test client", t, func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer server.Close() - - client, err := New(Config{ - URL: server.URL, - TLSVerify: false, - }, log.NewLogger("", "")) - So(err, ShouldBeNil) - - Convey("Test Ping() fails", func() { - ok := client.Ping() - So(ok, ShouldBeFalse) - }) - - Convey("Test makeAndDoRequest() fails", func() { - client.authType = tokenAuth - //nolint: bodyclose - _, _, err := client.makeAndDoRequest(http.MethodGet, "application/json", "catalog", server.URL) - So(err, ShouldNotBeNil) - }) - - Convey("Test setupAuth() fails", func() { - request, err := http.NewRequest(http.MethodGet, server.URL, nil) //nolint: noctx - So(err, ShouldBeNil) - - client.authType = tokenAuth - err = client.setupAuth(request, "catalog") - So(err, ShouldNotBeNil) - }) - }) -} diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index c44a955c5..75a8a148e 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -5,41 +5,28 @@ package sync import ( "fmt" - "os" "path" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/types" "github.com/gofrs/uuid" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/extensions/sync/constants" "zotregistry.dev/zot/pkg/storage" - storageConstants "zotregistry.dev/zot/pkg/storage/constants" "zotregistry.dev/zot/pkg/test/inject" ) type OciLayoutStorageImpl struct { storeController storage.StoreController - context *types.SystemContext } func NewOciLayoutStorage(storeController storage.StoreController) OciLayoutStorage { - context := &types.SystemContext{} - // preserve compression - context.OCIAcceptUncompressedLayers = true - return OciLayoutStorageImpl{ storeController: storeController, - context: context, } } -func (oci OciLayoutStorageImpl) GetContext() *types.SystemContext { - return oci.context -} - -func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (types.ImageReference, error) { +func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (ref.Ref, error) { localImageStore := oci.storeController.GetImageStore(repo) if localImageStore == nil { return nil, zerr.ErrLocalImgStoreNotFound @@ -50,26 +37,26 @@ func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) uuid, err := uuid.NewV4() // hard to reach test case, injected error, see pkg/test/dev.go if err := inject.Error(err); err != nil { - return nil, err + return ref.Ref{}, err } sessionRepoPath := path.Join(tempSyncPath, uuid.String()) - localRepo := path.Join(sessionRepoPath, repo) - if err := os.MkdirAll(localRepo, storageConstants.DefaultDirPerms); err != nil { - return nil, err - } + sessionRepo := path.Join(sessionRepoPath, repo) - _, refIsDigest := parseReference(reference) + var imageRefPath string - if !refIsDigest { - localRepo = fmt.Sprintf("%s:%s", localRepo, reference) + digest, ok := parseReference(reference) + if ok { + imageRefPath = fmt.Sprintf("ocidir://%s@%s", sessionRepo, digest.String()) + } else { + imageRefPath = fmt.Sprintf("ocidir://%s:%s", sessionRepo, reference) //nolint: nosprintfhostport } - localImageRef, err := layout.ParseReference(localRepo) + imageReference, err := ref.New(imageRefPath) if err != nil { - return nil, err + return ref.Ref{}, err } - return localImageRef, nil + return imageReference, nil } diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index 02ef21be4..da9241954 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -7,9 +7,6 @@ import ( "context" "errors" "sync" - "time" - - "github.com/containers/common/pkg/retry" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" @@ -83,49 +80,16 @@ func (onDemand *BaseOnDemand) SyncImage(ctx context.Context, repo, reference str return err } -func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, repo string, - subjectDigestStr string, referenceType string, -) error { - var err error - - for _, service := range onDemand.services { - err = service.SetNextAvailableURL() - if err != nil { - return err - } - - err = service.SyncReference(ctx, repo, subjectDigestStr, referenceType) - if err != nil { - continue - } else { - return nil - } - } - - return err -} - func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference string, syncResult chan error) { var err error for serviceID, service := range onDemand.services { - err = service.SetNextAvailableURL() - - isPingErr := errors.Is(err, zerr.ErrSyncPingRegistry) - if err != nil && !isPingErr { - syncResult <- err - - return - } - - // no need to try to sync inline if there is a ping error, we want to retry in background - if !isPingErr { - err = service.SyncImage(ctx, repo, reference) - } - - if err != nil || isPingErr { + err = service.SyncImage(ctx, repo, reference) + if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrSyncImageFilteredOut) || - errors.Is(err, zerr.ErrSyncImageNotSigned) { + errors.Is(err, zerr.ErrSyncImageNotSigned) || + // some public registries may return 401 for not found. + errors.Is(err, zerr.ErrUnauthorizedAccess) { continue } @@ -141,34 +105,24 @@ func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference str continue } - retryOptions := service.GetRetryOptions() - - if retryOptions.MaxRetry > 0 { - // retry in background - go func(service Service) { - // remove image after syncing - defer func() { - onDemand.requestStore.Delete(req) - onDemand.log.Info().Str("repo", repo).Str("reference", reference). - Msg("sync routine for image exited") - }() - - onDemand.log.Info().Str("repo", repo).Str(reference, "reference").Str("err", err.Error()). - Str("component", "sync").Msg("starting routine to copy image, because of error") - - time.Sleep(retryOptions.Delay) - - // retrying in background, can't use the same context which should be cancelled by now. - if err = retry.RetryIfNecessary(context.Background(), func() error { - err := service.SyncImage(context.Background(), repo, reference) - - return err - }, retryOptions); err != nil { - onDemand.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). - Err(err).Str("component", "sync").Msg("failed to copy image") - } - }(service) - } + // retry in background + go func(service Service) { + // remove image after syncing + defer func() { + onDemand.requestStore.Delete(req) + onDemand.log.Info().Str("repo", repo).Str("reference", reference). + Msg("sync routine for image exited") + }() + + onDemand.log.Info().Str("repo", repo).Str(reference, "reference").Str("err", err.Error()). + Msg("sync routine: starting routine to copy image, because of error") + + err := service.SyncImage(context.Background(), repo, reference) + if err != nil { + onDemand.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). + Err(err).Msg("sync routine: error while copying image") + } + }(service) } else { break } diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go deleted file mode 100644 index 16c78f4d3..000000000 --- a/pkg/extensions/sync/references/oci.go +++ /dev/null @@ -1,237 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" - storageTypes "zotregistry.dev/zot/pkg/storage/types" -) - -type OciReferences struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewOciReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) OciReferences { - return OciReferences{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref OciReferences) Name() string { - return constants.OCI -} - -func (ref OciReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool { - // use artifactTypeFilter - index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return false - } - - if len(getNotationManifestsFromOCIRefs(index)) > 0 || len(getCosignManifestsFromOCIRefs(index)) > 0 { - return true - } - - return false -} - -func (ref OciReferences) canSkipReferences(localRepo, subjectDigestStr string, index ispec.Index) (bool, error) { - imageStore := ref.storeController.GetImageStore(localRepo) - digest := godigest.Digest(subjectDigestStr) - - // check oci references already synced - if len(index.Manifests) > 0 { - localRefs, err := imageStore.GetReferrers(localRepo, digest, nil) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get local oci references for image") - - return false, err - } - - if !descriptorsEqual(localRefs.Manifests, index.Manifests) { - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("remote oci references for image changed, syncing again") - - return false, nil - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("skipping oci references for image, already synced") - - return true, nil -} - -func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - refsDigests := make([]godigest.Digest, 0, 10) - - index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return refsDigests, err - } - - skipOCIRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, index) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the upstream oci references for image can be skipped") - } - - if skipOCIRefs { - /* even if it's skip we need to return the digests, - because maybe in the meantime a reference pointing to this one was pushed */ - for _, man := range index.Manifests { - refsDigests = append(refsDigests, man.Digest) - } - - return refsDigests, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing oci references for image") - - for _, referrer := range index.Manifests { - referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo, - referrer, subjectDigestStr, ref.log) - if err != nil { - return refsDigests, err - } - - refsDigests = append(refsDigests, referenceDigest) - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to add oci references for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added oci references to MetaDB for image") - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced oci references for image") - - return refsDigests, nil -} - -func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) { - var index ispec.Index - - _, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, - "v2", repo, "referrers", subjectDigestStr) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr). - Msg("couldn't find any oci reference for image, skipping") - - return index, zerr.ErrSyncReferrerNotFound - } - - return index, err - } - - return index, nil -} - -func syncManifest(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore, localRepo, - remoteRepo string, desc ispec.Descriptor, subjectDigestStr string, log log.Logger, -) ([]byte, godigest.Digest, error) { - var manifest ispec.Manifest - - var refDigest godigest.Digest - - OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, - "v2", remoteRepo, "manifests", desc.Digest.String()) - if err != nil { - if statusCode == http.StatusNotFound { - return []byte{}, refDigest, zerr.ErrSyncReferrerNotFound - } - - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get oci reference manifest for image") - - return []byte{}, refDigest, err - } - - if desc.MediaType == ispec.MediaTypeImageManifest { - // read manifest - var manifest ispec.Manifest - - err = json.Unmarshal(OCIRefBuf, &manifest) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't unmarshal oci reference manifest for image") - - return []byte{}, refDigest, err - } - - for _, layer := range manifest.Layers { - if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, layer.Digest, log); err != nil { - return []byte{}, refDigest, err - } - } - - // sync config blob - if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, log); err != nil { - return []byte{}, refDigest, err - } - } else { - return []byte{}, refDigest, nil - } - - refDigest, _, err = imageStore.PutImageManifest(localRepo, desc.Digest.String(), - desc.MediaType, OCIRefBuf) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload oci reference for image") - - return []byte{}, refDigest, err - } - - return OCIRefBuf, refDigest, nil -} diff --git a/pkg/extensions/sync/references/references.go b/pkg/extensions/sync/references/references.go deleted file mode 100644 index 3e7ec2748..000000000 --- a/pkg/extensions/sync/references/references.go +++ /dev/null @@ -1,234 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/http" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sigstore/cosign/v2/pkg/oci/static" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - "zotregistry.dev/zot/pkg/extensions/sync/features" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" - storageTypes "zotregistry.dev/zot/pkg/storage/types" -) - -type Reference interface { - // Returns name of reference (OCIReference/CosignReference) - Name() string - // Returns whether or not image is signed - IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool - // Sync recursively all references for a subject digest (can be image/artifacts/signatures) - SyncReferences(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) ([]godigest.Digest, error) -} - -type References struct { - referenceList []Reference - features *features.Map - log log.Logger -} - -func NewReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) References { - refs := References{features: features.New(), log: log} - - refs.referenceList = append(refs.referenceList, NewCosignReference(httpClient, storeController, metaDB, log)) - refs.referenceList = append(refs.referenceList, NewTagReferences(httpClient, storeController, metaDB, log)) - refs.referenceList = append(refs.referenceList, NewOciReferences(httpClient, storeController, metaDB, log)) - - return refs -} - -func (refs References) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool { - for _, ref := range refs.referenceList { - ok := ref.IsSigned(ctx, upstreamRepo, subjectDigestStr) - if ok { - return true - } - } - - return false -} - -func (refs References) SyncAll(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) error { - seen := &[]godigest.Digest{} - - return refs.syncAll(ctx, localRepo, upstreamRepo, subjectDigestStr, seen) -} - -func (refs References) syncAll(ctx context.Context, localRepo, upstreamRepo, - subjectDigestStr string, seen *[]godigest.Digest, -) error { - var err error - - var syncedRefsDigests []godigest.Digest - - // mark subject digest as seen as soon as it comes in - *seen = append(*seen, godigest.Digest(subjectDigestStr)) - - // for each reference type(cosign/oci reference) - for _, ref := range refs.referenceList { - supported, ok := refs.features.Get(ref.Name(), upstreamRepo) - if !supported && ok { - continue - } - - syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr) - if err != nil { - // for all referrers we can stop querying same repo (for ten minutes) if the errors are different than 404 - if !errors.Is(err, zerr.ErrSyncReferrerNotFound) { - refs.features.Set(ref.Name(), upstreamRepo, false) - } - - // in the case of oci referrers, it will return 404 only if the repo is not found or refferers API is not supported - // no need to continue to make requests to the same repo - if ref.Name() == constants.OCI && errors.Is(err, zerr.ErrSyncReferrerNotFound) { - refs.features.Set(ref.Name(), upstreamRepo, false) - } - - refs.log.Debug().Err(err). - Str("reference type", ref.Name()). - Str("image", fmt.Sprintf("%s:%s", upstreamRepo, subjectDigestStr)). - Msg("couldn't sync image referrer") - } else { - refs.features.Set(ref.Name(), upstreamRepo, true) - } - - // for each synced references - for _, refDigest := range syncedRefsDigests { - if !common.Contains(*seen, refDigest) { - // sync all references pointing to this one - err = refs.syncAll(ctx, localRepo, upstreamRepo, refDigest.String(), seen) - } - } - } - - return err -} - -func (refs References) SyncReference(ctx context.Context, localRepo, upstreamRepo, - subjectDigestStr, referenceType string, -) error { - var err error - - var syncedRefsDigests []godigest.Digest - - for _, ref := range refs.referenceList { - if ref.Name() == referenceType { - syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr) - if err != nil { - refs.log.Debug().Err(err). - Str("reference type", ref.Name()). - Str("image", fmt.Sprintf("%s:%s", upstreamRepo, subjectDigestStr)). - Msg("couldn't sync image referrer") - - return err - } - - for _, refDigest := range syncedRefsDigests { - err = refs.SyncAll(ctx, localRepo, upstreamRepo, refDigest.String()) - } - } - } - - return err -} - -func syncBlob(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore, - localRepo, remoteRepo string, digest godigest.Digest, log log.Logger, -) error { - var resultPtr interface{} - - body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "v2", remoteRepo, "blobs", digest.String()) - if err != nil { - if statusCode != http.StatusOK { - log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob") - - return err - } - } - - _, _, err = imageStore.FullBlobUpload(localRepo, bytes.NewBuffer(body), digest) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)).Str("digest", digest.String()).Str("repo", localRepo). - Err(err).Msg("couldn't upload blob") - - return err - } - - return nil -} - -func manifestsEqual(manifest1, manifest2 ispec.Manifest) bool { - if manifest1.Config.Digest == manifest2.Config.Digest && - manifest1.Config.MediaType == manifest2.Config.MediaType && - manifest1.Config.Size == manifest2.Config.Size { - if descriptorsEqual(manifest1.Layers, manifest2.Layers) { - return true - } - } - - return false -} - -func descriptorsEqual(desc1, desc2 []ispec.Descriptor) bool { - if len(desc1) != len(desc2) { - return false - } - - for id, desc := range desc1 { - if !descriptorEqual(desc, desc2[id]) { - return false - } - } - - return true -} - -func descriptorEqual(desc1, desc2 ispec.Descriptor) bool { - if desc1.Size == desc2.Size && - desc1.Digest == desc2.Digest && - desc1.MediaType == desc2.MediaType && - desc1.Annotations[static.SignatureAnnotationKey] == desc2.Annotations[static.SignatureAnnotationKey] { - return true - } - - return false -} - -func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { - notaryManifests := []ispec.Descriptor{} - - for _, ref := range ociRefs.Manifests { - if ref.ArtifactType == common.ArtifactTypeNotation { - notaryManifests = append(notaryManifests, ref) - } - } - - return notaryManifests -} - -func getCosignManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { - cosignManifests := []ispec.Descriptor{} - - for _, ref := range ociRefs.Manifests { - if ref.ArtifactType == common.ArtifactTypeCosign { - cosignManifests = append(cosignManifests, ref) - } - } - - return cosignManifests -} diff --git a/pkg/extensions/sync/references/references_internal_test.go b/pkg/extensions/sync/references/references_internal_test.go deleted file mode 100644 index a17020ce3..000000000 --- a/pkg/extensions/sync/references/references_internal_test.go +++ /dev/null @@ -1,306 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "testing" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - . "github.com/smartystreets/goconvey/convey" - - zerr "zotregistry.dev/zot/errors" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/storage" - "zotregistry.dev/zot/pkg/test/mocks" -) - -var errRef = errors.New("err") - -func TestCosign(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - cosign := NewCosignReference(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", errRef - }, - }}, nil, log.NewLogger("debug", "")) - - ok, err := cosign.canSkipReferences("repo", "tag", nil) - So(err, ShouldBeNil) - So(ok, ShouldBeTrue) - - // trigger GetImageManifest err - ok, err = cosign.canSkipReferences("repo", "tag", &ispec.Manifest{MediaType: ispec.MediaTypeImageManifest}) - So(err, ShouldNotBeNil) - So(ok, ShouldBeFalse) - - cosign = NewCosignReference(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "digest", "", nil - }, - }}, nil, log.NewLogger("debug", "")) - - // different digest - ok, err = cosign.canSkipReferences("repo", "tag", &ispec.Manifest{MediaType: ispec.MediaTypeImageManifest}) - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestOci(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - oci := NewOciReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) { - return ispec.Index{}, zerr.ErrManifestNotFound - }, - }}, nil, log.NewLogger("debug", "")) - - ok := oci.IsSigned(context.Background(), "repo", "") - So(ok, ShouldBeFalse) - - // trigger GetReferrers err - ok, err = oci.canSkipReferences("repo", "tag", ispec.Index{Manifests: []ispec.Descriptor{{Digest: "digest1"}}}) - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestReferrersTag(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - referrersTag := NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", errRef - }, - }}, nil, log.NewLogger("debug", "")) - - ok := referrersTag.IsSigned(context.Background(), "repo", "") - So(ok, ShouldBeFalse) - - // trigger GetImageManifest err - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest") - So(err, ShouldNotBeNil) - So(ok, ShouldBeFalse) - - referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", zerr.ErrManifestNotFound - }, - }}, nil, log.NewLogger("debug", "")) - - // trigger GetImageManifest err - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest") - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - - referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "digest", "", nil - }, - }}, nil, log.NewLogger("debug", "")) - - // different digest - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "newdigest") - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestSyncManifest(t *testing.T) { - Convey("sync manifest not found err", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - digest := godigest.FromString("test") - - buf, refDigest, err := syncManifest(context.Background(), client, mocks.MockedImageStore{}, - "repo", "repo", ispec.Descriptor{ - Digest: digest, - Size: 10, - MediaType: ispec.MediaTypeImageManifest, - }, digest.String(), log.Logger{}) - - So(buf, ShouldBeEmpty) - So(refDigest, ShouldBeEmpty) - So(err, ShouldNotBeNil) - }) -} - -func TestCompareManifest(t *testing.T) { - testCases := []struct { - manifest1 ispec.Manifest - manifest2 ispec.Manifest - expected bool - }{ - { - manifest1: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest1", - }, - }, - manifest2: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest2", - }, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest", - }, - }, - manifest2: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest", - }, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest1", - Size: 1, - }}, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest2", - Size: 2, - }}, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest1", - Size: 1, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest1", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest1", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest1", - Size: 1, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - expected: false, - }, - } - - Convey("Test manifestsEqual()", t, func() { - for _, test := range testCases { - actualResult := manifestsEqual(test.manifest1, test.manifest2) - So(actualResult, ShouldEqual, test.expected) - } - }) -} diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go deleted file mode 100644 index 21ffd2755..000000000 --- a/pkg/extensions/sync/references/referrers_tag.go +++ /dev/null @@ -1,172 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" -) - -type TagReferences struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewTagReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) TagReferences { - return TagReferences{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref TagReferences) Name() string { - return constants.Tag -} - -func (ref TagReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool { - return false -} - -func (ref TagReferences) canSkipReferences(localRepo, subjectDigestStr, digest string) (bool, error) { - imageStore := ref.storeController.GetImageStore(localRepo) - - _, localDigest, _, err := imageStore.GetImageManifest(localRepo, getReferrersTagFromSubjectDigest(subjectDigestStr)) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get local index with referrers tag for image") - - return false, err - } - - if localDigest.String() != digest { - return false, nil - } - - ref.log.Info().Str("repository", localRepo).Str("reference", subjectDigestStr). - Msg("skipping index with referrers tag for image, already synced") - - return true, nil -} - -func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - refsDigests := make([]godigest.Digest, 0, 10) - - index, indexContent, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return refsDigests, err - } - - skipTagRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, string(godigest.FromBytes(indexContent))) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the upstream index with referrers tag for image can be skipped") - } - - if skipTagRefs { - return refsDigests, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing oci references for image") - - for _, referrer := range index.Manifests { - referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo, - referrer, subjectDigestStr, ref.log) - if err != nil { - return refsDigests, err - } - - refsDigests = append(refsDigests, referenceDigest) - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to add oci references for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added oci references to MetaDB for image") - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing index with referrers tag for image") - - referrersTag := getReferrersTagFromSubjectDigest(subjectDigestStr) - - _, _, err = imageStore.PutImageManifest(localRepo, referrersTag, index.MediaType, indexContent) - if err != nil { - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload index with referrers tag for image") - - return refsDigests, err - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced index with referrers tag for image") - - return refsDigests, nil -} - -func (ref TagReferences) getIndex( - ctx context.Context, repo, subjectDigestStr string, -) (ispec.Index, []byte, error) { - var index ispec.Index - - content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, - "v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr)) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr). - Msg("couldn't find any index with referrers tag for image, skipping") - - return index, []byte{}, zerr.ErrSyncReferrerNotFound - } - - return index, []byte{}, err - } - - return index, content, nil -} - -func getReferrersTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) -} diff --git a/pkg/extensions/sync/referrers.go b/pkg/extensions/sync/referrers.go new file mode 100644 index 000000000..40e196c15 --- /dev/null +++ b/pkg/extensions/sync/referrers.go @@ -0,0 +1,48 @@ +//go:build sync +// +build sync + +package sync + +import ( + "strings" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/referrer" + + "zotregistry.dev/zot/pkg/common" +) + +const ( + cosignSignatureTagSuffix = "sig" + SBOMTagSuffix = "sbom" +) + +func hasSignatureReferrers(refs referrer.ReferrerList) bool { + for _, desc := range refs.Descriptors { + tag := desc.Annotations[ispec.AnnotationRefName] + + if isCosignTag(tag) { + return true + } + + if desc.ArtifactType == common.ArtifactTypeNotation { + return true + } + + if desc.ArtifactType == common.ArtifactTypeCosign { + return true + } + } + + return false +} + +// this function will check if tag is a cosign tag (signature or sbom). +func isCosignTag(tag string) bool { + if strings.HasPrefix(tag, "sha256-") && + (strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) { + return true + } + + return false +} diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go index fef205d42..18aacab1b 100644 --- a/pkg/extensions/sync/remote.go +++ b/pkg/extensions/sync/remote.go @@ -5,160 +5,348 @@ package sync import ( "context" + "encoding/json" + "errors" "fmt" - "strings" - "github.com/containers/image/v5/docker" - dockerReference "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - - "zotregistry.dev/zot/pkg/api/constants" + "github.com/regclient/regclient" + "github.com/regclient/regclient/types/descriptor" + "github.com/regclient/regclient/types/errs" + "github.com/regclient/regclient/types/manifest" + "github.com/regclient/regclient/types/mediatype" + "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/repo" + + zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" "zotregistry.dev/zot/pkg/log" ) -type catalog struct { - Repositories []string `json:"repositories"` -} - type RemoteRegistry struct { - client *client.Client - context *types.SystemContext - log log.Logger + client *regclient.RegClient + hosts []string + host string + log log.Logger } -func NewRemoteRegistry(client *client.Client, logger log.Logger) Remote { +func NewRemoteRegistry(client *regclient.RegClient, hosts []string, logger log.Logger) Remote { registry := &RemoteRegistry{} registry.log = logger + registry.hosts = hosts registry.client = client - clientConfig := client.GetConfig() - registry.context = getUpstreamContext(clientConfig.CertDir, clientConfig.Username, - clientConfig.Password, clientConfig.TLSVerify) + registry.host = hosts[0] return registry } -func (registry *RemoteRegistry) GetContext() *types.SystemContext { - return registry.context +func (registry *RemoteRegistry) GetHostName() string { + return registry.host } func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) { - var catalog catalog + var err error + + var repoList *repo.RepoList + + for _, host := range registry.hosts { + repoList, err = registry.client.RepoList(ctx, host) + if err != nil { + registry.log.Error().Err(err).Str("remote", host).Msg("failed to list repositories in remote registry") + + continue + } + + return repoList.Repositories, nil + } + + return []string{}, err +} - _, _, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", //nolint: dogsled - constants.RoutePrefix, constants.ExtCatalogPrefix) +func (registry *RemoteRegistry) GetImageReference(repo, reference string) (ref.Ref, error) { + digest, ok := parseReference(reference) + + var imageRefPath string + if ok { + imageRefPath = fmt.Sprintf("%s/%s@%s", registry.host, repo, digest.String()) + } else { + // is tag + imageRefPath = fmt.Sprintf("%s/%s:%s", registry.host, repo, reference) + } + + imageRef, err := ref.New(imageRefPath) if err != nil { - return []string{}, err + return ref.Ref{}, err } - return catalog.Repositories, nil + return imageRef, nil } -func (registry *RemoteRegistry) GetDockerRemoteRepo(repo string) string { - dockerNamespace := "library" - dockerRegistry := "docker.io" +func (registry *RemoteRegistry) GetOCIManifest(ctx context.Context, repo, reference string, +) ([]byte, ispec.Descriptor, bool, error) { + var isConverted bool + + var buf []byte - remoteHost := registry.client.GetHostname() + var desc ispec.Descriptor - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", remoteHost, repo)) + imageReference, err := registry.GetImageReference(repo, reference) if err != nil { - return repo + return nil, ispec.Descriptor{}, false, err } - if !strings.Contains(repo, dockerNamespace) && - strings.Contains(repoRef.String(), dockerNamespace) && - strings.Contains(repoRef.String(), dockerRegistry) { - return fmt.Sprintf("%s/%s", dockerNamespace, repo) + /// check what error it gives when not found + man, err := registry.client.ManifestGet(ctx, imageReference) + if err != nil { + /* public registries may return 401 for image not found + they will try to check private registries as a fallback => 401 */ + if errors.Is(err, errs.ErrHTTPUnauthorized) { + registry.log.Info().Str("errorType", common.TypeOf(err)). + Str("repository", repo).Str("reference", reference). + Err(err).Msg("failed to get manifest: unauthorized") + + return nil, ispec.Descriptor{}, false, zerr.ErrUnauthorizedAccess + } else if errors.Is(err, errs.ErrNotFound) { + registry.log.Info().Str("errorType", common.TypeOf(err)). + Str("repository", repo).Str("reference", reference). + Err(err).Msg("failed to find manifest") + + return nil, ispec.Descriptor{}, false, zerr.ErrManifestNotFound + } + + return nil, ispec.Descriptor{}, false, err + } + + switch man.GetDescriptor().MediaType { + case mediatype.Docker2Manifest: + buf, desc, err = convertDockerManifestToOCI(ctx, man, imageReference, registry.client) + isConverted = true + case mediatype.Docker2ManifestList: + buf, desc, err = convertDockerListToOCI(ctx, man, imageReference, registry.client) + isConverted = true + case mediatype.OCI1Manifest, mediatype.OCI1ManifestList: + buf, err = man.MarshalJSON() + desc = toOCIDescriptor(man.GetDescriptor()) + default: + return nil, desc, false, zerr.ErrMediaTypeNotSupported } - return repo + return buf, desc, isConverted, err } -func (registry *RemoteRegistry) GetImageReference(repo, reference string) (types.ImageReference, error) { - remoteHost := registry.client.GetHostname() +func (registry *RemoteRegistry) GetTags(ctx context.Context, repo string) ([]string, error) { + repoRefPath := fmt.Sprintf("%s/%s", registry.host, repo) - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", remoteHost, repo)) + repoReference, err := ref.New(repoRefPath) if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). - Str("reference", reference).Str("remote", remoteHost). - Err(err).Msg("couldn't parse repository reference") + return []string{}, err + } - return nil, err + tl, err := registry.client.TagList(ctx, repoReference) + if err != nil { + return []string{}, err } - var namedRepoRef dockerReference.Named + return tl.GetTags() +} - digest, ok := parseReference(reference) - if ok { - namedRepoRef, err = dockerReference.WithDigest(repoRef, digest) +func convertDockerListToOCI(ctx context.Context, man manifest.Manifest, imageReference ref.Ref, + regclient *regclient.RegClient, +) ( + []byte, ispec.Descriptor, error, +) { + var index ispec.Index + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{} + index.MediaType = ispec.MediaTypeImageIndex + + indexer, ok := man.(manifest.Indexer) + if !ok { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } + + ociIndex, err := manifest.OCIIndexFromAny(man.GetOrig()) + if err != nil { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } + + manifests, err := indexer.GetManifestList() + if err != nil { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } + + for _, manDesc := range manifests { + ref := imageReference + ref.Digest = manDesc.Digest.String() + + manEntry, err := regclient.ManifestGet(ctx, ref) if err != nil { - return nil, err + return nil, ispec.Descriptor{}, err } - } else { - namedRepoRef, err = dockerReference.WithTag(repoRef, reference) + + regclient.Close(ctx, manEntry.GetRef()) + + _, desc, err := convertDockerManifestToOCI(ctx, manEntry, ref, regclient) if err != nil { - return nil, err + return nil, ispec.Descriptor{}, err } + + // copy desc platform from docker desc + if manDesc.Platform != nil { + desc.Platform = &ispec.Platform{ + Architecture: manDesc.Platform.Architecture, + OS: manDesc.Platform.OS, + OSVersion: manDesc.Platform.OSVersion, + OSFeatures: manDesc.Platform.OSFeatures, + Variant: manDesc.Platform.Variant, + } + } + + index.Manifests = append(index.Manifests, desc) } - imageRef, err := docker.NewReference(namedRepoRef) - if err != nil { - registry.log.Err(err).Str("transport", docker.Transport.Name()).Str("reference", namedRepoRef.String()). - Msg("cannot obtain a valid image reference for given transport and reference") + index.Annotations = ociIndex.Annotations + index.ArtifactType = ociIndex.ArtifactType - return nil, err + if ociIndex.Subject != nil { + subject := toOCIDescriptor(*ociIndex.Subject) + index.Subject = &subject } - return imageRef, nil + indexBuf, err := json.Marshal(index) + if err != nil { + return nil, ispec.Descriptor{}, err + } + + indexDesc := toOCIDescriptor(man.GetDescriptor()) + + indexDesc.MediaType = ispec.MediaTypeImageIndex + indexDesc.Digest = godigest.FromBytes(indexBuf) + indexDesc.Size = int64(len(indexBuf)) + + return indexBuf, indexDesc, nil } -func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageReference) ( - []byte, string, digest.Digest, error, +func convertDockerManifestToOCI(ctx context.Context, man manifest.Manifest, imageReference ref.Ref, + regclient *regclient.RegClient, +) ( + []byte, ispec.Descriptor, error, ) { - imageSource, err := imageReference.NewImageSource(context.Background(), registry.GetContext()) + imager, ok := man.(manifest.Imager) + if !ok { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } + + var ociManifest ispec.Manifest + + manifest, err := man.RawBody() if err != nil { - return []byte{}, "", "", err + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported } - defer imageSource.Close() + if err := json.Unmarshal(manifest, &ociManifest); err != nil { + return nil, ispec.Descriptor{}, err + } - manifestBuf, mediaType, err := imageSource.GetManifest(context.Background(), nil) + configDesc, err := imager.GetConfig() if err != nil { - return []byte{}, "", "", err + return nil, ispec.Descriptor{}, err } - // if mediatype is docker then convert to OCI - switch mediaType { - case manifest.DockerV2Schema2MediaType: - manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf) - if err != nil { - return []byte{}, "", "", err - } - case manifest.DockerV2ListMediaType: - manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf) - if err != nil { - return []byte{}, "", "", err - } + // get config blob + config, err := regclient.BlobGetOCIConfig(ctx, imageReference, configDesc) + if err != nil { + return nil, ispec.Descriptor{}, err } - return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil -} + configBuf, err := config.RawBody() + if err != nil { + return nil, ispec.Descriptor{}, err + } + + // var ociConfig ispec.Image -func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) { - remoteHost := registry.client.GetHostname() + // if err := json.Unmarshal(configBuf, &ociConfig); err != nil { + // return nil, ispec.Descriptor{}, err + // } - tags, err := getRepoTags(context.Background(), registry.GetContext(), remoteHost, repo) + // ociConfigContent, err := json.Marshal(ociConfig) + // if err != nil { + // return nil, ispec.Descriptor{}, err + // } + + // convert config and manifest mediatype + ociManifest.Config.Size = int64(len(configBuf)) + ociManifest.Config.Digest = godigest.FromBytes(configBuf) + ociManifest.Config.MediaType = ispec.MediaTypeImageConfig + ociManifest.MediaType = ispec.MediaTypeImageManifest + + layersDesc, err := imager.GetLayers() if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). - Str("remote", remoteHost).Err(err).Msg("couldn't fetch tags for repo") + return nil, ispec.Descriptor{}, err + } - return []string{}, err + ociManifest.Layers = []ispec.Descriptor{} + + for _, layerDesc := range layersDesc { + ociManifest.Layers = append(ociManifest.Layers, toOCIDescriptor(layerDesc)) + } + + manifestBuf, err := json.Marshal(ociManifest) + if err != nil { + return nil, ispec.Descriptor{}, err } - return tags, nil + manifestDesc := toOCIDescriptor(man.GetDescriptor()) + + manifestDesc.MediaType = ispec.MediaTypeImageManifest + manifestDesc.Digest = godigest.FromBytes(manifestBuf) + manifestDesc.Size = int64(len(manifestBuf)) + + return manifestBuf, manifestDesc, nil +} + +func toOCIDescriptor(desc descriptor.Descriptor) ispec.Descriptor { + ispecPlatform := &ispec.Platform{} + + platform := desc.Platform + if platform != nil { + ispecPlatform.Architecture = platform.Architecture + ispecPlatform.OS = platform.OS + ispecPlatform.OSFeatures = platform.OSFeatures + ispecPlatform.OSVersion = platform.OSVersion + ispecPlatform.Variant = platform.Variant + } else { + ispecPlatform = nil + } + + var mediaType string + + switch desc.MediaType { + case mediatype.Docker2Manifest: + mediaType = ispec.MediaTypeImageManifest + case mediatype.Docker2ManifestList: + mediaType = ispec.MediaTypeImageIndex + case mediatype.Docker2ImageConfig: + mediaType = ispec.MediaTypeImageConfig + case mediatype.Docker2ForeignLayer: + mediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck + case mediatype.Docker2LayerGzip: + mediaType = ispec.MediaTypeImageLayerGzip + default: + mediaType = desc.MediaType + } + + return ispec.Descriptor{ + MediaType: mediaType, + Digest: desc.Digest, + Size: desc.Size, + URLs: desc.URLs, + Annotations: desc.Annotations, + Platform: ispecPlatform, + ArtifactType: desc.ArtifactType, + } } diff --git a/pkg/extensions/sync/service.go b/pkg/extensions/sync/service.go index 4f1fca23d..2ac1c290c 100644 --- a/pkg/extensions/sync/service.go +++ b/pkg/extensions/sync/service.go @@ -8,19 +8,21 @@ import ( "errors" "fmt" "strconv" + "time" - "github.com/containers/common/pkg/retry" - "github.com/containers/image/v5/copy" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + "github.com/regclient/regclient" + "github.com/regclient/regclient/config" + "github.com/regclient/regclient/mod" + "github.com/regclient/regclient/scheme/reg" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/api/config" + zconfig "zotregistry.dev/zot/pkg/api/config" "zotregistry.dev/zot/pkg/api/constants" "zotregistry.dev/zot/pkg/cluster" "zotregistry.dev/zot/pkg/common" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/extensions/sync/references" "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/storage" @@ -29,23 +31,22 @@ import ( type BaseService struct { config syncconf.RegistryConfig credentials syncconf.CredentialsFile - clusterConfig *config.ClusterConfig remote Remote destination Destination - retryOptions *retry.RetryOptions + clusterConfig *zconfig.ClusterConfig + copyOptions []regclient.ImageOpts contentManager ContentManager storeController storage.StoreController metaDB mTypes.MetaDB repositories []string - references references.References - client *client.Client + regclient *regclient.RegClient log log.Logger } func New( opts syncconf.RegistryConfig, credentialsFilepath string, - clusterConfig *config.ClusterConfig, + clusterConfig *zconfig.ClusterConfig, tmpDir string, storeController storage.StoreController, metadb mTypes.MetaDB, @@ -56,6 +57,8 @@ func New( service.config = opts service.log = log service.metaDB = metadb + service.contentManager = NewContentManager(opts.Content, log) + service.storeController = storeController var err error @@ -91,100 +94,105 @@ func New( ) } - retryOptions := &retry.RetryOptions{} + var maxRetries int + + var retryDelay time.Duration if opts.MaxRetries != nil { - retryOptions.MaxRetry = *opts.MaxRetries + maxRetries = *opts.MaxRetries + if opts.RetryDelay != nil { - retryOptions.Delay = *opts.RetryDelay + retryDelay = *opts.RetryDelay } } - service.retryOptions = retryOptions service.storeController = storeController - // try to set next client. - if err := service.SetNextAvailableClient(); err != nil { - // if it's a ping issue, it will be retried - if !errors.Is(err, zerr.ErrSyncPingRegistry) { - return service, err - } + urls, err := parseRegistryURLs(opts.URLs) + if err != nil { + return nil, err } - service.references = references.NewReferences( - service.client, - service.storeController, - service.metaDB, - service.log, - ) - - service.remote = NewRemoteRegistry( - service.client, - service.log, - ) + tls := config.TLSEnabled // default - return service, nil -} + mainHost := urls[0].Host -func (service *BaseService) SetNextAvailableClient() error { - if service.client != nil && service.client.Ping() { - return nil + if urls[0].Scheme == "http" { + tls = config.TLSDisabled } - found := false - - for _, url := range service.config.URLs { - // skip current client - if service.client != nil && service.client.GetBaseURL() == url { - continue - } + mirrorsHosts := make([]string, 0) + for _, url := range urls[1:] { + mirrorsHosts = append(mirrorsHosts, url.Host) + } - remoteAddress := StripRegistryTransport(url) - credentials := service.credentials[remoteAddress] + hostConfig := config.Host{} + hostConfig.Name = mainHost + hostConfig.Mirrors = mirrorsHosts + hostConfig.TLS = tls - tlsVerify := true - if service.config.TLSVerify != nil { - tlsVerify = *service.config.TLSVerify + if opts.CertDir != "" { + clientCert, clientKey, regCert, err := getCertificates(opts.CertDir) + if err != nil { + return nil, err } - options := client.Config{ - URL: url, - Username: credentials.Username, - Password: credentials.Password, - TLSVerify: tlsVerify, - CertDir: service.config.CertDir, - } + hostConfig.ClientCert = clientCert + hostConfig.ClientKey = clientKey + hostConfig.RegCert = regCert + } - var err error + if mainHost == regclient.DockerRegistryAuth || + mainHost == regclient.DockerRegistryDNS || + mainHost == regclient.DockerRegistry || + mainHost == "index.docker.io" { + hostConfig.Name = regclient.DockerRegistry + hostConfig.Hostname = regclient.DockerRegistryDNS + hostConfig.CredHost = regclient.DockerRegistryAuth + } - if service.client != nil { - err = service.client.SetConfig(options) - } else { - service.client, err = client.New(options, service.log) - } + credentials, ok := credentialsFile[mainHost] + if ok { + hostConfig.User = credentials.Username + hostConfig.Pass = credentials.Password + } - if err != nil { - service.log.Error().Err(err).Str("url", url).Msg("failed to initialize http client") + hostConfigOpts := []config.Host{} + hostConfigOpts = append(hostConfigOpts, hostConfig) - return err - } + for _, mirror := range mirrorsHosts { + mirroHostConfig := hostConfig + mirroHostConfig.Name = mirror + hostConfigOpts = append(hostConfigOpts, mirroHostConfig) + } - if service.client.Ping() { - found = true + service.regclient = regclient.New( + regclient.WithDockerCerts(), + regclient.WithDockerCreds(), + regclient.WithRegOpts( + reg.WithCertDirs([]string{opts.CertDir}), + reg.WithRetryLimit(maxRetries), + reg.WithDelay(1*time.Second, retryDelay), + ), + regclient.WithConfigHost(hostConfigOpts...), + ) - break - } - } + hosts := []string{} + hosts = append(hosts, mainHost) + hosts = append(hosts, mirrorsHosts...) - if service.client == nil || !found { - return zerr.ErrSyncPingRegistry - } + service.remote = NewRemoteRegistry( + service.regclient, + hosts, + service.log, + ) - return nil -} + // we want referrers using sha-.*" tags + // service.copyOptions = append(service.copyOptions, regclient.ImageWithDigestTags()) + // we want oci referrers + service.copyOptions = append(service.copyOptions, regclient.ImageWithReferrers()) -func (service *BaseService) GetRetryOptions() *retry.Options { - return service.retryOptions + return service, nil } func (service *BaseService) getNextRepoFromCatalog(lastRepo string) string { @@ -219,13 +227,10 @@ func (service *BaseService) GetNextRepo(lastRepo string) (string, error) { var err error if len(service.repositories) == 0 { - if err = retry.RetryIfNecessary(context.Background(), func() error { - service.repositories, err = service.remote.GetRepositories(context.Background()) - - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("remote registry", service.client.GetConfig().URL). - Err(err).Msg("failed to get repository list from remote registry") + service.repositories, err = service.remote.GetRepositories(context.Background()) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("remote registry", service.remote.GetHostName()). + Err(err).Msg("error while getting repositories from remote registry") return "", err } @@ -262,82 +267,41 @@ func (service *BaseService) GetNextRepo(lastRepo string) (string, error) { return lastRepo, nil } -// SyncReference on demand. -func (service *BaseService) SyncReference(ctx context.Context, repo string, - subjectDigestStr string, referenceType string, -) error { - remoteRepo := repo - - remoteURL := service.client.GetConfig().URL - - if len(service.config.Content) > 0 { - remoteRepo = service.contentManager.GetRepoSource(repo) - if remoteRepo == "" { - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("subject", subjectDigestStr). - Str("reference type", referenceType).Msg("will not sync reference for image, filtered out by content") - - return zerr.ErrSyncImageFilteredOut - } - } - - remoteRepo = service.remote.GetDockerRemoteRepo(remoteRepo) - - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("subject", subjectDigestStr). - Str("reference type", referenceType).Msg("syncing reference for image") - - return service.references.SyncReference(ctx, repo, remoteRepo, subjectDigestStr, referenceType) -} - // SyncImage on demand. func (service *BaseService) SyncImage(ctx context.Context, repo, reference string) error { remoteRepo := repo - remoteURL := service.client.GetConfig().URL + remoteURL := service.remote.GetHostName() if len(service.config.Content) > 0 { remoteRepo = service.contentManager.GetRepoSource(repo) if remoteRepo == "" { - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("reference", reference). + service.log.Info().Str("remote", remoteURL).Str("repo", repo).Str("reference", reference). Msg("will not sync image, filtered out by content") return zerr.ErrSyncImageFilteredOut } } - remoteRepo = service.remote.GetDockerRemoteRepo(remoteRepo) + service.log.Info().Str("remote", remoteURL).Str("repo", repo).Str("reference", reference). + Msg("sync: syncing image") - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("reference", reference). - Msg("syncing image") - - manifestDigest, err := service.syncTag(ctx, repo, remoteRepo, reference) - if err != nil { - return err - } - - err = service.references.SyncAll(ctx, repo, remoteRepo, manifestDigest.String()) - if err != nil && !errors.Is(err, zerr.ErrSyncReferrerNotFound) { - return err - } - - return nil + return service.syncTagAndReferrers(ctx, repo, remoteRepo, reference) } // sync repo periodically. func (service *BaseService) SyncRepo(ctx context.Context, repo string) error { - service.log.Info().Str("repository", repo).Str("registry", service.client.GetConfig().URL). - Msg("syncing repo") + service.log.Info().Str("repo", repo).Str("registry", service.remote.GetHostName()). + Msg("sync: syncing repo") var err error var tags []string - if err = retry.RetryIfNecessary(ctx, func() error { - tags, err = service.remote.GetRepoTags(repo) - - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to get tags for repository") + tags, err = service.remote.GetTags(ctx, repo) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). + Err(err).Msg("error while getting tags for repo") return err } @@ -348,154 +312,187 @@ func (service *BaseService) SyncRepo(ctx context.Context, repo string) error { return err } - service.log.Info().Str("repository", repo).Msgf("syncing tags %v", tags) + service.log.Info().Str("repo", repo).Msgf("sync: syncing tags %v", tags) // apply content.destination rule - destinationRepo := service.contentManager.GetRepoDestination(repo) + localRepo := service.contentManager.GetRepoDestination(repo) for _, tag := range tags { if common.IsContextDone(ctx) { return ctx.Err() } - if references.IsCosignTag(tag) || common.IsReferrersTag(tag) { - continue - } - - var manifestDigest digest.Digest + // if isCosignTag(tag) || common.IsReferrersTag(tag) { + // continue + // } - if err = retry.RetryIfNecessary(ctx, func() error { - manifestDigest, err = service.syncTag(ctx, destinationRepo, repo, tag) - - return err - }, service.retryOptions); err != nil { - if errors.Is(err, zerr.ErrSyncImageNotSigned) || errors.Is(err, zerr.ErrMediaTypeNotSupported) { + err = service.syncTagAndReferrers(ctx, localRepo, repo, tag) + if err != nil { + if errors.Is(err, zerr.ErrSyncImageNotSigned) || + errors.Is(err, zerr.ErrUnauthorizedAccess) || + errors.Is(err, zerr.ErrMediaTypeNotSupported) || + errors.Is(err, zerr.ErrManifestNotFound) { // skip unsigned images or unsupported image mediatype continue } - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to sync tags for repository") + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). + Err(err).Msg("error while syncing tags for repo") return err } - - if manifestDigest != "" { - if err = retry.RetryIfNecessary(ctx, func() error { - err = service.references.SyncAll(ctx, destinationRepo, repo, manifestDigest.String()) - if errors.Is(err, zerr.ErrSyncReferrerNotFound) { - return nil - } - - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to sync tags for repository") - } - } } - service.log.Info().Str("component", "sync").Str("repository", repo).Msg("finished syncing repository") + service.log.Info().Str("repo", repo).Msg("sync: finished syncing repo") return nil } -func (service *BaseService) syncTag(ctx context.Context, destinationRepo, remoteRepo, tag string, -) (digest.Digest, error) { - copyOptions := getCopyOptions(service.remote.GetContext(), service.destination.GetContext()) +func (service *BaseService) syncReference(ctx context.Context, localRepo string, remoteImageRef, localImageRef ref.Ref, + remoteManifestDigest godigest.Digest, +) (bool, error) { + var reference string + + if remoteImageRef.Tag != "" { + reference = remoteImageRef.Tag + } else { + reference = remoteImageRef.Digest + } - policyContext, err := getPolicyContext(service.log) + // check if image digest + its referrers digests are already synced, otherwise sync everything again + skipImage, err := service.destination.CanSkipImage(localRepo, reference, remoteManifestDigest) if err != nil { - return "", err + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("repo", localRepo).Str("reference", remoteImageRef.Tag). + Msg("couldn't check if the local image can be skipped") + } + + if !skipImage { + service.log.Info().Str("remote image", remoteImageRef.CommonName()). + Str("local image", fmt.Sprintf("%s:%s", localRepo, remoteImageRef.Tag)).Msg("syncing image") + + err = service.regclient.ImageCopy(ctx, remoteImageRef, localImageRef) + if err != nil { + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("remote image", remoteImageRef.CommonName()). + Str("local image", fmt.Sprintf("%s:%s", localRepo, remoteImageRef.Tag)).Msg("failed to sync image") + + return false, err + } + } else { + service.log.Info().Str("image", remoteImageRef.CommonName()). + Msg("skipping image because it's already synced") + + return true, nil } - defer func() { - _ = policyContext.Destroy() - }() + return false, nil +} + +func (service *BaseService) syncTagAndReferrers(ctx context.Context, localRepo, remoteRepo, tag string) error { + var shouldCommit bool remoteImageRef, err := service.remote.GetImageReference(remoteRepo, tag) if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). Str("repository", remoteRepo).Str("reference", tag).Msg("couldn't get a remote image reference") - return "", err + return err } - _, mediaType, manifestDigest, err := service.remote.GetManifestContent(remoteImageRef) + defer service.regclient.Close(ctx, remoteImageRef) + + _, remoteManifestDesc, isConverted, err := service.remote.GetOCIManifest(ctx, remoteRepo, tag) if err != nil { service.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", tag). - Msg("couldn't get upstream image manifest details") + Msg("failed to get upstream image manifest details") - return "", err + return err } - if !isSupportedMediaType(mediaType) { - return "", zerr.ErrMediaTypeNotSupported + referrers, err := service.regclient.ReferrerList(ctx, remoteImageRef) + if err != nil { + return err } if service.config.OnlySigned != nil && *service.config.OnlySigned && - !references.IsCosignTag(tag) && !common.IsReferrersTag(tag) { - signed := service.references.IsSigned(ctx, remoteRepo, manifestDigest.String()) + !isCosignTag(tag) && !common.IsReferrersTag(tag) { + signed := hasSignatureReferrers(referrers) if !signed { // skip unsigned images - service.log.Info().Str("image", remoteImageRef.DockerReference().String()). + service.log.Info().Str("image", remoteImageRef.CommonName()). Msg("skipping image without mandatory signature") - return "", zerr.ErrSyncImageNotSigned + return zerr.ErrSyncImageNotSigned } } - skipImage, err := service.destination.CanSkipImage(destinationRepo, tag, manifestDigest) + localImageRef, err := service.destination.GetImageReference(localRepo, tag) if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag). - Msg("couldn't check if the local image can be skipped") + Str("repo", localRepo).Str("reference", localImageRef.Tag).Msg("failed to get a local image reference") + + return err } - if !skipImage { - localImageRef, err := service.destination.GetImageReference(destinationRepo, tag) - if err != nil { - service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag).Msg("couldn't get a local image reference") + defer service.regclient.Close(ctx, localImageRef) - return "", err - } + // first sync image + skipped, err := service.syncReference(ctx, localRepo, remoteImageRef, localImageRef, remoteManifestDesc.Digest) + if err != nil { + return err + } - service.log.Info().Str("remote image", remoteImageRef.DockerReference().String()). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)).Msg("syncing image") + shouldCommit = !skipped - _, err = copy.Image(ctx, policyContext, localImageRef, remoteImageRef, ©Options) - if err != nil { - // cleanup in cases of copy.Image errors while copying. - if cErr := service.destination.CleanupImage(localImageRef, destinationRepo, tag); cErr != nil { - service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)). - Msg("couldn't cleanup temp local image") - } + // if image was skipped, then copy it's referrers if needed (they may have changed in the meantime) + for _, desc := range referrers.Descriptors { + remoteImageRef = remoteImageRef.SetDigest(desc.Digest.String()) + localImageRef = localImageRef.SetDigest(desc.Digest.String()) + + skipped, err := service.syncReference(ctx, localRepo, remoteImageRef, localImageRef, desc.Digest) + if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("remote image", remoteImageRef.DockerReference().String()). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)).Msg("coulnd't sync image") + Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("failed to sync referrer") + } - return "", err + if skipped { + service.log.Info().Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("skipping syncing referrer because it's already synced") + } else { + shouldCommit = true } + } + + // convert image to oci if needed + if isConverted { + localImageRef, err = mod.Apply(ctx, service.regclient, localImageRef, + mod.WithRefTgt(localImageRef), + mod.WithManifestToOCI(), + mod.WithManifestToOCIReferrers(), + ) + if err != nil { + return err + } + + defer service.regclient.Close(ctx, localImageRef) + } - err = service.destination.CommitImage(localImageRef, destinationRepo, tag) + if shouldCommit { + err = service.destination.CommitAll(localRepo, localImageRef) if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag).Msg("couldn't commit image to local image store") + Str("repo", localRepo).Str("reference", tag).Msg("failed to commit image to local image store") - return "", err + return err } - } else { - service.log.Info().Str("image", remoteImageRef.DockerReference().String()). - Msg("skipping image because it's already synced") } - service.log.Info().Str("component", "sync"). - Str("image", remoteImageRef.DockerReference().String()).Msg("finished syncing image") + service.log.Info().Str("repo", localRepo).Str("reference", tag).Msg("successfully synced image") - return manifestDigest, nil + return nil } func (service *BaseService) ResetCatalog() { @@ -503,9 +500,3 @@ func (service *BaseService) ResetCatalog() { service.repositories = []string{} } - -func (service *BaseService) SetNextAvailableURL() error { - service.log.Info().Msg("getting available client") - - return service.SetNextAvailableClient() -} diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 1afd11172..125eb2d62 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -9,9 +9,9 @@ import ( "sync" "time" - "github.com/containers/common/pkg/retry" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/ref" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/scheduler" @@ -29,23 +29,14 @@ type Service interface { SyncRepo(ctx context.Context, repo string) error // used by periodically sync // Sync an image (repo:tag || repo:digest) into ImageStore. SyncImage(ctx context.Context, repo, reference string) error // used by sync on demand - // Sync a single reference for an image. - SyncReference(ctx context.Context, repo string, subjectDigestStr string, - referenceType string) error // used by sync on demand // Remove all internal catalog entries. ResetCatalog() // used by scheduler to empty out the catalog after a sync periodically roundtrip finishes - // Sync supports multiple urls per registry, before a sync repo/image/ref 'ping' each url. - SetNextAvailableURL() error // used by all sync methods - // Returns retry options from registry config. - GetRetryOptions() *retry.Options // used by sync on demand to retry in background } // Local and remote registries must implement this interface. type Registry interface { // Get temporary ImageReference, is used by functions in containers/image package - GetImageReference(repo string, tag string) (types.ImageReference, error) - // Get local oci layout context, is used by functions in containers/image package - GetContext() *types.SystemContext + GetImageReference(repo string, tag string) (ref.Ref, error) } /* @@ -59,26 +50,25 @@ type OciLayoutStorage interface { // Remote registry. type Remote interface { Registry + // Get host name + GetHostName() string // Get a list of repos (catalog) GetRepositories(ctx context.Context) ([]string, error) // Get a list of tags given a repo - GetRepoTags(repo string) ([]string, error) - // Get manifest content, mediaType, digest given an ImageReference - GetManifestContent(imageReference types.ImageReference) ([]byte, string, digest.Digest, error) - // In the case of public dockerhub images 'library' namespace is added to the repo names of images - // eg: alpine -> library/alpine - GetDockerRemoteRepo(repo string) string + GetTags(ctx context.Context, repo string) ([]string, error) + // Get manifest content, mediaType, descriptor given an image(if remote image is docker type then convert it to OCI) + GetOCIManifest(ctx context.Context, repo, reference string) ([]byte, ispec.Descriptor, bool, error) } // Local registry. type Destination interface { Registry - // Check if an image is already synced - CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error) - // CommitImage moves a synced repo/ref from temporary oci layout to ImageStore - CommitImage(imageReference types.ImageReference, repo, tag string) error + // Check if descriptors are already synced + CanSkipImage(repo string, tag string, digest godigest.Digest) (bool, error) + // CommitAll moves a synced repo and all its manifests from temporary oci layout to ImageStore + CommitAll(repo string, imageReference ref.Ref) error // Removes image reference, used when copy.Image() errors out - CleanupImage(imageReference types.ImageReference, repo, reference string) error + CleanupImage(imageReference ref.Ref, repo string) error } type TaskGenerator struct { @@ -117,12 +107,6 @@ func (gen *TaskGenerator) Next() (scheduler.Task, error) { return nil, nil //nolint:nilnil } - if err := gen.Service.SetNextAvailableURL(); err != nil { - gen.increaseWaitTime() - - return nil, err - } - repo, err := gen.Service.GetNextRepo(gen.lastRepo) if err != nil { gen.increaseWaitTime() diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 5a03ae0fa..74bc2d880 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -7,15 +7,9 @@ import ( "bytes" "context" "encoding/json" - "errors" - "fmt" "os" - "path" "testing" - dockerManifest "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" @@ -26,7 +20,6 @@ import ( syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" "zotregistry.dev/zot/pkg/extensions/lint" "zotregistry.dev/zot/pkg/extensions/monitoring" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/storage" @@ -36,48 +29,22 @@ import ( . "zotregistry.dev/zot/pkg/test/image-utils" "zotregistry.dev/zot/pkg/test/inject" "zotregistry.dev/zot/pkg/test/mocks" - ociutils "zotregistry.dev/zot/pkg/test/oci-utils" ) const ( testImage = "zot-test" testImageTag = "0.0.1" - - host = "127.0.0.1:45117" ) -var ErrTestError = errors.New("testError") - func TestInjectSyncUtils(t *testing.T) { - Convey("Inject errors in utils functions", t, func() { - repositoryReference := fmt.Sprintf("%s/%s", host, testImage) - ref, err := parseRepositoryReference(repositoryReference) - So(err, ShouldBeNil) - So(ref.Name(), ShouldEqual, repositoryReference) - - injected := inject.InjectFailure(0) - if injected { - _, err = getRepoTags(context.Background(), &types.SystemContext{}, host, testImage) - So(err, ShouldNotBeNil) - } - - injected = inject.InjectFailure(0) - _, err = getPolicyContext(log.NewLogger("debug", "")) - - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - + Convey("Inject errors in ols.GetImageReference", t, func() { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) imageStore := local.NewImageStore(t.TempDir(), false, false, log, metrics, nil, nil) - injected = inject.InjectFailure(0) + injected := inject.InjectFailure(0) ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore}) - _, err = ols.GetImageReference(testImage, testImageTag) - + _, err := ols.GetImageReference(testImage, testImageTag) if injected { So(err, ShouldNotBeNil) } else { @@ -86,87 +53,6 @@ func TestInjectSyncUtils(t *testing.T) { }) } -func TestNilDefaultStore(t *testing.T) { - Convey("Nil default store", t, func() { - ols := NewOciLayoutStorage(storage.StoreController{}) - _, err := ols.GetImageReference(testImage, testImageTag) - So(err, ShouldEqual, zerr.ErrLocalImgStoreNotFound) - }) -} - -func TestSyncInternal(t *testing.T) { - Convey("Verify parseRepositoryReference func", t, func() { - repositoryReference := fmt.Sprintf("%s/%s", host, testImage) - ref, err := parseRepositoryReference(repositoryReference) - So(err, ShouldBeNil) - So(ref.Name(), ShouldEqual, repositoryReference) - - repositoryReference = fmt.Sprintf("%s/%s:tagged", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldEqual, zerr.ErrInvalidRepositoryName) - - repositoryReference = fmt.Sprintf("http://%s/%s", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldNotBeNil) - - repositoryReference = fmt.Sprintf("docker://%s/%s", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldNotBeNil) - - _, err = getFileCredentials("/path/to/inexistent/file") - So(err, ShouldNotBeNil) - - tempFile, err := os.CreateTemp("", "sync-credentials-") - if err != nil { - panic(err) - } - - content := []byte(`{`) - if err := os.WriteFile(tempFile.Name(), content, 0o600); err != nil { - panic(err) - } - - _, err = getFileCredentials(tempFile.Name()) - So(err, ShouldNotBeNil) - - srcCtx := &types.SystemContext{} - _, err = getRepoTags(context.Background(), srcCtx, host, testImage) - So(err, ShouldNotBeNil) - - _, err = getRepoTags(context.Background(), srcCtx, host, testImage) - So(err, ShouldNotBeNil) - - _, err = getFileCredentials("/invalid/path/to/file") - So(err, ShouldNotBeNil) - - ok := isSupportedMediaType("unknown") - So(ok, ShouldBeFalse) - }) -} - -func TestRemoteRegistry(t *testing.T) { - Convey("test remote registry", t, func() { - logger := log.NewLogger("debug", "") - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, logger) - So(err, ShouldBeNil) - - remote := NewRemoteRegistry(client, logger) - imageRef, err := layout.NewReference("dir", "image") - So(err, ShouldBeNil) - _, _, _, err = remote.GetManifestContent(imageRef) - So(err, ShouldNotBeNil) - - tags, err := remote.GetRepoTags("repo") - So(tags, ShouldBeEmpty) - So(err, ShouldNotBeNil) - }) -} - func TestService(t *testing.T) { Convey("trigger fetch tags error", t, func() { conf := syncconf.RegistryConfig{ @@ -181,29 +67,6 @@ func TestService(t *testing.T) { }) } -func TestSyncRepo(t *testing.T) { - Convey("trigger context error", t, func() { - conf := syncconf.RegistryConfig{ - URLs: []string{"http://localhost"}, - } - - service, err := New(conf, "", nil, os.TempDir(), storage.StoreController{}, mocks.MetaDBMock{}, log.Logger{}) - So(err, ShouldBeNil) - - service.remote = mocks.SyncRemote{ - GetRepoTagsFn: func(repo string) ([]string, error) { - return []string{"repo1", "repo2"}, nil - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - err = service.SyncRepo(ctx, "repo") - So(err, ShouldEqual, ctx.Err()) - }) -} - func TestDestinationRegistry(t *testing.T) { Convey("make StoreController", t, func() { dir := t.TempDir() @@ -225,7 +88,7 @@ func TestDestinationRegistry(t *testing.T) { So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) - imgStore := getImageStoreFromImageReference(imageReference, repoName, "1.0", log) + imgStore := getImageStoreFromImageReference(repoName, imageReference, log) // create a blob/layer upload, err := imgStore.NewBlobUpload(repoName) @@ -313,7 +176,7 @@ func TestDestinationRegistry(t *testing.T) { So(ok, ShouldBeFalse) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) @@ -321,7 +184,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -340,7 +203,7 @@ func TestDestinationRegistry(t *testing.T) { storeController := storage.StoreController{DefaultStore: syncImgStore} registry := NewDestinationRegistry(storeController, storeController, nil, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) @@ -348,7 +211,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, digest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -356,7 +219,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, bdgst1), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -367,7 +230,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(syncImgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -383,7 +246,7 @@ func TestDestinationRegistry(t *testing.T) { }, }, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -395,7 +258,7 @@ func TestDestinationRegistry(t *testing.T) { }, }, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -404,7 +267,7 @@ func TestDestinationRegistry(t *testing.T) { So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) - imgStore := getImageStoreFromImageReference(imageReference, repoName, "2.0", log) + imgStore := getImageStoreFromImageReference(repoName, imageReference, log) // upload image @@ -473,206 +336,9 @@ func TestDestinationRegistry(t *testing.T) { So(ok, ShouldBeFalse) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "2.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) }) }) } - -func TestConvertDockerToOCI(t *testing.T) { - Convey("test converting docker to oci functions", t, func() { - dir := t.TempDir() - - srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log.NewLogger("debug", "")) - - err := WriteImageToFileSystem(CreateDefaultImage(), "zot-test", "0.0.1", srcStorageCtlr) - So(err, ShouldBeNil) - - imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1") - So(err, ShouldBeNil) - - imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) - So(err, ShouldBeNil) - - defer imageSource.Close() - - Convey("trigger Unmarshal manifest error", func() { - _, err = convertDockerManifestToOCI(imageSource, []byte{}) - So(err, ShouldNotBeNil) - }) - - Convey("trigger getImageConfigContent() error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger Unmarshal config error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), - []byte{}, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger convertDockerLayersToOCI error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBuf) - - manifest.Layers[0].MediaType = "unknown" - - newManifest, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()), - newManifest, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger convertDockerIndexToOCI error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - - // make zot-test image an index image - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - dockerNewManifest := ispec.Manifest{ - MediaType: dockerManifest.DockerV2Schema2MediaType, - Config: manifest.Config, - Layers: manifest.Layers, - } - - dockerNewManifestBuf, err := json.Marshal(dockerNewManifest) - So(err, ShouldBeNil) - - dockerManifestDigest := godigest.FromBytes(manifestBuf) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), - dockerNewManifestBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - var index ispec.Index - - index.Manifests = append(index.Manifests, ispec.Descriptor{ - Digest: dockerManifestDigest, - Size: int64(len(dockerNewManifestBuf)), - MediaType: dockerManifest.DockerV2Schema2MediaType, - }) - - index.MediaType = dockerManifest.DockerV2ListMediaType - - dockerIndexBuf, err := json.Marshal(index) - So(err, ShouldBeNil) - - dockerIndexDigest := godigest.FromBytes(dockerIndexBuf) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()), - dockerIndexBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - // write index.json - - var indexJSON ispec.Index - - indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json")) - So(err, ShouldBeNil) - - err = json.Unmarshal(indexJSONBuf, &indexJSON) - So(err, ShouldBeNil) - - indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{ - Digest: dockerIndexDigest, - Size: int64(len(dockerIndexBuf)), - MediaType: ispec.MediaTypeImageIndex, - Annotations: map[string]string{ - ispec.AnnotationRefName: "0.0.2", - }, - }) - - indexJSONBuf, err = json.Marshal(indexJSON) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2") - So(err, ShouldBeNil) - - imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) - So(err, ShouldNotBeNil) - - err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestConvertDockerLayersToOCI(t *testing.T) { - Convey("test converting docker to oci functions", t, func() { - dockerLayers := []ispec.Descriptor{ - { - MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType, - }, - { - MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip, - }, - { - MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed, - }, - { - MediaType: dockerManifest.DockerV2Schema2LayerMediaType, - }, - } - - err := convertDockerLayersToOCI(dockerLayers) - So(err, ShouldBeNil) - - So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck - So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck - So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer) - So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip) - }) -} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index d0fc8a79a..0bf83829b 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -74,6 +74,7 @@ var ( password = "test" //nolint: gochecknoglobals errSync = errors.New("sync error, src oci repo differs from dest one") errBadStatus = errors.New("bad http status") + ErrTestError = fmt.Errorf("testError") ) type TagsList struct { @@ -555,6 +556,7 @@ func TestOnDemand(t *testing.T) { hostname, err := os.Hostname() So(err, ShouldBeNil) + So(hostname, ShouldNotBeEmpty) syncRegistryConfig := syncconf.RegistryConfig{ Content: []syncconf.Content{ @@ -568,8 +570,10 @@ func TestOnDemand(t *testing.T) { }, // include self url, should be ignored URLs: []string{ - "http://" + hostname, destBaseURL, - srcBaseURL, "http://localhost:" + destPort, + fmt.Sprintf("http://%s:%s", hostname, destPort), //nolint:nosprintfhostport + destBaseURL, + srcBaseURL, + fmt.Sprintf("http://localhost:%s", destPort), }, TLSVerify: &tlsVerify, CertDir: "", @@ -602,7 +606,7 @@ func TestOnDemand(t *testing.T) { sm mTypes.SignatureMetadata, ) error { if sm.SignatureType == zcommon.CosignSignature || sm.SignatureType == zcommon.NotationSignature { - return sync.ErrTestError + return ErrTestError } return nil @@ -612,7 +616,7 @@ func TestOnDemand(t *testing.T) { (strings.HasSuffix(reference, remote.SignatureTagSuffix) || strings.HasSuffix(reference, remote.SBOMTagSuffix)) || strings.HasPrefix(reference, "sha256:") { - return sync.ErrTestError + return ErrTestError } // don't return err for normal image with tag @@ -717,8 +721,8 @@ func TestOnDemand(t *testing.T) { }, // include self url, should be ignored URLs: []string{ - "http://" + hostname, destBaseURL, - srcBaseURL, "http://localhost:" + destPort, + fmt.Sprintf("http://%s:%s", hostname, destPort), destBaseURL, //nolint:nosprintfhostport + srcBaseURL, fmt.Sprintf("http://localhost:%s", destPort), }, TLSVerify: &tlsVerify, CertDir: "", @@ -749,7 +753,7 @@ func TestOnDemand(t *testing.T) { dctlr.MetaDB = mocks.MetaDBMock{ SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error { if imageMeta.Digest.String() == ociRefImage.ManifestDescriptor.Digest.String() { - return sync.ErrTestError + return ErrTestError } return nil @@ -1968,7 +1972,7 @@ func TestPermsDenied(t *testing.T) { dcm.StartAndWait(destPort) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get a local image reference", 50*time.Second) + "failed to sync image", 50*time.Second) if err != nil { panic(err) } @@ -2343,7 +2347,7 @@ func TestMandatoryAnnotations(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't upload manifest because of missing annotations", 15*time.Second) + "failed to upload manifest because of missing annotations", 15*time.Second) if err != nil { panic(err) } @@ -2517,7 +2521,7 @@ func TestTLS(t *testing.T) { Registries: []syncconf.RegistryConfig{syncRegistryConfig}, } - dctlr, _, destDir, _ := makeDownstreamServer(t, true, syncConfig) + dctlr, destBaseURL, destDir, destClient := makeDownstreamServer(t, true, syncConfig) dcm := test.NewControllerManager(dctlr) dcm.StartAndWait(dctlr.Config.HTTP.Port) @@ -2549,6 +2553,10 @@ func TestTLS(t *testing.T) { } waitSyncFinish(dctlr.Config.Log.Output) + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) }) } @@ -2996,7 +3004,7 @@ func TestBasicAuth(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "authentication required", 15*time.Second) + "unauthorized", 15*time.Second) if err != nil { panic(err) } @@ -4114,7 +4122,7 @@ func TestMultipleURLs(t *testing.T) { }, }, }, - URLs: []string{"badURL", "@!#!$#@%", "http://invalid.invalid/invalid/", srcBaseURL}, + URLs: []string{"http://badURL", srcBaseURL}, PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", @@ -4307,7 +4315,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to get upstream image manifest details", 60*time.Second) if err != nil { panic(err) } @@ -4355,7 +4363,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to sync image", 60*time.Second) if err != nil { panic(err) } @@ -4425,7 +4433,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to sync image", 30*time.Second) if err != nil { panic(err) } @@ -4506,7 +4514,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't sync image referrer", 15*time.Second) + "failed to sync image", 30*time.Second) if err != nil { panic(err) } @@ -4563,7 +4571,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't sync image referrer", 15*time.Second) + "failed to sync image", 30*time.Second) if err != nil { panic(err) } @@ -4591,72 +4599,18 @@ func TestPeriodicallySignaturesErr(t *testing.T) { So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 0) }) - - Convey("of type OCI image, error on downstream in canSkipReference()", func() { //nolint: dupl - // start downstream server - updateDuration, err = time.ParseDuration("1s") - So(err, ShouldBeNil) - - syncConfig.Registries[0].PollInterval = updateDuration - dctlr, _, destDir, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - - time.Sleep(time.Second) - - blob := referrers.Manifests[0] - blobsDir := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm())) - blobPath := path.Join(blobsDir, blob.Digest.Encoded()) - err = os.MkdirAll(blobsDir, storageConstants.DefaultDirPerms) - So(err, ShouldBeNil) - err = os.WriteFile(blobPath, []byte("blob"), storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - err = os.Chmod(blobPath, 0o000) - So(err, ShouldBeNil) - - found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't check if the upstream oci references for image can be skipped", 30*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - }) }) }) } func TestSignatures(t *testing.T) { Convey("Verify sync signatures", t, func() { - updateDuration, _ := time.ParseDuration("30m") + // updateDuration, _ := time.ParseDuration("30m") sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false) scm := test.NewControllerManager(sctlr) + scm.StartAndWait(sctlr.Config.HTTP.Port) defer scm.StopServer() @@ -4760,12 +4714,12 @@ func TestSignatures(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - PollInterval: updateDuration, - TLSVerify: &tlsVerify, - CertDir: "", - OnlySigned: &onlySigned, - OnDemand: true, + URLs: []string{srcBaseURL}, + // PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + OnlySigned: &onlySigned, + OnDemand: true, } defaultVal := true @@ -4781,31 +4735,15 @@ func TestSignatures(t *testing.T) { defer dcm.StopServer() - // wait for sync - var destTagsList TagsList - - for { - resp, err := destClient.R().Get(destBaseURL + "/v2/" + repoName + "/tags/list") - if err != nil { - panic(err) - } - - err = json.Unmarshal(resp.Body(), &destTagsList) - if err != nil { - panic(err) - } - - if len(destTagsList.Tags) > 0 { - break - } - - time.Sleep(500 * time.Millisecond) - } + // sync image with all its refs + resp, err = destClient.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) splittedURL = strings.SplitAfter(destBaseURL, ":") destPort := splittedURL[len(splittedURL)-1] - time.Sleep(1 * time.Second) + time.Sleep(5 * time.Second) // notation verify the image image := fmt.Sprintf("localhost:%s/%s@%s", destPort, repoName, digest) @@ -5295,7 +5233,6 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { Convey("Verify that metadb update correctly when syncing a signature", t, func() { repoName := "signed-repo" tag := "random-signed-image" - updateDuration := 30 * time.Minute // Create source registry @@ -5341,11 +5278,10 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { Tags: &syncconf.Tags{Regex: ®ex, Semver: &semver}, }, }, - URLs: []string{srcBaseURL}, - PollInterval: updateDuration, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, }, }, } @@ -5363,10 +5299,12 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) + // regclient will put all referrers under ref tag "alg-subjectDigest" repoMeta, err := dctlr.MetaDB.GetRepoMeta(context.Background(), repoName) So(err, ShouldBeNil) So(repoMeta.Tags, ShouldContainKey, tag) - So(len(repoMeta.Tags), ShouldEqual, 1) + // one tag for refs and the tag we pushed earlier + So(len(repoMeta.Tags), ShouldEqual, 2) So(repoMeta.Signatures, ShouldContainKey, signedImage.DigestStr()) imageSignatures := repoMeta.Signatures[signedImage.DigestStr()] @@ -5578,7 +5516,7 @@ func TestOnDemandRetryGoroutineErr(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "failed to copy image", 15*time.Second) + "failed to sync image", 15*time.Second) if err != nil { panic(err) } @@ -5701,7 +5639,7 @@ func TestOnDemandMultipleImage(t *testing.T) { time.Sleep(500 * time.Millisecond) } - waitSync(destDir, testImage) + // waitSync(destDir, testImage) So(len(populatedDirs), ShouldEqual, 1) @@ -6036,7 +5974,7 @@ func TestSignaturesOnDemand(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusOK) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't find any oci reference", 15*time.Second) + "failed to sync referrer", 15*time.Second) if err != nil { panic(err) } @@ -6614,7 +6552,7 @@ func TestSyncSignaturesDiff(t *testing.T) { So(reflect.DeepEqual(cosignManifest, syncedCosignManifest), ShouldEqual, true) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "skipping syncing cosign reference", 15*time.Second) + "skipping syncing referrer because it's already synced", 15*time.Second) if err != nil { panic(err) } @@ -6629,7 +6567,7 @@ func TestSyncSignaturesDiff(t *testing.T) { So(found, ShouldBeTrue) found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "skipping oci references", 15*time.Second) + "skipping syncing referrer because it's already synced", 15*time.Second) if err != nil { panic(err) } diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 16aab5e69..95281344b 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -4,29 +4,16 @@ package sync import ( - "bytes" - "context" "encoding/json" - "fmt" - "io" + "net/url" "os" + "path" "strings" - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/docker" - "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/pkg/blobinfocache/none" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/test/inject" ) // Get sync.FileCredentials from file. @@ -46,71 +33,6 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { return creds, nil } -func getUpstreamContext(certDir, username, password string, tlsVerify bool) *types.SystemContext { - upstreamCtx := &types.SystemContext{} - upstreamCtx.DockerCertPath = certDir - upstreamCtx.DockerDaemonCertPath = certDir - - if tlsVerify { - upstreamCtx.DockerDaemonInsecureSkipTLSVerify = false - upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(false) - } else { - upstreamCtx.DockerDaemonInsecureSkipTLSVerify = true - upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) - } - - if username != "" && password != "" { - upstreamCtx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: username, - Password: password, - } - } - - return upstreamCtx -} - -// sync needs transport to be stripped to not be wrongly interpreted as an image reference -// at a non-fully qualified registry (hostname as image and port as tag). -func StripRegistryTransport(url string) string { - return strings.Replace(strings.Replace(url, "http://", "", 1), "https://", "", 1) -} - -// getRepoTags lists all tags in a repository. -// It returns a string slice of tags and any error encountered. -func getRepoTags(ctx context.Context, sysCtx *types.SystemContext, host, repo string) ([]string, error) { - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", host, repo)) - if err != nil { - return []string{}, err - } - - dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) - // hard to reach test case, injected error, see pkg/test/dev.go - if err = inject.Error(err); err != nil { - return nil, err // Should never happen for a reference with tag and no digest - } - - tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef) - if err != nil { - return nil, err - } - - return tags, nil -} - -// parseRepositoryReference parses input into a reference.Named, and verifies that it names a repository, not an image. -func parseRepositoryReference(input string) (reference.Named, error) { - ref, err := reference.ParseNormalizedNamed(input) - if err != nil { - return nil, err - } - - if !reference.IsNameOnly(ref) { - return nil, zerr.ErrInvalidRepositoryName - } - - return ref, nil -} - // parse a reference, return its digest and if it's valid. func parseReference(reference string) (digest.Digest, bool) { var ok bool @@ -123,176 +45,89 @@ func parseReference(reference string) (digest.Digest, bool) { return d, ok } -func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { - options := copy.Options{ - DestinationCtx: localCtx, - SourceCtx: upstreamCtx, - ReportWriter: io.Discard, - ForceManifestMIMEType: ispec.MediaTypeImageManifest, // force only oci manifest MIME type - ImageListSelection: copy.CopyAllImages, - } - - return options -} +// Given a list of registry string URLs parse them and return *url.URLs slice. +func parseRegistryURLs(rawURLs []string) ([]*url.URL, error) { + urls := make([]*url.URL, 0) -func getPolicyContext(log log.Logger) (*signature.PolicyContext, error) { - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - - policyContext, err := signature.NewPolicyContext(policy) - if err := inject.Error(err); err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't create policy context") + for _, rawURL := range rawURLs { + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } - return nil, err + urls = append(urls, u) } - return policyContext, nil -} - -func getSupportedMediaType() []string { - return []string{ - ispec.MediaTypeImageIndex, - ispec.MediaTypeImageManifest, - manifest.DockerV2ListMediaType, - manifest.DockerV2Schema2MediaType, - } + return urls, nil } -func isSupportedMediaType(mediaType string) bool { - mediaTypes := getSupportedMediaType() - for _, m := range mediaTypes { - if m == mediaType { - return true - } +func GetDescriptorReference(desc ispec.Descriptor) string { + v, ok := desc.Annotations[ispec.AnnotationRefName] + if ok { + return v } - return false + return desc.Digest.String() } -// given an imageSource and a docker manifest, convert it to OCI. -func convertDockerManifestToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { - var ociManifest ispec.Manifest - - // unmarshal docker manifest into OCI manifest - err := json.Unmarshal(dockerManifestBuf, &ociManifest) - if err != nil { - return []byte{}, err - } - - configContent, err := getImageConfigContent(imageSource, ociManifest.Config.Digest) - if err != nil { - return []byte{}, err - } +func StripRegistryTransport(url string) string { + return strings.Replace(strings.Replace(url, "http://", "", 1), "https://", "", 1) +} - // marshal config blob into OCI config, will remove keys specific to docker - var ociConfig ispec.Image +func getCertificates(certDir string) (string, string, string, error) { + var clientCert string - err = json.Unmarshal(configContent, &ociConfig) - if err != nil { - return []byte{}, err - } + var clientKey string - ociConfigContent, err := json.Marshal(ociConfig) - if err != nil { - return []byte{}, err - } + var regCert string - // convert layers - err = convertDockerLayersToOCI(ociManifest.Layers) + files, err := os.ReadDir(certDir) if err != nil { - return []byte{}, err - } - - // convert config and manifest mediatype - ociManifest.Config.Size = int64(len(ociConfigContent)) - ociManifest.Config.Digest = digest.FromBytes(ociConfigContent) - ociManifest.Config.MediaType = ispec.MediaTypeImageConfig - ociManifest.MediaType = ispec.MediaTypeImageManifest - - return json.Marshal(ociManifest) -} - -// convert docker layers mediatypes to OCI mediatypes. -func convertDockerLayersToOCI(dockerLayers []ispec.Descriptor) error { - for idx, layer := range dockerLayers { - switch layer.MediaType { - case manifest.DockerV2Schema2ForeignLayerMediaType: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck - case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributableGzip //nolint: staticcheck - case manifest.DockerV2SchemaLayerMediaTypeUncompressed: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayer - case manifest.DockerV2Schema2LayerMediaType: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerGzip - default: - return zerr.ErrMediaTypeNotSupported + if os.IsNotExist(err) { + return "", "", "", nil } - } - - return nil -} -// given an imageSource and a docker index manifest, convert it to OCI. -func convertDockerIndexToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { - // get docker index - originalIndex, err := manifest.ListFromBlob(dockerManifestBuf, manifest.DockerV2ListMediaType) - if err != nil { - return []byte{}, err + return "", "", "", err } - // get manifests digests - manifestsDigests := originalIndex.Instances() + for _, file := range files { + if file.IsDir() { + continue + } - manifestsUpdates := make([]manifest.ListUpdate, 0, len(manifestsDigests)) + if strings.HasSuffix(file.Name(), ".cert") { + certPath := path.Join(certDir, file.Name()) - // convert each manifests in index from docker to OCI - for _, manifestDigest := range manifestsDigests { - digestCopy := manifestDigest + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } - indexManifestBuf, _, err := imageSource.GetManifest(context.Background(), &digestCopy) - if err != nil { - return []byte{}, err + clientCert = string(buf) } - convertedIndexManifest, err := convertDockerManifestToOCI(imageSource, indexManifestBuf) - if err != nil { - return []byte{}, err - } + if strings.HasSuffix(file.Name(), ".key") { + certPath := path.Join(certDir, file.Name()) - manifestsUpdates = append(manifestsUpdates, manifest.ListUpdate{ - Digest: digest.FromBytes(convertedIndexManifest), - Size: int64(len(convertedIndexManifest)), - MediaType: ispec.MediaTypeImageManifest, - }) - } + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } - // update all manifests in index - if err := originalIndex.UpdateInstances(manifestsUpdates); err != nil { - return []byte{}, err - } + clientKey = string(buf) + } - // convert index to OCI - convertedList, err := originalIndex.ConvertToMIMEType(ispec.MediaTypeImageIndex) - if err != nil { - return []byte{}, err - } + if strings.HasSuffix(file.Name(), ".crt") { + certPath := path.Join(certDir, file.Name()) - return convertedList.Serialize() -} + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } -// given an image source and a config blob digest, get blob config content. -func getImageConfigContent(imageSource types.ImageSource, configDigest digest.Digest, -) ([]byte, error) { - configBlob, _, err := imageSource.GetBlob(context.Background(), types.BlobInfo{ - Digest: configDigest, - }, none.NoCache) - if err != nil { - return nil, err + regCert = string(buf) + } } - configBuf := new(bytes.Buffer) - - _, err = configBuf.ReadFrom(configBlob) - - return configBuf.Bytes(), err + return clientCert, clientKey, regCert, nil } diff --git a/pkg/test/auth/bearer.go b/pkg/test/auth/bearer.go index f305d5b02..87da7a922 100644 --- a/pkg/test/auth/bearer.go +++ b/pkg/test/auth/bearer.go @@ -36,10 +36,19 @@ func MakeAuthTestServer(serverKey string, unauthorizedNamespace string) *httptes } authTestServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet { + response.WriteHeader(http.StatusMethodNotAllowed) + + return + } + var access []auth.AccessEntry + scopes := request.URL.Query()["scope"] + for _, scope := range scopes { + if scope == "" { + continue + } - scope := request.URL.Query().Get("scope") - if scope != "" { parts := strings.Split(scope, ":") name := parts[1] actions := strings.Split(parts[2], ",") @@ -47,14 +56,11 @@ func MakeAuthTestServer(serverKey string, unauthorizedNamespace string) *httptes if name == unauthorizedNamespace { actions = []string{} } - - access = []auth.AccessEntry{ - { - Name: name, - Type: "repository", - Actions: actions, - }, - } + access = append(access, auth.AccessEntry{ + Name: name, + Type: "repository", + Actions: actions, + }) } token, err := cmTokenGenerator.GenerateToken(access, time.Minute*1)