feat(sync): use regclient for sync extension

replaced containers/image package with regclient/regclient package

Signed-off-by: Petu Eusebiu <[email protected]>

go.mod

@@ -116,7 +116,6 @@ require (
 	github.com/ProtonMail/go-crypto v1.1.0-alpha.5-proton // indirect
 	github.com/ThalesIgnite/crypto11 v1.2.5 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
-	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect

go.sum

@@ -292,8 +292,6 @@ github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY
 github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
@@ -1209,6 +1207,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olareg/olareg v0.1.0 h1:1dXBOgPrig5N7zoXyIZVQqU0QBo6sD9pbL6UYjY75CA=
+github.com/olareg/olareg v0.1.0/go.mod h1:RBuU7JW7SoIIxZKzLRhq8sVtQeAHzCAtRrXEBx2KlM4=
 github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM=
 github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -1273,8 +1273,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
 github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
-github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0=
-github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
 github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a h1:525aNEKSyDcqJcawiGtA2NPNApJMta8bUe9SoYuhQ+o=
 github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a/go.mod h1:ltIE6ZO/czh/g4xdNQlFGkl7DAfaLLFYmitB4taA5ys=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1452,8 +1450,6 @@ github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64
 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
 github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
 github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
 github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes=

pkg/api/controller.go

@@ -520,5 +520,4 @@ func (c *Controller) StartBackgroundTasks() {
 
 type SyncOnDemand interface {
 	SyncImage(ctx context.Context, repo, reference string) error
-	SyncReference(ctx context.Context, repo string, subjectDigestStr string, referenceType string) error
 }

pkg/api/routes.go

@@ -41,7 +41,6 @@ import (
 	"zotregistry.dev/zot/pkg/debug/pprof"
 	debug "zotregistry.dev/zot/pkg/debug/swagger"
 	ext "zotregistry.dev/zot/pkg/extensions"
-	syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants"
 	"zotregistry.dev/zot/pkg/log"
 	"zotregistry.dev/zot/pkg/meta"
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
@@ -568,8 +567,7 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler,
 			routeHandler.c.Log.Info().Str("repository", name).Str("reference", digest.String()).
 				Msg("referrers not found, trying to get reference by syncing on demand")
 
-			if errSync := routeHandler.c.SyncOnDemand.SyncReference(ctx, name, digest.String(),
-				syncConstants.OCI); errSync != nil {
+			if errSync := routeHandler.c.SyncOnDemand.SyncImage(ctx, name, digest.String()); errSync != nil {
 				routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", digest.String()).
 					Msg("failed to sync OCI reference for image")
 			}

pkg/extensions/extension_sync.go

@@ -102,17 +102,15 @@ func getLocalIPs() ([]string, error) {
 	return localIPs, nil
 }
 
-func getIPFromHostName(host string) ([]string, error) {
+func getIPFromHostName(host string) ([]net.IP, error) {
 	addrs, err := net.LookupIP(host)
 	if err != nil {
-		return []string{}, err
+		return []net.IP{}, err
 	}
 
-	ips := make([]string, 0, len(addrs))
+	ips := make([]net.IP, 0, len(addrs))
 
-	for _, ip := range addrs {
-		ips = append(ips, ip.String())
-	}
+	ips = append(ips, addrs...)
 
 	return ips, nil
 }
@@ -163,7 +161,8 @@ func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConf
 		for _, localIP := range localIPs {
 			// if ip resolved from hostname/dns is equal with any local ip
 			for _, ip := range ips {
-				if net.JoinHostPort(ip, url.Port()) == net.JoinHostPort(localIP, port) {
+				if (ip.IsLoopback() && (url.Port() == port)) ||
+					(net.JoinHostPort(ip.String(), url.Port()) == net.JoinHostPort(localIP, port)) {
 					registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...)
 
 					removed = true

pkg/extensions/sync/content_internal_test.go → pkg/extensions/sync/content_test.go

@@ -1,7 +1,7 @@
 //go:build sync
 // +build sync
 
-package sync
+package sync //nolint: testpackage
 
 import (
 	"testing"

pkg/extensions/sync/destination.go

@@ -13,9 +13,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/containers/image/v5/types"
-	"github.com/opencontainers/go-digest"
+	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/regclient/regclient/types/ref"
 
 	zerr "zotregistry.dev/zot/errors"
 	"zotregistry.dev/zot/pkg/common"
@@ -52,7 +52,8 @@ func NewDestinationRegistry(
 	}
 }
 
-func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error) {
+// Check if image is already synced.
+func (registry *DestinationRegistry) CanSkipImage(repo, tag string, digest godigest.Digest) (bool, error) {
 	// check image already synced
 	imageStore := registry.storeController.GetImageStore(repo)
 
@@ -68,10 +69,10 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest
 		return false, err
 	}
 
-	if localImageManifestDigest != imageDigest {
+	if localImageManifestDigest != digest {
 		registry.log.Info().Str("repo", repo).Str("reference", tag).
 			Str("localDigest", localImageManifestDigest.String()).
-			Str("remoteDigest", imageDigest.String()).
+			Str("remoteDigest", digest.String()).
 			Msg("remote image digest changed, syncing again")
 
 		return false, nil
@@ -80,116 +81,121 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest
 	return true, nil
 }
 
-func (registry *DestinationRegistry) GetContext() *types.SystemContext {
-	return registry.tempStorage.GetContext()
-}
-
-func (registry *DestinationRegistry) GetImageReference(repo, reference string) (types.ImageReference, error) {
+func (registry *DestinationRegistry) GetImageReference(repo, reference string) (ref.Ref, error) {
 	return registry.tempStorage.GetImageReference(repo, reference)
 }
 
 // finalize a syncing image.
-func (registry *DestinationRegistry) CommitImage(imageReference types.ImageReference, repo, reference string) error {
+func (registry *DestinationRegistry) CommitAll(repo string, imageReference ref.Ref) error {
 	imageStore := registry.storeController.GetImageStore(repo)
 
 	tempImageStore := getImageStoreFromImageReference(imageReference, repo, reference, registry.log)
 
 	defer os.RemoveAll(tempImageStore.RootDir())
 
-	registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("reference", reference).
+	registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("repository", repo).
 		Msg("pushing synced local image to local registry")
 
 	var lockLatency time.Time
 
-	manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference)
+	index, err := storageCommon.GetIndex(tempImageStore, repo, registry.log)
 	if err != nil {
 		registry.log.Error().Str("errorType", common.TypeOf(err)).
-			Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference).
-			Msg("couldn't find synced manifest in temporary sync dir")
+			Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).
+			Msg("failed to get repo index from temp sync dir")
 
 		return err
 	}
 
-	// is image manifest
-	switch mediaType {
-	case ispec.MediaTypeImageManifest:
-		if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil {
-			if errors.Is(err, zerr.ErrImageLintAnnotations) {
-				registry.log.Error().Str("errorType", common.TypeOf(err)).
-					Err(err).Msg("couldn't upload manifest because of missing annotations")
+	for _, desc := range index.Manifests {
+		reference := GetDescriptorReference(desc)
 
-				return nil
-			}
+		manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference)
+		if err != nil {
+			registry.log.Error().Str("errorType", common.TypeOf(err)).
+				Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference).
+				Msg("failed to get manifest from temporary sync dir")
 
 			return err
 		}
-	case ispec.MediaTypeImageIndex:
-		// is image index
-		var indexManifest ispec.Index
 
-		if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil {
-			registry.log.Error().Str("errorType", common.TypeOf(err)).
-				Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).
-				Msg("invalid JSON")
+		// is image manifest
+		switch mediaType {
+		case ispec.MediaTypeImageManifest:
+			if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil {
+				if errors.Is(err, zerr.ErrImageLintAnnotations) {
+					registry.log.Error().Str("errorType", common.TypeOf(err)).
+						Err(err).Msg("failed to upload manifest because of missing annotations")
 
-			return err
-		}
+					return nil
+				}
 
-		for _, manifest := range indexManifest.Manifests {
-			tempImageStore.RLock(&lockLatency)
-			manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest)
-			tempImageStore.RUnlock(&lockLatency)
+				return err
+			}
+		case ispec.MediaTypeImageIndex:
+			// is image index
+			var indexManifest ispec.Index
 
-			if err != nil {
+			if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil {
 				registry.log.Error().Str("errorType", common.TypeOf(err)).
-					Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()).
-					Msg("couldn't find manifest which is part of an image index")
+					Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).
+					Msg("invalid JSON")
 
 				return err
 			}
 
-			if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(),
-				tempImageStore); err != nil {
-				if errors.Is(err, zerr.ErrImageLintAnnotations) {
+			for _, manifest := range indexManifest.Manifests {
+				tempImageStore.RLock(&lockLatency)
+				manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest)
+				tempImageStore.RUnlock(&lockLatency)
+
+				if err != nil {
 					registry.log.Error().Str("errorType", common.TypeOf(err)).
-						Err(err).Msg("couldn't upload manifest because of missing annotations")
+						Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()).
+						Msg("failed find manifest which is part of an image index")
 
-					return nil
+					return err
 				}
 
-				return err
-			}
-		}
+				if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(),
+					tempImageStore); err != nil {
+					if errors.Is(err, zerr.ErrImageLintAnnotations) {
+						registry.log.Error().Str("errorType", common.TypeOf(err)).
+							Err(err).Msg("failed to upload manifest because of missing annotations")
 
-		_, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob)
-		if err != nil {
-			registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference).
-				Err(err).Msg("couldn't upload manifest")
+						return nil
+					}
 
-			return err
-		}
+					return err
+				}
+			}
 
-		if registry.metaDB != nil {
-			err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType,
-				manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log)
+			_, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob)
 			if err != nil {
-				return fmt.Errorf("failed to set metadata for image '%s %s': %w", repo, reference, err)
+				registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference).
+					Err(err).Msg("failed to upload manifest")
+
+				return err
 			}
 
-			registry.log.Debug().Str("repo", repo).Str("reference", reference).Str("component", "metadb").
-				Msg("successfully set metadata for image")
+			if registry.metaDB != nil {
+				err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType,
+					manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log)
+				if err != nil {
+					return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err)
+				}
+
+				registry.log.Debug().Str("repo", repo).Str("reference", reference).
+					Msg("metaDB: successfully set metadata for image")
+			}
 		}
 	}
 
-	registry.log.Info().Str("image", fmt.Sprintf("%s:%s", repo, reference)).Msg("successfully synced image")
-
 	return nil
 }
 
-func (registry *DestinationRegistry) CleanupImage(imageReference types.ImageReference, repo, reference string) error {
-	tmpDir := getTempRootDirFromImageReference(imageReference, repo, reference)
-
-	return os.RemoveAll(tmpDir)
+func (registry *DestinationRegistry) CleanupImage(imageReference ref.Ref, repo string) error {
+	return os.RemoveAll(strings.TrimSuffix(imageReference.Path, repo))
 }
 
 func (registry *DestinationRegistry) copyManifest(repo string, manifestContent []byte, reference string,
@@ -251,7 +257,7 @@ func (registry *DestinationRegistry) copyManifest(repo string, manifestContent [
 }
 
 // Copy a blob from one image store to another image store.
-func (registry *DestinationRegistry) copyBlob(repo string, blobDigest digest.Digest, blobMediaType string,
+func (registry *DestinationRegistry) copyBlob(repo string, blobDigest godigest.Digest, blobMediaType string,
 	tempImageStore storageTypes.ImageStore,
 ) error {
 	imageStore := registry.storeController.GetImageStore(repo)