From 6881b118268d2df16c3bb6c914c35e080227a519 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 3 Nov 2023 03:21:02 -0500 Subject: [PATCH] feat: add redis cache support https://github.com/project-zot/zot/pull/2005 Fixes https://github.com/project-zot/zot/issues/2004 --- go.mod | 4 + go.sum | 16 ++- pkg/storage/cache.go | 4 + pkg/storage/cache/redis.go | 196 ++++++++++++++++++++++++++++++++ pkg/storage/cache/redis_test.go | 124 ++++++++++++++++++++ 5 files changed, 338 insertions(+), 6 deletions(-) create mode 100644 pkg/storage/cache/redis.go create mode 100644 pkg/storage/cache/redis_test.go diff --git a/go.mod b/go.mod index fcb54ca08..7b1dee1e7 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 require ( github.com/99designs/gqlgen v0.17.61 github.com/Masterminds/semver v1.5.0 + github.com/alicebob/miniredis/v2 v2.34.0 github.com/aquasecurity/trivy v0.58.1 github.com/aquasecurity/trivy-db v0.0.0-20241209111357-8c398f13db0e github.com/aws/aws-sdk-go v1.55.5 @@ -48,6 +49,7 @@ require ( github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 + github.com/redis/go-redis/v9 v9.7.0 github.com/rs/zerolog v1.33.0 github.com/sigstore/cosign/v2 v2.4.1 github.com/sigstore/sigstore v1.8.11 @@ -137,6 +139,7 @@ require ( github.com/alibabacloud-go/tea-utils v1.4.5 // indirect github.com/alibabacloud-go/tea-utils/v2 v2.0.6 // indirect github.com/alibabacloud-go/tea-xml v1.1.3 // indirect + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/aliyun/credentials-go v1.3.6 // indirect github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect @@ -460,6 +463,7 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/zclconf/go-cty v1.15.0 // indirect github.com/zclconf/go-cty-yaml v1.1.0 // indirect github.com/zeebo/errs v1.3.0 // indirect diff --git a/go.sum b/go.sum index 1cf363a29..2492f5b46 100644 --- a/go.sum +++ b/go.sum @@ -369,10 +369,10 @@ github.com/alibabacloud-go/tea-utils/v2 v2.0.6/go.mod h1:qxn986l+q33J5VkialKMqT/ github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0= github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUiji6lZrA= -github.com/alicebob/miniredis/v2 v2.33.0/go.mod h1:MhP4a3EU7aENRi9aO+tHfTBZicLqQevyi/DJpoj6mi0= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= +github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= github.com/aliyun/credentials-go v1.3.6 h1:K5STbhaWjoj5Ht0juOj9mWE2lGelShHLzu5QR3cQ5X8= github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= @@ -497,6 +497,10 @@ github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Pt github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buildkite/agent/v3 v3.81.0 h1:JVfkng2XnsXesFXwiFwLJFkuzVu4zvoJCvedfoIXD6E= github.com/buildkite/agent/v3 v3.81.0/go.mod h1:edJeyycODRxaFvpT22rDGwaQ5oa4eB8GjtbjgX5VpFw= github.com/buildkite/go-pipeline v0.13.1 h1:Y9p8pQIwPtauVwNrcmTDH6+XK7jE1nLuvWVaK8oymA8= @@ -1353,8 +1357,8 @@ github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fO github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= -github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= -github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go index eed2ab4d6..6cc71ae86 100644 --- a/pkg/storage/cache.go +++ b/pkg/storage/cache.go @@ -60,6 +60,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache { return cache.NewDynamoDBCache(parameters, log) } + case "redis": + { + return cache.NewRedisCache(parameters, log), nil + } default: { return nil, zerr.ErrBadConfig diff --git a/pkg/storage/cache/redis.go b/pkg/storage/cache/redis.go new file mode 100644 index 000000000..ad9fcfcc9 --- /dev/null +++ b/pkg/storage/cache/redis.go @@ -0,0 +1,196 @@ +package cache + +import ( + "context" + goerrors "errors" + "path/filepath" + "strings" + + godigest "github.com/opencontainers/go-digest" + "github.com/redis/go-redis/v9" + + "zotregistry.dev/zot/errors" + zlog "zotregistry.dev/zot/pkg/log" + "zotregistry.dev/zot/pkg/storage/constants" +) + +type RedisDriver struct { + rootDir string + db redis.UniversalClient + log zlog.Logger + useRelPaths bool // whether or not to use relative paths, should be true for filesystem and false for s3 +} + +type RedisDriverParameters struct { + RootDir string + Url string // https://github.com/redis/redis-specifications/blob/master/uri/redis.txt + UseRelPaths bool +} + +func NewRedisCache(parameters interface{}, log zlog.Logger) Cache { + properParameters, ok := parameters.(RedisDriverParameters) + if !ok { + panic("Failed type assertion") + } + + connOpts, err := redis.ParseURL(properParameters.Url) + if err != nil { + log.Error().Err(err).Str("directory", properParameters.Url).Msg("unable to connect to redis") + } + cacheDB := redis.NewClient(connOpts) + + if _, err := cacheDB.Ping(context.Background()).Result(); err != nil { + log.Error().Err(err).Msg("unable to ping redis cache") + return nil + } + + return &RedisDriver{ + db: cacheDB, + log: log, + rootDir: properParameters.RootDir, + useRelPaths: properParameters.UseRelPaths, + } +} + +func join(xs ...string) string { + return "zot:" + strings.Join(xs, ":") +} + +func (d *RedisDriver) UsesRelativePaths() bool { + return d.useRelPaths +} + +func (d *RedisDriver) Name() string { + return "redis" +} + +func (d *RedisDriver) PutBlob(digest godigest.Digest, path string) error { + ctx := context.TODO() + if path == "" { + d.log.Error().Err(errors.ErrEmptyValue).Str("digest", digest.String()).Msg("empty path provided") + return errors.ErrEmptyValue + } + + // use only relative (to rootDir) paths on blobs + var err error + if d.useRelPaths { + path, err = filepath.Rel(d.rootDir, path) + if err != nil { + d.log.Error().Err(err).Str("path", path).Msg("unable to get relative path") + } + } + if len(path) == 0 { + return errors.ErrEmptyValue + } + // see if the blob digest exists. + exists, err := d.db.HExists(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + return err + } + if _, err := d.db.TxPipelined(ctx, func(tx redis.Pipeliner) error { + if !exists { + // add the key value pair [digest, path] to blobs:origin if not exist already. the path becomes the canonical blob + // we do this in a transaction to make sure that if something is in the set, then it is guaranteed to always have a path + // note that there is a race, but the worst case is that a different origin path that is still valid is used. + if err := tx.HSet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String(), path).Err(); err != nil { + d.log.Error().Err(err).Str("hset", join(constants.BlobsCache, constants.OriginalBucket)).Str("value", path).Msg("unable to put record") + return err + } + } + // add path to the set of paths which the digest represents + if err := d.db.SAdd(ctx, join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()), path).Err(); err != nil { + d.log.Error().Err(err).Str("sadd", join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())).Str("value", path).Msg("unable to put record") + return err + } + return nil + }); err != nil { + return err + } + + return nil +} + +func (d *RedisDriver) GetBlob(digest godigest.Digest) (string, error) { + ctx := context.TODO() + path, err := d.db.HGet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + if goerrors.Is(err, redis.Nil) { + return "", errors.ErrCacheMiss + } + d.log.Error().Err(err).Str("hget", join(constants.BlobsCache, constants.OriginalBucket)).Str("digest", digest.String()).Msg("unable to get record") + return "", err + } + return path, nil +} + +func (d *RedisDriver) HasBlob(digest godigest.Digest, blob string) bool { + ctx := context.TODO() + // see if we are in the set + exists, err := d.db.SIsMember(ctx, join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()), blob).Result() + if err != nil { + d.log.Error().Err(err).Str("sismember", join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())).Str("digest", digest.String()).Msg("unable to get record") + return false + } + if !exists { + return false + } + // see if the path entry exists. is this actually needed? i guess it doesn't really hurt (it is fast) + exists, err = d.db.HExists(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + d.log.Error().Err(err).Str("hexists", join(constants.BlobsCache, constants.OriginalBucket)).Str("digest", digest.String()).Msg("unable to get record") + if err != nil { + return false + } + if !exists { + return false + } + return true +} + +func (d *RedisDriver) DeleteBlob(digest godigest.Digest, path string) error { + ctx := context.TODO() + + // use only relative (to rootDir) paths on blobs + var err error + if d.useRelPaths { + path, err = filepath.Rel(d.rootDir, path) + if err != nil { + d.log.Error().Err(err).Str("path", path).Msg("unable to get relative path") + } + } + + pathSet := join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()) + + // delete path from the set of paths which the digest represents + _, err = d.db.SRem(ctx, pathSet, path).Result() + if err != nil { + d.log.Error().Err(err).Str("srem", pathSet).Str("value", path).Msg("unable to delete record") + return err + } + currentPath, err := d.GetBlob(digest) + if err != nil { + return err + } + if currentPath != path { + // nothing we need to do, return nil yay + return nil + } + // we need to set a new path + newPath, err := d.db.SRandMember(ctx, pathSet).Result() + if err != nil { + if goerrors.Is(err, redis.Nil) { + _, err := d.db.HDel(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result() + if err != nil { + return err + } + return nil + } + d.log.Error().Err(err).Str("srandmember", pathSet).Msg("unable to get new path") + return err + } + if _, err := d.db.HSet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String(), newPath).Result(); err != nil { + d.log.Error().Err(err).Str("hset", join(constants.BlobsCache, constants.OriginalBucket)).Str("value", newPath).Msg("unable to put record") + return err + } + + return nil +} diff --git a/pkg/storage/cache/redis_test.go b/pkg/storage/cache/redis_test.go new file mode 100644 index 000000000..261eb996d --- /dev/null +++ b/pkg/storage/cache/redis_test.go @@ -0,0 +1,124 @@ +package cache_test + +import ( + "path" + "testing" + + "github.com/alicebob/miniredis/v2" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.dev/zot/errors" + "zotregistry.dev/zot/pkg/log" + "zotregistry.dev/zot/pkg/storage" + "zotregistry.dev/zot/pkg/storage/cache" +) + +func TestRedisCache(t *testing.T) { + mr := miniredis.RunT(t) + Convey("Make a new cache", t, func() { + + dir := t.TempDir() + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + So(func() { _, _ = storage.Create("redis", "failTypeAssertion", log) }, ShouldPanic) + + cacheDriver, _ := storage.Create("redis", cache.RedisDriverParameters{dir, "redis://" + mr.Addr(), true}, log) + So(cacheDriver, ShouldNotBeNil) + + name := cacheDriver.Name() + So(name, ShouldEqual, "redis") + + val, err := cacheDriver.GetBlob("key") + So(err, ShouldEqual, errors.ErrCacheMiss) + So(val, ShouldBeEmpty) + + exists := cacheDriver.HasBlob("key", "value") + So(exists, ShouldBeFalse) + + err = cacheDriver.PutBlob("key", path.Join(dir, "value")) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key", "value") + So(err, ShouldNotBeNil) + + exists = cacheDriver.HasBlob("key", "value") + So(exists, ShouldBeTrue) + + val, err = cacheDriver.GetBlob("key") + So(err, ShouldBeNil) + So(val, ShouldNotBeEmpty) + + err = cacheDriver.DeleteBlob("bogusKey", "bogusValue") + So(err, ShouldEqual, errors.ErrCacheMiss) + + err = cacheDriver.DeleteBlob("key", "bogusValue") + So(err, ShouldBeNil) + + // try to insert empty path + err = cacheDriver.PutBlob("key", "") + So(err, ShouldNotBeNil) + So(err, ShouldEqual, errors.ErrEmptyValue) + + cacheDriver, _ = storage.Create("redis", cache.RedisDriverParameters{t.TempDir(), "redis://" + mr.Addr() + "/5", false}, log) + So(cacheDriver, ShouldNotBeNil) + + err = cacheDriver.PutBlob("key1", "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "originalBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "originalBlobPath") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key1") + So(val, ShouldEqual, "duplicateBlobPath") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key1", "duplicateBlobPath") + So(err, ShouldBeNil) + + // should be empty + val, err = cacheDriver.GetBlob("key1") + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + + // try to add three same values + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob("key2", "duplicate") + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob("key2") + So(val, ShouldEqual, "duplicate") + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob("key2", "duplicate") + So(err, ShouldBeNil) + + // should be empty + val, err = cacheDriver.GetBlob("key2") + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + }) +}