From 7dba77f00e3f2baeb31491fdbe7dcecd1bb77cd8 Mon Sep 17 00:00:00 2001 From: Gaius Date: Mon, 30 Dec 2024 16:22:38 +0800 Subject: [PATCH] feat: add cacher for gorm to reduce the load of db Signed-off-by: Gaius --- go.mod | 4 +- go.sum | 8 ++- manager/config/constants.go | 6 +-- manager/database/cacher.go | 96 ++++++++++++++++++++++++++++++++++++ manager/database/database.go | 16 ++++++ 5 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 manager/database/cacher.go diff --git a/go.mod b/go.mod index f242b0b641d..8d1b7a4f592 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/docker/docker v27.3.1+incompatible github.com/docker/go-connections v0.5.0 github.com/docker/go-units v0.4.0 + github.com/elastic/go-freelru v0.16.0 github.com/gaius-qi/ping v1.0.0 github.com/gammazero/deque v1.0.0 github.com/gin-contrib/gzip v1.0.1 @@ -29,6 +30,7 @@ require ( github.com/gin-contrib/zap v1.1.3 github.com/gin-gonic/gin v1.10.0 github.com/go-echarts/statsview v0.3.4 + github.com/go-gorm/caches/v4 v4.0.5 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a github.com/go-playground/validator/v10 v10.23.0 github.com/go-redis/cache/v9 v9.0.0 @@ -94,7 +96,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 - gorm.io/gorm v1.24.6 + gorm.io/gorm v1.25.0 gorm.io/plugin/soft_delete v1.2.1 k8s.io/component-base v0.31.2 moul.io/zapgorm2 v1.3.0 diff --git a/go.sum b/go.sum index b92e1d750a6..429d50d51bb 100644 --- a/go.sum +++ b/go.sum @@ -468,6 +468,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-freelru v0.16.0 h1:gG2HJ1WXN2tNl5/p40JS/l59HjvjRhjyAa+oFTRArYs= +github.com/elastic/go-freelru v0.16.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -531,6 +533,8 @@ github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gorm/caches/v4 v4.0.5 h1:Sdj9vxbEM0sCmv5+s5o6GzoVMuraWF0bjJJvUU+7c1U= +github.com/go-gorm/caches/v4 v4.0.5/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -2272,8 +2276,8 @@ gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.6 h1:wy98aq9oFEetsc4CAbKD2SoBCdMzsbSIvSUUFJuHi5s= -gorm.io/gorm v1.24.6/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= +gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/plugin/dbresolver v1.1.0/go.mod h1:tpImigFAEejCALOttyhWqsy4vfa2Uh/vAUVnL5IRF7Y= gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M= gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk= diff --git a/manager/config/constants.go b/manager/config/constants.go index d1d996c1ea5..04085ff5549 100644 --- a/manager/config/constants.go +++ b/manager/config/constants.go @@ -75,7 +75,7 @@ const ( DefaultLFUCacheTTL = 3 * time.Minute // DefaultLFUCacheSize is default size for lfu cache. - DefaultLFUCacheSize = 50 * 1000 + DefaultLFUCacheSize = 30 * 1000 ) const ( @@ -88,10 +88,10 @@ const ( const ( // DefaultJobGCInterval is the default interval for gc job. - DefaultJobGCInterval = 6 * time.Hour + DefaultJobGCInterval = 3 * time.Hour // DefaultJobGCTTL is the default ttl for job. - DefaultJobGCTTL = 12 * time.Hour + DefaultJobGCTTL = 6 * time.Hour // DefaultJobGCBatchSize is the default batch size for operating on the database in gc job. DefaultJobGCBatchSize = 5000 diff --git a/manager/database/cacher.go b/manager/database/cacher.go new file mode 100644 index 00000000000..f8a64f47873 --- /dev/null +++ b/manager/database/cacher.go @@ -0,0 +1,96 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package database + +import ( + "context" + "time" + + xxhash "github.com/cespare/xxhash/v2" + lru "github.com/elastic/go-freelru" + caches "github.com/go-gorm/caches/v4" +) + +const ( + // defaultCacheSize is the default size of the cache. + defaultCacheSize = 1024 + + // defaultTTL is the default TTL of the cache. + defaultTTL = 30 * time.Second +) + +// cacher is a cache implementation using LRU for gorm. +type cacher struct { + // store is the LRU cache. + store *lru.LRU[string, any] +} + +// hashStringXXHASH returns the hash of the string s. +func hashStringXXHASH(s string) uint32 { + return uint32(xxhash.Sum64String(s)) +} + +// newCacher creates a new cacher. +func newCacher() (caches.Cacher, error) { + store, err := lru.New[string, any](defaultCacheSize, hashStringXXHASH) + if err != nil { + return nil, err + } + + store.SetLifetime(defaultTTL) + return &cacher{store: store}, nil +} + +// Get impl should check if a specific key exists in the cache and return its value +// look at Query.Marshal. +func (c *cacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) { + val, ok := c.store.Get(key) + if !ok { + return nil, nil + } + + if err := q.Unmarshal(val.([]byte)); err != nil { + return nil, err + } + + return q, nil +} + +// Store impl should store a cached representation of the val param +// look at Query.Unmarshal. +func (c *cacher) Store(ctx context.Context, key string, val *caches.Query[any]) error { + res, err := val.Marshal() + if err != nil { + return err + } + + c.store.Add(key, res) + return nil +} + +// Invalidate impl should invalidate all cached values. It will be called when +// INSERT / UPDATE / DELETE queries are sent to the DB. +func (c *cacher) Invalidate(ctx context.Context) error { + var err error + c.store, err = lru.New[string, any](defaultCacheSize, hashStringXXHASH) + if err != nil { + return err + } + + c.store.SetLifetime(defaultTTL) + return nil +} diff --git a/manager/database/database.go b/manager/database/database.go index 1f9dc7acf45..fd6ca6c7ccc 100644 --- a/manager/database/database.go +++ b/manager/database/database.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" + caches "github.com/go-gorm/caches/v4" "github.com/redis/go-redis/v9" "gorm.io/gorm" @@ -63,6 +64,21 @@ func New(cfg *config.Config) (*Database, error) { return nil, fmt.Errorf("invalid database type %s", cfg.Database.Type) } + // Add cache for reducing database load. + cacher, err := newCacher() + if err != nil { + logger.Errorf("new cacher: %s", err.Error()) + return nil, err + } + + if err := db.Use(&caches.Caches{Conf: &caches.Config{ + Easer: true, + Cacher: cacher, + }}); err != nil { + logger.Errorf("use cache: %s", err.Error()) + return nil, err + } + rdb, err := pkgredis.NewRedis(&redis.UniversalOptions{ Addrs: cfg.Database.Redis.Addrs, MasterName: cfg.Database.Redis.MasterName,