From dd079bf9a36bf3f0aa2716cd2a84c70b9b1a67c0 Mon Sep 17 00:00:00 2001 From: Alexei Dodon Date: Wed, 15 Nov 2023 02:22:24 +0200 Subject: [PATCH] fix: TestPopulateStorageMetrics fails occasionally in CI (#2042) Signed-off-by: Alexei Dodon --- pkg/extensions/monitoring/monitoring_test.go | 25 ++++++++++++++++++-- pkg/storage/common/common.go | 7 ++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pkg/extensions/monitoring/monitoring_test.go b/pkg/extensions/monitoring/monitoring_test.go index 9511518a8..948fe5531 100644 --- a/pkg/extensions/monitoring/monitoring_test.go +++ b/pkg/extensions/monitoring/monitoring_test.go @@ -6,6 +6,7 @@ package monitoring_test import ( "context" "fmt" + "io" "math/rand" "net/http" "os" @@ -435,8 +436,19 @@ func TestPopulateStorageMetrics(t *testing.T) { Prometheus: &extconf.PrometheusConfig{Path: "/metrics"}, } + logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt") + if err != nil { + panic(err) + } + + logPath := logFile.Name() + defer os.Remove(logPath) + + writers := io.MultiWriter(os.Stdout, logFile) + ctlr := api.NewController(conf) So(ctlr, ShouldNotBeNil) + ctlr.Log.Logger = ctlr.Log.Output(writers) cm := test.NewControllerManager(ctlr) cm.StartAndWait(port) @@ -444,7 +456,7 @@ func TestPopulateStorageMetrics(t *testing.T) { // write a couple of images srcStorageCtlr := ociutils.GetDefaultStoreController(rootDir, ctlr.Log) - err := WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr) + err = WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr) So(err, ShouldBeNil) err = WriteImageToFileSystem(CreateDefaultImage(), "busybox", "0.0.1", srcStorageCtlr) So(err, ShouldBeNil) @@ -462,7 +474,16 @@ func TestPopulateStorageMetrics(t *testing.T) { sch.SubmitGenerator(generator, time.Duration(0), scheduler.LowPriority) - time.Sleep(5 * time.Second) + // Wait for storage metrics to update + found, err := test.ReadLogFileAndSearchString(logPath, + "monitoring: computed storage usage for repo alpine", time.Minute) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + found, err = test.ReadLogFileAndSearchString(logPath, + "monitoring: computed storage usage for repo busybox", time.Minute) + So(err, ShouldBeNil) + So(found, ShouldBeTrue) + cancel() alpineSize, err := monitoring.GetDirSize(path.Join(rootDir, "alpine")) So(err, ShouldBeNil) diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index ab6105886..b058e81f3 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -1095,7 +1095,7 @@ func (gen *StorageMetricsInitGenerator) Next() (scheduler.Task, error) { } gen.lastRepo = repo - return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo), nil + return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo, gen.Log), nil } func (gen *StorageMetricsInitGenerator) IsDone() bool { @@ -1116,16 +1116,19 @@ type smTask struct { imgStore storageTypes.ImageStore metrics monitoring.MetricServer repo string + log zlog.Logger } func NewStorageMetricsTask(imgStore storageTypes.ImageStore, metrics monitoring.MetricServer, repo string, + log zlog.Logger, ) *smTask { - return &smTask{imgStore, metrics, repo} + return &smTask{imgStore, metrics, repo, log} } func (smt *smTask) DoWork(ctx context.Context) error { // run task monitoring.SetStorageUsage(smt.metrics, smt.imgStore.RootDir(), smt.repo) + smt.log.Debug().Msg("monitoring: computed storage usage for repo " + smt.repo) return nil }