fix: TestPopulateStorageMetrics fails occasionally in CI (#2042)

Signed-off-by: Alexei Dodon <[email protected]>
project-zot · Nov 15, 2023 · dd079bf · dd079bf
1 parent 272eb7c
commit dd079bf
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 4 deletions.
diff --git a/pkg/extensions/monitoring/monitoring_test.go b/pkg/extensions/monitoring/monitoring_test.go
@@ -6,6 +6,7 @@ package monitoring_test
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"net/http"
 	"os"
@@ -435,16 +436,27 @@ func TestPopulateStorageMetrics(t *testing.T) {
 			Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
 		}
 
+		logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt")
+		if err != nil {
+			panic(err)
+		}
+
+		logPath := logFile.Name()
+		defer os.Remove(logPath)
+
+		writers := io.MultiWriter(os.Stdout, logFile)
+
 		ctlr := api.NewController(conf)
 		So(ctlr, ShouldNotBeNil)
+		ctlr.Log.Logger = ctlr.Log.Output(writers)
 
 		cm := test.NewControllerManager(ctlr)
 		cm.StartAndWait(port)
 		defer cm.StopServer()
 
 		// write a couple of images
 		srcStorageCtlr := ociutils.GetDefaultStoreController(rootDir, ctlr.Log)
-		err := WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr)
+		err = WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr)
 		So(err, ShouldBeNil)
 		err = WriteImageToFileSystem(CreateDefaultImage(), "busybox", "0.0.1", srcStorageCtlr)
 		So(err, ShouldBeNil)
@@ -462,7 +474,16 @@ func TestPopulateStorageMetrics(t *testing.T) {
 
 		sch.SubmitGenerator(generator, time.Duration(0), scheduler.LowPriority)
 
-		time.Sleep(5 * time.Second)
+		// Wait for storage metrics to update
+		found, err := test.ReadLogFileAndSearchString(logPath,
+			"monitoring: computed storage usage for repo alpine", time.Minute)
+		So(err, ShouldBeNil)
+		So(found, ShouldBeTrue)
+		found, err = test.ReadLogFileAndSearchString(logPath,
+			"monitoring: computed storage usage for repo busybox", time.Minute)
+		So(err, ShouldBeNil)
+		So(found, ShouldBeTrue)
+
 		cancel()
 		alpineSize, err := monitoring.GetDirSize(path.Join(rootDir, "alpine"))
 		So(err, ShouldBeNil)

diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go
@@ -1095,7 +1095,7 @@ func (gen *StorageMetricsInitGenerator) Next() (scheduler.Task, error) {
 	}
 	gen.lastRepo = repo
 
-	return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo), nil
+	return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo, gen.Log), nil
 }
 
 func (gen *StorageMetricsInitGenerator) IsDone() bool {
@@ -1116,16 +1116,19 @@ type smTask struct {
 	imgStore storageTypes.ImageStore
 	metrics  monitoring.MetricServer
 	repo     string
+	log      zlog.Logger
 }
 
 func NewStorageMetricsTask(imgStore storageTypes.ImageStore, metrics monitoring.MetricServer, repo string,
+	log zlog.Logger,
 ) *smTask {
-	return &smTask{imgStore, metrics, repo}
+	return &smTask{imgStore, metrics, repo, log}
 }
 
 func (smt *smTask) DoWork(ctx context.Context) error {
 	// run task
 	monitoring.SetStorageUsage(smt.metrics, smt.imgStore.RootDir(), smt.repo)
+	smt.log.Debug().Msg("monitoring: computed storage usage for repo " + smt.repo)
 
 	return nil
 }