feat(redis): update .github/workflows/cluster.yaml to use redis

Note .github/workflows/cluster.yaml has been broken for a while and silently failing. The reason it has been broken is the 3 zot instances were trying to open the same boltdb file in order to write. This became apparent in: 8553712#diff-afd8186ad1553f2a9918413c02ec9e361ae0a11aa973af7fd447115f2ae4202e Signed-off-by: Andrei Aaron <[email protected]>
project-zot · Jan 16, 2025 · 78a3289 · 78a3289
1 parent 1099917
commit 78a3289
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 4 deletions.
diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml
@@ -72,6 +72,15 @@ jobs:
             --retry-max-time 120 \
             'http://localhost:9000/minio/health/live'
 
+      - name: Setup redis service
+        run: |
+            docker run -d -p 6379:6379 --name redis \
+                --health-cmd "redis-cli ping" \
+                --health-interval 10s \
+                --health-timeout 5s \
+                --health-retries 5 \
+                redis:7.4.2
+
       - name: Create minio bucket
         run: |
             python3 - <<'EOF'
@@ -101,12 +110,15 @@ jobs:
             cp test/cluster/config-minio.json test/cluster/config-minio1.json
             sed -i 's/8081/8081/g' test/cluster/config-minio1.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot1/g' test/cluster/config-minio1.json
+            sed -i 's/\/dev\/null/\/tmp\/zot1.log/g' test/cluster/config-minio1.json
             cp test/cluster/config-minio.json test/cluster/config-minio2.json
             sed -i 's/8081/8082/g' test/cluster/config-minio2.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot2/g' test/cluster/config-minio2.json
+            sed -i 's/\/dev\/null/\/tmp\/zot2.log/g' test/cluster/config-minio2.json
             cp test/cluster/config-minio.json test/cluster/config-minio3.json
             sed -i 's/8081/8083/g' test/cluster/config-minio3.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot3/g' test/cluster/config-minio3.json
+            sed -i 's/\/dev\/null/\/tmp\/zot3.log/g' test/cluster/config-minio3.json
 
       - name: Free up disk space
         uses: jlumbroso/free-disk-space@main
@@ -126,6 +138,12 @@ jobs:
             ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json &
             sleep 20
+
+            # ensure the instances are online
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+
             # run tests
             skopeo --debug copy --format=oci --dest-tls-verify=false docker://ghcr.io/project-zot/golang:1.20 docker://localhost:8080/golang:1.20
             skopeo --debug copy --src-tls-verify=false docker://localhost:8080/golang:1.20 oci:golang:1.20
@@ -139,46 +157,102 @@ jobs:
             oras pull --plain-http localhost:8080/hello-artifact:v2 -d -v
             grep -q "hello world" artifact.txt  # should print "hello world"
             if [ $? -ne 0 ]; then \
-              killall -r zot-*; \
+              killall --wait -r zot-*; \
               exit 1; \
             fi
 
-            killall -r zot-*
+            killall --wait -r zot-*
+
+            # archive logs
+            zip logs-push-pull.zip /tmp/*.log -r
+
+            # clean zot storage
+            sudo rm -rf /tmp/data/zot-storage/zot
+            # clean zot cache and metadb
+            docker exec redis redis-cli FLUSHDB
+            # clean zot logs
+            rm /tmp/*.log
         env:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - name: Upload zot logs for push-pull tests
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs-push-pull.zip
+          path: logs-push-pull.zip
+          if-no-files-found: error
+
       - name: Run benchmark with --src-cidr arg
         run: |
             ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json &
             sleep 20
+
+            # ensure the instances are online
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+
             # run zb with --src-cidr
             bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-cidr 127.0.0.0/8 http://localhost:8080
 
-            killall -r zot-*
+            killall --wait -r zot-*
+
+            # archive logs
+            zip logs-src-cidr.zip /tmp/*.log -r
 
             # clean zot storage
             sudo rm -rf /tmp/data/zot-storage/zot
+            # clean zot cache and metadb
+            docker exec redis redis-cli FLUSHDB
+            # clean zot logs
+            rm /tmp/*.log
         env:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - name: Upload zot logs for cidr tests
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs-src-cidr.zip
+          path: logs-src-cidr.zip
+          if-no-files-found: error
+
       - name: Run benchmark with --src-ips arg
         run: |
             ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json &
             sleep 20
+
+            # ensure the instances are online
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8081/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+            curl --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused http://localhost:8082/v2/
+
             # run zb with --src-ips
             bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-ips 127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6,127.0.12.5,127.0.12.6 http://localhost:8080
 
-            killall -r zot-*
+            killall --wait -r zot-*
+
+            # archive logs
+            zip logs-src-ips.zip /tmp/*.log -r
         env:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - name: Upload zot logs for src-ips tests
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs-src-ips.zip
+          path: logs-src-ips.zip
+          if-no-files-found: error
+
       # Download previous benchmark result from cache (if exists)
       - name: Download previous benchmark data
         uses: actions/cache@v4

diff --git a/test/cluster/config-minio.json b/test/cluster/config-minio.json
@@ -4,6 +4,10 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "dedupe": false,
+        "cacheDriver": {
+            "name": "redis",
+            "url": "redis://localhost:6379"
+        },
         "storageDriver": {
             "name": "s3",
             "rootdirectory": "/zot",