feat(redis): update .github/workflows/cluster.yaml to use redis

Note .github/workflows/cluster.yaml has been broken for a while and silently failing. The reason it has been broken is the 3 zot instances were trying to open the same boltdb file in order to write. Signed-off-by: Andrei Aaron <[email protected]>
project-zot · Jan 16, 2025 · cdccab6 · cdccab6
1 parent 1099917
commit cdccab6
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 0 deletions.
diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml
@@ -72,6 +72,15 @@ jobs:
             --retry-max-time 120 \
             'http://localhost:9000/minio/health/live'
 
+      - name: Setup redis service
+        run: |
+            docker run -d -p 6379:6379 --name redis
+                --health-cmd "redis-cli ping" \
+                --health-interval 10s \
+                --health-timeout 5s \
+                --health-retries 5 \
+                redis:7.4.2
+
       - name: Create minio bucket
         run: |
             python3 - <<'EOF'
@@ -101,12 +110,15 @@ jobs:
             cp test/cluster/config-minio.json test/cluster/config-minio1.json
             sed -i 's/8081/8081/g' test/cluster/config-minio1.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot1/g' test/cluster/config-minio1.json
+            sed -i 's/\/dev\/null/\/tmp\/zot1.log/g' test/cluster/config-minio1.json
             cp test/cluster/config-minio.json test/cluster/config-minio2.json
             sed -i 's/8081/8082/g' test/cluster/config-minio2.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot2/g' test/cluster/config-minio2.json
+            sed -i 's/\/dev\/null/\/tmp\/zot2.log/g' test/cluster/config-minio2.json
             cp test/cluster/config-minio.json test/cluster/config-minio3.json
             sed -i 's/8081/8083/g' test/cluster/config-minio3.json
             sed -i 's/\/tmp\/zot/\/tmp\/zot3/g' test/cluster/config-minio3.json
+            sed -i 's/\/dev\/null/\/tmp\/zot3.log/g' test/cluster/config-minio3.json
 
       - name: Free up disk space
         uses: jlumbroso/free-disk-space@main
@@ -154,31 +166,63 @@ jobs:
             ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json &
             sleep 20
+
+            # ensure the instances are online
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8081/v2/
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8082/v2/
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8082/v2/
+
             # run zb with --src-cidr
             bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-cidr 127.0.0.0/8 http://localhost:8080
 
             killall -r zot-*
 
+            # archive logs
+            zip logs-src-cidr.zip /tmp/*.log -r
+
             # clean zot storage
             sudo rm -rf /tmp/data/zot-storage/zot
+            # clean zot cache and metadb
+            docker exec redis "redis-cli FLUSHDB"
+            # clean zot logs
+            rm /tmp/*.log
         env:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - uses: actions/upload-artifact@v4
+        with:
+          path: logs-src-cidr.zip
+          if-no-files-found: error
+
       - name: Run benchmark with --src-ips arg
         run: |
             ./bin/zot-linux-amd64 serve test/cluster/config-minio1.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio2.json &
             ./bin/zot-linux-amd64 serve test/cluster/config-minio3.json &
             sleep 20
+
+            # ensure the instances are online
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8081/v2/
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8082/v2/
+            curl -k --connect-timeout 3 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 --retry-connrefused https://localhost:8082/v2/
+
             # run zb with --src-ips
             bin/zb-linux-amd64 -c 10 -n 50 -o ci-cd --src-ips 127.0.0.2,127.0.0.3,127.0.0.4,127.0.0.5,127.0.0.6,127.0.12.5,127.0.12.6 http://localhost:8080
 
             killall -r zot-*
+
+            # archive logs
+            zip logs-src-ips.zip /tmp/*.log -r
         env:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - uses: actions/upload-artifact@v4
+        with:
+          path: logs-src-ips.zip
+          if-no-files-found: error
+
       # Download previous benchmark result from cache (if exists)
       - name: Download previous benchmark data
         uses: actions/cache@v4

diff --git a/test/cluster/config-minio.json b/test/cluster/config-minio.json
@@ -4,6 +4,10 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "dedupe": false,
+        "cacheDriver": {
+            "name": "redis",
+            "url": "redis://localhost:6379"
+        },
         "storageDriver": {
             "name": "s3",
             "rootdirectory": "/zot",