Fix docker compose config and script

devops/run-job.sh

@@ -164,14 +164,37 @@ echo "Cloning green-directory"
 $SSHCMD git clone --progress --depth 1 https://$GIT_TOKEN@git.verdigado.com/NB-Public/green-directory.git /root/cache/green-directory
 
 echo ""
-echo "Pulling container images"
-$SSHCMD docker compose pull --quiet redis manager
+echo "Pulling redis container image"
+$SSHCMD docker compose pull redis
+
+echo ""
+echo "Set 'sysctl vm.overcommit_memory=1' to avoid OOM errors and allow background saving in redis"
+$SSHCMD sysctl "vm.overcommit_memory=1"
+
+echo ""
+echo "Creating volume directories for redis"
+$SSHCMD mkdir -p ./volumes/redis-data && chmod 777 ./volumes/redis-data
 
 echo ""
 echo "Starting redis in background"
 $SSHCMD docker compose up --detach redis
 sleep 5
 
+echo ""
+echo "Allow writing to various paths in the redis container"
+$SSHCMD docker compose exec redis chown -R redis:redis /var/spool/cron
+$SSHCMD docker compose exec redis chown -R redis:redis /etc
+
+echo ""
+echo "Debugging redis: dir"
+$SSHCMD docker compose exec redis redis-cli config get dir
+echo "Debugging redis: dbfilename"
+$SSHCMD docker compose exec redis redis-cli config get dbfilename
+
+echo ""
+echo "Pulling green-spider container image"
+$SSHCMD docker compose pull manager
+
 echo ""
 echo "Creating jobs"
 $SSHCMD docker compose up manager

docker-compose.yaml

@@ -1,17 +1,13 @@
-version: "2"
 services:
 
   redis:
-    image: redis:5-alpine
-    command: redis-server --save "" --appendonly no
+    image: redis:6-alpine
+    command: redis-server
     volumes:
-      - ${PWD}/volumes/redis-data:/data
+      - ${PWD}/volumes/redis-data:/data:rw
     restart: unless-stopped
-    networks:
-      - internal_network
-      - external_network
     ports:
-      - "6379:6379"
+      - "127.0.0.1:6379:6379"
 
   # manager fills the job queue with spider jobs.
   manager:
@@ -26,9 +22,6 @@ services:
     volumes:
       - ${PWD}/secrets:/secrets
       - ${PWD}/cache/green-directory:/workdir/cache/green-directory
-    networks:
-      - internal_network
-      - external_network
     depends_on:
       - redis
 
@@ -37,16 +30,7 @@ services:
   #   image: eoranged/rq-dashboard:v0.6.1
   #   environment:
   #     RQ_DASHBOARD_REDIS_URL: redis://redis:6379/0
-  #   networks:
-  #     - internal_network
-  #     - external_network
   #   ports:
   #     - "9181:9181"
   #   depends_on:
   #     - redis
-
-networks:
-  internal_network:
-    internal: true
-  external_network:
-    internal: false