use postgresql for testing (#569)

* use postgresql for testing

* use network in build

* use sqlite3 for docker-ci instead of postgres
this is because we actually only need at least one test workflow to use postgres, and setting up docker-ci with postgres is a bit tricky

* use apt install in auth ci setup

.github/workflows/auth-ci.yml

@@ -46,6 +46,10 @@ jobs:
       working-directory: ./server
       run: cp .env.local.online-auth .env
 
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
+
     - name: Run django server in background with generated certs
       working-directory: ./server
       run: sh setup-dev-server.sh & sleep 6

.github/workflows/docker-ci.yml

@@ -45,7 +45,7 @@ jobs:
 
     - name: Set server environment vars
       working-directory: ./server
-      run: cp .env.local.local-auth .env
+      run: cp .env.local.local-auth.sqlite .env
 
     - name: Generate SSL certificate
       working-directory: ./server

.github/workflows/local-ci.yml

@@ -47,6 +47,10 @@ jobs:
       working-directory: ./server
       run: cp .env.local.local-auth .env
 
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
+
     - name: Run django server in background with generated certs
       working-directory: ./server
       run: sh setup-dev-server.sh & sleep 6

.github/workflows/unittests.yml

@@ -42,6 +42,9 @@ jobs:
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
     - name: Run migrations
       working-directory: ./server
       run: python manage.py migrate

server/.env.local.local-auth

@@ -4,7 +4,7 @@
 
 DEBUG=True
 SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
-DATABASE_URL=sqlite:///db.sqlite3
+DATABASE_URL=postgres://devuser:[email protected]/devdb
 SUPERUSER_USERNAME=admin
 SUPERUSER_PASSWORD=admin
 ALLOWED_HOSTS=*

server/.env.local.local-auth.sqlite

@@ -0,0 +1,23 @@
+##############################################################
+############## ALERT: DO NOT USE FOR PRODUCTION ##############
+##############################################################
+
+DEBUG=True
+SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
+DATABASE_URL=sqlite:///db.sqlite3
+SUPERUSER_USERNAME=admin
+SUPERUSER_PASSWORD=admin
+ALLOWED_HOSTS=*
+
+#Valid deployment environments are local, gcp-ci, gcp-prod(case-sensitive)
+DEPLOY_ENV=local
+
+#Production settings when deployed in GCP 
+CORS_ALLOWED_ORIGINS=
+GS_BUCKET_NAME=
+
+#Auth configuration
+AUTH_AUDIENCE=https://localhost-localdev/
+AUTH_ISSUER=https://localhost:8000/
+AUTH_JWK_URL=
+AUTH_VERIFYING_KEY="-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtKO1SzU6N/sZTJmYNk0C\n/5XbK8eWfcKX2HxFl7fr0V++wrXXGsMs9A8hQEbVWtgYbWaOSkXN0ojmcUt1NFcb\nSPYLmOK/oUXVASEbuZAdIi+ByQ1EnIIAmYSKjRBDUQM8wc73Z9AvrjnhrvEHyrIN\nKyXeLnaCKj/r0s5sQA85SngnCWQbZsRQyHysfsQLwguG0SKFF9EfdNJiaoD8lLBo\nqvUQIYi8MXuVAB7O5EomJoZJe7KEeemsLhCnjTlKHcumjnAiRy5Y0rL6aFXgQkg0\nY4NWxMbsIWAplzh2qCs2jEd88mAUJnHkMzeOKhb1Q+tcmg6ZG6GmwT9fujsOjYrn\na/RTx83B1rRVRHHBFsEP4/ctVf2VdARz+RO+mIh5yZsPiqmRSKpHfbKgnkBpQlAj\nwVrzP9HYT11EXGFesLKRt6Oin0I5FkJ1Ji4w680XjeyZ4KInMY87OvQtltIyrZI9\nR9uY9EnpISGYch6kxbVw0GzdQdP/0mUnYlIeWwyvsXsWB/b3pZ9BiQuCMtlxoWlk\naRjWk9dWIZKFL2uhgeNeY5Wh3Qx9EFx8hnz9ohdaNBPB5BNO2qI61NedFrjYN9LF\nSfcGL7iATU1JQS4rDisnyjDikkTHL9B1u6sMrTsoaqi9Dl5b0gC8RnPVnJItasMN\n9HcW8Pfo2Ava4ler7oU47jUCAwEAAQ==\n-----END PUBLIC KEY-----"

server/.env.local.online-auth

@@ -4,7 +4,7 @@
 
 DEBUG=True
 SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
-DATABASE_URL=sqlite:///db.sqlite3
+DATABASE_URL=postgres://devuser:[email protected]/devdb
 SUPERUSER_USERNAME=admin
 SUPERUSER_PASSWORD=admin
 ALLOWED_HOSTS=*

server/reset_db_postgresql.sh

@@ -0,0 +1,12 @@
+while getopts n: flag; do
+    case "${flag}" in
+    n) CONTAINER_NAME=${OPTARG} ;;
+    esac
+done
+
+CONTAINER_NAME="${CONTAINER_NAME:-postgreserver}"
+
+docker container stop $CONTAINER_NAME
+sh run_dev_postgresql.sh -n $CONTAINER_NAME
+sleep 6
+python manage.py migrate

server/run_dev_postgresql.sh

@@ -0,0 +1,19 @@
+# we should frequently check to ensure that the postgres version
+# matches the one we use in production
+# NOTE: postgresql docker images show vulnerabilities, but we are using it for dev.
+#       Also the vulnerabilities don't affect how the container is primarily used.
+
+while getopts n: flag; do
+    case "${flag}" in
+    n) CONTAINER_NAME=${OPTARG} ;;
+    esac
+done
+
+CONTAINER_NAME="${CONTAINER_NAME:-postgreserver}"
+
+docker run -d --name $CONTAINER_NAME \
+    -p 127.0.0.1:5432:5432 \
+    -e POSTGRES_USER=devuser \
+    -e POSTGRES_PASSWORD=devpassword \
+    -e POSTGRES_DB=devdb \
+    postgres:14.10-alpine3.17

server/setup-dev-server.sh

@@ -47,7 +47,7 @@ if [ "$RESET_DB" -eq 1 ]
 then
   # Clean DB for a fresh start
   echo "Cleaning DB as RESET_DB flag is enabled"
-  rm db.sqlite3
+  sh reset_db.sh
 fi
 
 if [ "$DEPLOY" -eq 0 ]