From 448f94dceea201e189743e798d249c2e64c457ec Mon Sep 17 00:00:00 2001 From: hasan7n Date: Mon, 1 Apr 2024 17:57:49 +0200 Subject: [PATCH 1/4] use postgresql for testing --- .github/workflows/auth-ci.yml | 4 ++++ .github/workflows/docker-ci.yml | 7 ++++++- .github/workflows/local-ci.yml | 4 ++++ .github/workflows/unittests.yml | 3 +++ server/.env.local.local-auth | 2 +- server/.env.local.local-auth.sqlite | 23 +++++++++++++++++++++++ server/.env.local.online-auth | 2 +- server/run_dev_postgresql.sh | 19 +++++++++++++++++++ 8 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 server/.env.local.local-auth.sqlite create mode 100644 server/run_dev_postgresql.sh diff --git a/.github/workflows/auth-ci.yml b/.github/workflows/auth-ci.yml index b41b695bc..25a76621a 100644 --- a/.github/workflows/auth-ci.yml +++ b/.github/workflows/auth-ci.yml @@ -46,6 +46,10 @@ jobs: working-directory: ./server run: cp .env.local.online-auth .env + - name: Run postgresql server in background + working-directory: ./server + run: sh run_dev_postgresql.sh && sleep 6 + - name: Run django server in background with generated certs working-directory: ./server run: sh setup-dev-server.sh & sleep 6 diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index b7d778782..c1b840fc4 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -57,9 +57,14 @@ jobs: cp ../mock_tokens/tokens.json tokens.json docker build -t ${{ env.IMAGE_NAME }} -f Dockerfile.gha . + - name: Run postgresql server in background + working-directory: ./server + run: sh run_dev_postgresql.sh -n postgreserver && sleep 6 + # NOTE: Actually there is no need to publish the database container port + - name: Run server in background working-directory: ./server - run: PORT=8080 && docker run --name medperf_api -d -p 8000:${PORT} -e PORT=${PORT} -e SSL_FLAGS="--certfile=cert.crt --keyfile=cert.key" ${{ env.IMAGE_NAME }} + run: PORT=8080 && docker run --name medperf_api --network container:postgreserver -d -p 127.0.0.1:8000:${PORT} -e PORT=${PORT} -e SSL_FLAGS="--certfile=cert.crt --keyfile=cert.key" ${{ env.IMAGE_NAME }} - name: Run server integration tests working-directory: ./server diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index 10fcb2550..9f04a4d8d 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -47,6 +47,10 @@ jobs: working-directory: ./server run: cp .env.local.local-auth .env + - name: Run postgresql server in background + working-directory: ./server + run: sh run_dev_postgresql.sh && sleep 6 + - name: Run django server in background with generated certs working-directory: ./server run: sh setup-dev-server.sh & sleep 6 diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 11c570b73..a74fbf8ef 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -42,6 +42,9 @@ jobs: - name: Set server environment vars working-directory: ./server run: cp .env.local.local-auth .env + - name: Run postgresql server in background + working-directory: ./server + run: sh run_dev_postgresql.sh && sleep 6 - name: Run migrations working-directory: ./server run: python manage.py migrate diff --git a/server/.env.local.local-auth b/server/.env.local.local-auth index fa5ea9c33..d582619b0 100644 --- a/server/.env.local.local-auth +++ b/server/.env.local.local-auth @@ -4,7 +4,7 @@ DEBUG=True SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME -DATABASE_URL=sqlite:///db.sqlite3 +DATABASE_URL=postgres://devuser:devpassword@127.0.0.1/devdb SUPERUSER_USERNAME=admin SUPERUSER_PASSWORD=admin ALLOWED_HOSTS=* diff --git a/server/.env.local.local-auth.sqlite b/server/.env.local.local-auth.sqlite new file mode 100644 index 000000000..fa5ea9c33 --- /dev/null +++ b/server/.env.local.local-auth.sqlite @@ -0,0 +1,23 @@ +############################################################## +############## ALERT: DO NOT USE FOR PRODUCTION ############## +############################################################## + +DEBUG=True +SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME +DATABASE_URL=sqlite:///db.sqlite3 +SUPERUSER_USERNAME=admin +SUPERUSER_PASSWORD=admin +ALLOWED_HOSTS=* + +#Valid deployment environments are local, gcp-ci, gcp-prod(case-sensitive) +DEPLOY_ENV=local + +#Production settings when deployed in GCP +CORS_ALLOWED_ORIGINS= +GS_BUCKET_NAME= + +#Auth configuration +AUTH_AUDIENCE=https://localhost-localdev/ +AUTH_ISSUER=https://localhost:8000/ +AUTH_JWK_URL= +AUTH_VERIFYING_KEY="-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtKO1SzU6N/sZTJmYNk0C\n/5XbK8eWfcKX2HxFl7fr0V++wrXXGsMs9A8hQEbVWtgYbWaOSkXN0ojmcUt1NFcb\nSPYLmOK/oUXVASEbuZAdIi+ByQ1EnIIAmYSKjRBDUQM8wc73Z9AvrjnhrvEHyrIN\nKyXeLnaCKj/r0s5sQA85SngnCWQbZsRQyHysfsQLwguG0SKFF9EfdNJiaoD8lLBo\nqvUQIYi8MXuVAB7O5EomJoZJe7KEeemsLhCnjTlKHcumjnAiRy5Y0rL6aFXgQkg0\nY4NWxMbsIWAplzh2qCs2jEd88mAUJnHkMzeOKhb1Q+tcmg6ZG6GmwT9fujsOjYrn\na/RTx83B1rRVRHHBFsEP4/ctVf2VdARz+RO+mIh5yZsPiqmRSKpHfbKgnkBpQlAj\nwVrzP9HYT11EXGFesLKRt6Oin0I5FkJ1Ji4w680XjeyZ4KInMY87OvQtltIyrZI9\nR9uY9EnpISGYch6kxbVw0GzdQdP/0mUnYlIeWwyvsXsWB/b3pZ9BiQuCMtlxoWlk\naRjWk9dWIZKFL2uhgeNeY5Wh3Qx9EFx8hnz9ohdaNBPB5BNO2qI61NedFrjYN9LF\nSfcGL7iATU1JQS4rDisnyjDikkTHL9B1u6sMrTsoaqi9Dl5b0gC8RnPVnJItasMN\n9HcW8Pfo2Ava4ler7oU47jUCAwEAAQ==\n-----END PUBLIC KEY-----" \ No newline at end of file diff --git a/server/.env.local.online-auth b/server/.env.local.online-auth index bff5de5f2..e1eee81d2 100644 --- a/server/.env.local.online-auth +++ b/server/.env.local.online-auth @@ -4,7 +4,7 @@ DEBUG=True SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME -DATABASE_URL=sqlite:///db.sqlite3 +DATABASE_URL=postgres://devuser:devpassword@127.0.0.1/devdb SUPERUSER_USERNAME=admin SUPERUSER_PASSWORD=admin ALLOWED_HOSTS=* diff --git a/server/run_dev_postgresql.sh b/server/run_dev_postgresql.sh new file mode 100644 index 000000000..cd6ef4463 --- /dev/null +++ b/server/run_dev_postgresql.sh @@ -0,0 +1,19 @@ +# we should frequently check to ensure that the postgres version +# matches the one we use in production +# NOTE: postgresql docker images show vulnerabilities, but we are using it for dev. +# Also the vulnerabilities don't affect how the container is primarily used. + +while getopts n: flag; do + case "${flag}" in + n) CONTAINER_NAME=${OPTARG} ;; + esac +done + +CONTAINER_NAME="${CONTAINER_NAME:-postgreserver}" + +docker run -d --name $CONTAINER_NAME \ + -p 127.0.0.1:5432:5432 \ + -e POSTGRES_USER=devuser \ + -e POSTGRES_PASSWORD=devpassword \ + -e POSTGRES_DB=devdb \ + postgres:14.10-alpine3.17 From 689e53947f6500047cdfe6b646a452ea32379b40 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Fri, 29 Nov 2024 18:18:01 +0100 Subject: [PATCH 2/4] use network in build --- .github/workflows/docker-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index c1b840fc4..5e41b15ce 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -51,17 +51,17 @@ jobs: working-directory: ./server run: sh setup-dev-server.sh -c cert.crt -k cert.key -d 0 - - name: Build container image - working-directory: ./server - run: | - cp ../mock_tokens/tokens.json tokens.json - docker build -t ${{ env.IMAGE_NAME }} -f Dockerfile.gha . - - name: Run postgresql server in background working-directory: ./server run: sh run_dev_postgresql.sh -n postgreserver && sleep 6 # NOTE: Actually there is no need to publish the database container port + - name: Build container image + working-directory: ./server + run: | + cp ../mock_tokens/tokens.json tokens.json + docker build --network container:postgreserver -t ${{ env.IMAGE_NAME }} -f Dockerfile.gha . + - name: Run server in background working-directory: ./server run: PORT=8080 && docker run --name medperf_api --network container:postgreserver -d -p 127.0.0.1:8000:${PORT} -e PORT=${PORT} -e SSL_FLAGS="--certfile=cert.crt --keyfile=cert.key" ${{ env.IMAGE_NAME }} From 59c71990cf39ac3760af8a7fb24dc91b36fbe498 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Sat, 30 Nov 2024 19:31:48 +0100 Subject: [PATCH 3/4] use sqlite3 for docker-ci instead of postgres this is because we actually only need at least one test workflow to use postgres, and setting up docker-ci with postgres is a bit tricky --- .github/workflows/docker-ci.yml | 11 +++-------- server/reset_db_postgresql.sh | 12 ++++++++++++ server/setup-dev-server.sh | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 server/reset_db_postgresql.sh diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index 5e41b15ce..da9b83aff 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -45,26 +45,21 @@ jobs: - name: Set server environment vars working-directory: ./server - run: cp .env.local.local-auth .env + run: cp .env.local.local-auth.sqlite .env - name: Generate SSL certificate working-directory: ./server run: sh setup-dev-server.sh -c cert.crt -k cert.key -d 0 - - name: Run postgresql server in background - working-directory: ./server - run: sh run_dev_postgresql.sh -n postgreserver && sleep 6 - # NOTE: Actually there is no need to publish the database container port - - name: Build container image working-directory: ./server run: | cp ../mock_tokens/tokens.json tokens.json - docker build --network container:postgreserver -t ${{ env.IMAGE_NAME }} -f Dockerfile.gha . + docker build -t ${{ env.IMAGE_NAME }} -f Dockerfile.gha . - name: Run server in background working-directory: ./server - run: PORT=8080 && docker run --name medperf_api --network container:postgreserver -d -p 127.0.0.1:8000:${PORT} -e PORT=${PORT} -e SSL_FLAGS="--certfile=cert.crt --keyfile=cert.key" ${{ env.IMAGE_NAME }} + run: PORT=8080 && docker run --name medperf_api -d -p 8000:${PORT} -e PORT=${PORT} -e SSL_FLAGS="--certfile=cert.crt --keyfile=cert.key" ${{ env.IMAGE_NAME }} - name: Run server integration tests working-directory: ./server diff --git a/server/reset_db_postgresql.sh b/server/reset_db_postgresql.sh new file mode 100644 index 000000000..5f0dbe650 --- /dev/null +++ b/server/reset_db_postgresql.sh @@ -0,0 +1,12 @@ +while getopts n: flag; do + case "${flag}" in + n) CONTAINER_NAME=${OPTARG} ;; + esac +done + +CONTAINER_NAME="${CONTAINER_NAME:-postgreserver}" + +docker container stop $CONTAINER_NAME +sh run_dev_postgresql.sh -n $CONTAINER_NAME +sleep 6 +python manage.py migrate diff --git a/server/setup-dev-server.sh b/server/setup-dev-server.sh index 4d0599f6d..df4ebc7a7 100644 --- a/server/setup-dev-server.sh +++ b/server/setup-dev-server.sh @@ -47,7 +47,7 @@ if [ "$RESET_DB" -eq 1 ] then # Clean DB for a fresh start echo "Cleaning DB as RESET_DB flag is enabled" - rm db.sqlite3 + sh reset_db.sh fi if [ "$DEPLOY" -eq 0 ] From ac09a9be275b8f3bbfc1f52f16cceaee35ec8742 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Sat, 30 Nov 2024 23:48:06 +0100 Subject: [PATCH 4/4] use apt install in auth ci setup --- .github/workflows/auth-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auth-ci.yml b/.github/workflows/auth-ci.yml index 12a3465ee..3070d73ac 100644 --- a/.github/workflows/auth-ci.yml +++ b/.github/workflows/auth-ci.yml @@ -30,7 +30,7 @@ jobs: run: | sudo apt-get install -y wget wget -O chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_126.0.6478.126-1_amd64.deb - sudo dpkg -i chrome.deb + sudo apt update && sudo apt install ./chrome.deb -y rm chrome.deb - name: Install dependencies