From 03405edf51cf4a78ab6b4e651b59f7f98f74036a Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 15 Aug 2024 14:16:37 +0000 Subject: [PATCH 1/2] Run CI tests in a container --- .github/workflows/ci.yml | 24 +++++++++++------------- code_check.py | 11 +++-------- temba/archives/tests.py | 2 +- temba/settings_ci.py | 18 ------------------ temba/settings_common.py | 13 ++++--------- 5 files changed, 19 insertions(+), 49 deletions(-) delete mode 100644 temba/settings_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d729bef13c4..03e3027affe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,11 +4,11 @@ jobs: test: name: Test runs-on: ubuntu-latest + container: + image: python:3.11-bullseye env: - python-version: '3.11.x' node-version: '20' - mailroom-version: '9.1.78' - DJANGO_SETTINGS_MODULE: temba.settings_ci + mailroom-version: '9.3.3' services: redis: @@ -40,13 +40,8 @@ jobs: - name: Install Linux packages run: | - sudo apt update - sudo apt install -y --no-install-recommends libgdal-dev ffmpeg - - - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.python-version }} + apt-get update + apt-get install -y --no-install-recommends libgdal-dev ffmpeg - name: Install Poetry uses: snok/install-poetry@v1 @@ -60,16 +55,17 @@ jobs: - name: Initialize environment run: | + npm install -g yarn less poetry install yarn install - sudo yarn global add less - ln -s ${{ github.workspace }}/temba/settings.py.dev ${{ github.workspace }}/temba/settings.py + ln -s settings.py.dev temba/settings.py poetry run python manage.py migrate # fetch, extract and start mailroom wget https://github.com/${{ github.repository_owner }}/mailroom/releases/download/v${{ env.mailroom-version }}/mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz tar -xvf mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz mailroom # start mailroom but use different redis db so that it doesn't actually handle tasks - ./mailroom -db=postgres://temba:temba@localhost:5432/temba?sslmode=disable -redis=redis://localhost:6379/15 -log-level=info > mailroom.log & + ./mailroom -db=postgres://temba:temba@postgres:5432/temba?sslmode=disable -redis=redis://redis:6379/15 -log-level=info > mailroom.log & + working-directory: ${{ github.workspace }} # https://github.com/actions/runner/issues/2058#issuecomment-2085119510 - name: Run pre-test checks run: | @@ -78,6 +74,7 @@ jobs: # check we can collect and compress all static files poetry run python manage.py collectstatic --noinput --verbosity=0 poetry run python manage.py compress --extension=".html" --settings=temba.settings_compress + working-directory: ${{ github.workspace }} - name: Run tests run: | @@ -85,6 +82,7 @@ jobs: poetry run coverage report -i poetry run coverage xml poetry run coverage html + working-directory: ${{ github.workspace }} - name: Save coverage report as artifact uses: actions/upload-artifact@v4 diff --git a/code_check.py b/code_check.py index 09f3deb2211..055a5aa6346 100755 --- a/code_check.py +++ b/code_check.py @@ -32,8 +32,8 @@ def status(line): if __name__ == "__main__": colorama.init() - status("Make any missing migrations") - cmd("python manage.py makemigrations") + status("Check for missing migrations") + cmd("python manage.py makemigrations --check") status("Running isort") cmd("isort temba") @@ -44,9 +44,4 @@ def status(line): status("Running ruff") cmd("ruff check temba") - # if any code changes were made, exit with error - if cmd("git diff temba locale"): - print("👎 " + colorama.Fore.RED + "Changes to be committed") - exit(1) - else: - print("👍 " + colorama.Fore.GREEN + "Code looks good. Make that PR!") + print("👍 " + colorama.Fore.GREEN + "Code looks good. Make that PR!") diff --git a/temba/archives/tests.py b/temba/archives/tests.py index 007baa85aca..aa3eeee02c6 100644 --- a/temba/archives/tests.py +++ b/temba/archives/tests.py @@ -238,7 +238,7 @@ def test_read(self): archive = self.create_archive(Archive.TYPE_MSG, "D", date(2020, 7, 31), [{"id": 1}, {"id": 2}]) download_url = ( - f"http://localhost:9000/test-archives/{self.org.id}/message_D20200731_{archive.hash}.jsonl.gz?response-con" + f"http://minio:9000/test-archives/{self.org.id}/message_D20200731_{archive.hash}.jsonl.gz?response-con" f"tent-disposition=attachment%3B&response-content-type=application%2Foctet&response-content-encoding=none" ) diff --git a/temba/settings_ci.py b/temba/settings_ci.py deleted file mode 100644 index fa903744abc..00000000000 --- a/temba/settings_ci.py +++ /dev/null @@ -1,18 +0,0 @@ -from .settings import * # noqa - -# instead of running tests against temba_test which the test framework creates, run against regular temba database so -# that other components (i.e. mailroom) can be run against same database -_default_database_config = { - "ENGINE": "django.contrib.gis.db.backends.postgis", - "NAME": "temba", - "USER": "temba", - "PASSWORD": "temba", - "HOST": "localhost", - "PORT": "5432", - "ATOMIC_REQUESTS": True, - "CONN_MAX_AGE": 60, - "OPTIONS": {}, - "TEST": {"NAME": "temba"}, # use this same database for unit tests -} - -DATABASES = {"default": _default_database_config, "readonly": _default_database_config.copy()} diff --git a/temba/settings_common.py b/temba/settings_common.py index a0bcc984e09..b0860efeade 100644 --- a/temba/settings_common.py +++ b/temba/settings_common.py @@ -18,6 +18,10 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = 2500 # needed for exports of big workspaces +_db_host = "postgres" +_redis_host = "redis" +_minio_host = "minio" + # ----------------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------------- @@ -30,15 +34,6 @@ TEST_RUNNER = "temba.tests.runner.TembaTestRunner" TEST_EXCLUDE = ("smartmin",) -if os.getenv("REMOTE_CONTAINERS") == "true": - _db_host = "postgres" - _redis_host = "redis" - _minio_host = "minio" -else: - _db_host = "localhost" - _redis_host = "localhost" - _minio_host = "localhost" - # ----------------------------------------------------------------------------------- # Email # ----------------------------------------------------------------------------------- From 7d124178a7102e3effe49c6a5b2c8de2424b1b61 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 19 Aug 2024 19:56:31 +0000 Subject: [PATCH 2/2] Build mailroom instead of downloading binary --- .github/workflows/ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e3027affe..869306e3e3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,15 @@ jobs: with: node-version: ${{ env.node-version }} + - name: Install Mailroom + run: | + git clone --depth 1 https://github.com/${{ github.repository_owner }}/mailroom + cd mailroom + go install github.com/${{ github.repository_owner }}/mailroom/cmd/mailroom + # start mailroom but use different redis db so that it doesn't actually handle tasks + ./mailroom -db=postgres://temba:temba@postgres:5432/temba?sslmode=disable -redis=redis://redis:6379/15 -log-level=info > mailroom.log & + working-directory: ${{ github.workspace }} + - name: Initialize environment run: | npm install -g yarn less @@ -60,11 +69,6 @@ jobs: yarn install ln -s settings.py.dev temba/settings.py poetry run python manage.py migrate - # fetch, extract and start mailroom - wget https://github.com/${{ github.repository_owner }}/mailroom/releases/download/v${{ env.mailroom-version }}/mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz - tar -xvf mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz mailroom - # start mailroom but use different redis db so that it doesn't actually handle tasks - ./mailroom -db=postgres://temba:temba@postgres:5432/temba?sslmode=disable -redis=redis://redis:6379/15 -log-level=info > mailroom.log & working-directory: ${{ github.workspace }} # https://github.com/actions/runner/issues/2058#issuecomment-2085119510 - name: Run pre-test checks