Upgrade Django (3.2.23 -> 4.2.14) and Wagtail (4.2.2 -> 5.2.6)

.github/workflows/deploy.yml

@@ -8,7 +8,9 @@ on:
 
 jobs:
   deploy:
-    runs-on: ubuntu-20.04
+    runs-on:
+      - self-hosted
+      - philly-hip
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/test.yml

@@ -7,7 +7,9 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on:
+      - self-hosted
+      - philly-hip
     env:
       DJANGO_SETTINGS_MODULE: hip.settings.dev
     services:

.pre-commit-config.yaml

@@ -29,3 +29,4 @@ repos:
         args: [--allow-missing-credentials]
       - id: detect-private-key
       - id: no-commit-to-branch
+

Dockerfile

@@ -123,6 +123,7 @@ RUN groupadd --gid $USER_GID $USERNAME \
 #   vim -- enhanced vi editor for commits
 ENV KUBE_CLIENT_VERSION="v1.29.3"
 ENV HELM_VERSION="3.14.4"
+ENV POSTGRESQL_CLIENT_VERSION="15"
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
     --mount=type=cache,mode=0755,target=/root/.cache/pip \
     set -ex \

README.md

@@ -294,3 +294,36 @@ To reset your local database from a deployed environment:
 As mentioned in the Database setup instructions, you may need to visit 
 [/cms/sites](http://localhost:8000/cms/sites/) and change the first entry's 
 `Hostname` field to `localhost` to enable page previews in the Wagtail admin.
+
+### GitHub Actions Runner
+
+There are [GitHub Actions self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) deployed on a virtual machine within the same VPC. Other runners may be added in the future, if needed.
+
+Setup instructions:
+
+* Obtain a [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with the `admin:org` scope that's valid for one week (it needs to be active only for the initial deployment). Add this to a local environment variable `RUNNER_CFG_PAT`:
+
+```sh
+export RUNNER_CFG_PAT="gh......"
+```
+
+* Run the playbook to deploy the runner:
+
+```sh
+cd deploy/
+# first time: connect as ubuntu user
+ansible-playbook -u ubuntu deploy-runner.yml
+# second time and beyond
+ansible-playbook deploy-runner.yml
+```
+
+* The runner can be forcibly reinstalled by passing `-e force_reinstall=yes` or removed by passing `-e force_removal=yes`.
+
+Run OS updates:
+
+```sh
+cd deploy/
+ansible-playbook run-os-updates.yml
+# You may need to specify your username
+ansible-playbook -u myusername run-os-updates.yaml
+```

apps/common/context_processors.py

@@ -32,7 +32,7 @@ def previous_url(request):
     # Note: we use the try/except logic here (rather than request.META.get...)
     # in order to avoid calling get_home_page_url() every time.
     try:
-        return {"previous_url": request.META["HTTP_REFERER"]}
+        return {"previous_url": request.headers["referer"]}
     except KeyError:
         return {"previous_url": get_home_page_url()}
 

apps/common/utils.py

@@ -89,7 +89,7 @@ def get_next_url_from_request(request):
     if next_parameter:
         return next_parameter
     else:
-        return request.META.get("HTTP_REFERER", None) or None
+        return request.headers.get("referer", None) or None
 
 
 def closedpod_user_check(user):

apps/reports/tests/factories.py

@@ -8,7 +8,6 @@
 
 class ExternalReportBlockFactory(wagtail_factories.StructBlockFactory):
     title = "title here"
-    # title = factory.faker.Faker("word")
     url = factory.faker.Faker("url")
     update_frequency = random.choice(
         ["Annually", "Warterly", "Monthly", "Weekly", "all the time!!!"]

apps/reports/tests/test_models.py

@@ -68,58 +68,36 @@ def test_datareportslistpage_context_only_internal_reports(db, rf):
 def test_datareportslistpage_context_only_external_reports(db, rf):
     """Test the context for a DataReportListPage with no children, but with external_reports."""
     # Create a DataReportListPage with some external reports.
+    external_report_hiv = {
+        "title": "HIV Report",
+        "url": "https://example.com/hiv",
+        "update_frequency": "Monthly",
+        "last_updated": date(year=2021, month=1, day=1),
+    }
     external_report_hep_a = {
-        "title": "Hepatitis A",
-        "url": "example.com/report-hepatitis-a",
+        "title": "Hepatitis A Report",
+        "url": "https://example.com/hep-a",
         "update_frequency": "Annually",
-        # The context should handle if last_updated is a string
         "last_updated": "2020-01-01",
-        "external": True,
     }
-    external_report_hiv = {
-        "title": "HIV/AIDS",
-        "url": "example.com/report-hiv-aids",
-        "update_frequency": "Sometime",
-        # The context should handle if last_updated is a date object
-        "last_updated": date(year=2021, month=1, day=1),
-        "external": True,
-    }
-    external_reports_data = [
-        {
-            "type": "external_reports",  # Block type
-            "value": {
-                "title": external_report_hiv["title"],
-                "url": external_report_hiv["url"],
-                "update_frequency": external_report_hiv["update_frequency"],
-                "last_updated": external_report_hiv["last_updated"],
-            },
-        },
-        {
-            "type": "external_reports",
-            "value": {
-                "title": external_report_hep_a["title"],
-                "url": external_report_hep_a["url"],
-                "update_frequency": external_report_hep_a["update_frequency"],
-                "last_updated": external_report_hep_a["last_updated"],
-            },
-        },
-    ]
     reports_list_page = DataReportListPageFactory(
-        external_reports=external_reports_data
+        external_reports=[
+            ("external_reports", external_report_hiv),
+            ("external_reports", external_report_hep_a),
+        ]
     )
-
     context = reports_list_page.get_context(rf.get("/someurl/"))
 
     expected_result_hep_a = external_report_hep_a.copy()
-    expected_result_hep_a["last_updated"] = date(
-        year=2020, month=1, day=1
-    )  # format date to be the expected type
+    expected_result_hep_a["last_updated"] = date(year=2020, month=1, day=1)
     expected_result_hiv = external_report_hiv.copy()
-    expected_result_hiv["last_updated"] = date(
-        year=2021, month=1, day=1
-    )  # format date to be the expected type
-    # The results should be data for the external reports alphabetical order.
-    assert [expected_result_hep_a, expected_result_hiv] == context["reports"]
+    expected_result_hiv["last_updated"] = date(year=2021, month=1, day=1)
+    context_hep_a = context["reports"][0]
+    context_hiv = context["reports"][1]
+    del context_hep_a["external"]
+    del context_hiv["external"]
+    assert expected_result_hep_a == context_hep_a
+    assert context_hiv == expected_result_hiv
 
 
 def test_datareportslistpage_context_internal_and_external_reports(db, rf):
@@ -130,7 +108,6 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
         "url": "example.com/report-hiv-aids",
         "update_frequency": "Sometime",
         "last_updated": "2021-01-01",
-        "external": True,
     }
     external_reports_data = [
         {
@@ -156,7 +133,6 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
     )
 
     context = reports_list_page.get_context(rf.get("/someurl/"))
-
     # The results should be data for the internal and external reports in alphabetical order.
     expected_result_external_report_hiv = external_report_hiv.copy()
     expected_result_external_report_hiv["last_updated"] = date(
@@ -181,6 +157,7 @@ def test_datareportslistpage_context_internal_and_external_reports(db, rf):
             "external": False,
         },
     ]
+    del context["reports"][1]["external"]
     assert expected_results == context["reports"]
 
 

apps/users/admin.py

@@ -18,6 +18,7 @@ class Meta:
         exclude = ("password",)
 
 
+@admin.register(User)
 class CustomUserAdmin(ExportMixin, UserAdmin):
     resource_class = UserResourceBlackList
 
@@ -48,6 +49,3 @@ class CustomUserAdmin(ExportMixin, UserAdmin):
     add_fieldsets = (
         (None, {"classes": ("wide",), "fields": ("email", "password1", "password2")}),
     )
-
-
-admin.site.register(User, CustomUserAdmin)

apps/users/models.py

@@ -8,8 +8,11 @@
 
 
 class User(AbstractBaseUser, PermissionsMixin, IndexedTimeStampedModel):
-    # use special Postgres-only Case Insensitive Email Field
-    # https://docs.djangoproject.com/en/dev/ref/contrib/postgres/fields/#django.contrib.postgres.fields.CIEmailField
+    # Support for CIEmailField will be depreacated after Django 4.2 in favor
+    # of EmailField. Below is how to migrate to the vanilla EmailField Whilst
+    # creating a non-deterministic collation. Can't migrate yet because it requires
+    # Postgres 12 or higher. Currently Hip Philly is running Postgres 11.
+    # https://adamj.eu/tech/2023/02/23/migrate-django-postgresql-ci-fields-case-insensitive-collation/
     email = CIEmailField(max_length=255, unique=True)
     first_name = models.CharField(_("first name"), max_length=150, blank=True)
     last_name = models.CharField(_("last name"), max_length=150, blank=True)

deploy/deploy-runner.yml

@@ -0,0 +1,126 @@
+---
+- hosts: runner
+  become: yes
+  tags: base
+  roles:
+    - caktus.hosting_services.users
+  tasks:
+    - name: Set hostname
+      hostname:
+        name: "{{ inventory_hostname_short }}"
+      when: inventory_hostname_short is defined and inventory_hostname_short
+    - name: Add new hostname to /etc/hosts
+      lineinfile:
+        path: /etc/hosts
+        regexp: '^127\.0\.1\.1'
+        line: '127.0.1.1 {{ inventory_hostname_short }}'
+        owner: root
+        group: root
+        mode: 0644
+      when: inventory_hostname_short is defined and inventory_hostname_short
+
+- name: Install GitHub Actions Runner
+  hosts: runner
+  tags: runner
+  become: yes
+  tasks:
+    - name: Create runner user
+      ansible.builtin.user:
+        name: "{{ github_runner_user }}"
+        comment: Github Actions Runner
+    # Install Docker
+    # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository
+    - name: Install dependencies
+      ansible.builtin.package:
+        name:
+          - jq
+          - ca-certificates
+          - curl
+          - gnupg
+          - lsb-release
+          - libpq-dev
+          - libffi-dev
+          - python3.10
+          - python3.10-dev
+    - name: Add Docker's official GPG key
+      ansible.builtin.apt_key:
+        url: https://download.docker.com/linux/ubuntu/gpg
+        state: present
+    - name: Add Docker repository
+      ansible.builtin.apt_repository:
+        repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
+        state: present
+    - name: Install Docker Engine
+      ansible.builtin.package:
+        name:
+          - docker-ce
+          - docker-ce-cli
+          - containerd.io
+          - docker-buildx-plugin
+          - docker-compose-plugin
+        update_cache: yes
+    - name: Task name
+      stat:
+        path: /home/{{ github_runner_user }}/runner
+      register: runner_dir
+    - name: Set vars
+      set_fact:
+        run_removal_tasks: >-
+          {{
+            runner_dir.stat.exists
+            and (
+              (force_reinstall is defined and force_reinstall == "yes")
+              or (force_removal is defined and force_removal == "yes")
+            )
+          }}
+    # Various complicated Ansible roles exist, but this just works:
+    # https://github.com/actions/runner/blob/main/docs/automate.md
+    - name: Remove the runner
+      ansible.builtin.shell:
+        cmd: >
+          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/remove-svc.sh
+          | bash -s {{ github_scope }}
+        chdir: /home/{{ github_runner_user }}
+      environment:
+        RUNNER_CFG_PAT: "{{ github_pat }}"
+      when: run_removal_tasks
+      ignore_errors: True
+    - name: Delete the runner
+      ansible.builtin.shell:
+        cmd: >
+          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/delete.sh
+          | bash -s {{ github_scope }} {{ github_runner_name }}
+        chdir: /home/{{ github_runner_user }}
+      environment:
+        RUNNER_CFG_PAT: "{{ github_pat }}"
+      when: run_removal_tasks
+      ignore_errors: True
+    - name: Remove old runner directory and files
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - /home/{{ github_runner_user }}/runner
+      when: run_removal_tasks
+    - name: Add user '{{ github_runner_user }}' to group docker
+      user:
+        name: "{{ github_runner_user }}"
+        groups: docker
+        append: yes
+    - name: Restart docker service
+      ansible.builtin.service:
+        name: docker
+        state: restarted
+    - name: Install the runner [If error, RUNNER_CFG_PAT might be missing or expired! See README.md]
+      ansible.builtin.shell:
+        cmd: >
+          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/create-latest-svc.sh
+          | bash -s --
+          -s {{ github_scope }}
+          -n {{ github_runner_name }}
+          -l {{ github_runner_location }},self-hosted
+          -u {{ github_runner_user }}
+        chdir: /home/{{ github_runner_user }}
+      environment:
+        RUNNER_CFG_PAT: "{{ github_pat }}"
+      when: (not runner_dir.stat.exists) or (force_reinstall is defined and force_reinstall=="yes")