Develop -> Main (#1329)

.github/workflows/_install-aws.yml

@@ -8,6 +8,7 @@ jobs:
     name: Install and configure AWS CLI
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v3
       - name: Install AWS CLI
         run: |
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"

.github/workflows/_tests.yml

@@ -74,17 +74,8 @@ jobs:
         run: poetry run python manage.py check
       - name: Run test suite
         run: poetry run coverage run manage.py test etna
-      - name: Prepare coverage report
-        run: poetry run coverage xml
       - name: Check for missing migrations
         run: poetry run python manage.py makemigrations --check --noinput
-      - name: Upload coverage report to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          env_vars: OS,PYTHON_VERSION
-          fail_ci_if_error: true
-          move_coverage_to_trash: true
 
   prettier:
     name: Prettier format

.github/workflows/branch-cleanup.yml

@@ -8,6 +8,7 @@ jobs:
     if: github.event.ref_type == 'branch' && startsWith(github.event.ref, 'feature/')
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v3
       - name: Get tag
         id: version-tag
         uses: ./.github/workflows/_get-version-tag.yml

.github/workflows/ci.yml

@@ -1,9 +1,7 @@
 name: CI
 
 on:
-  push:
-    branches-ignore:
-      - 'release/**'
+  pull_request:
 
 jobs:
   ci:

.github/workflows/run_codecov.yml

@@ -0,0 +1,61 @@
+name: Code coverage
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  code-coverage:
+    name: Code coverage
+    runs-on: ubuntu-latest
+    env:
+      DJANGO_SETTINGS_MODULE: config.settings.test
+      DATABASE_ENGINE: django.db.backends.postgresql
+      DATABASE_NAME: postgres
+      DATABASE_USER: postgres
+      DATABASE_PASSWORD: postgres
+      DATABASE_HOST: localhost
+      DATABASE_PORT: 5432
+    services:
+      postgres:
+        image: postgres:12.3
+        env:
+          POSTGRES_USER: ${{ env.DATABASE_USER }}
+          POSTGRES_PASSWORD: ${{ env.DATABASE_PASSWORD }}
+          POSTGRES_DB: ${{ env.DATABASE_NAME }}
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 1s --health-timeout 3s --health-retries 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ vars.CI_PYTHON_VERSION }}
+      - uses: snok/install-poetry@v1
+        with:
+          version: ${{ vars.CI_POETRY_VERSION }}
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          virtualenvs-path: .venv
+      - id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: ds-wagtail-venv-${{ runner.os }}-${{ vars.CI_POETRY_VERSION }}-${{ vars.CI_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      - name: Run Django checks
+        run: poetry run python manage.py check
+      - name: Run test suite
+        run: poetry run coverage run manage.py test etna
+      - name: Prepare coverage report
+        run: poetry run coverage xml
+      - name: Upload coverage report to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          env_vars: OS,PYTHON_VERSION
+          fail_ci_if_error: true
+          move_coverage_to_trash: true

.gitignore

@@ -62,4 +62,5 @@ config/settings/local.py
 .npm
 .nvm
 
-media
+media
+coverage.xml

dev/bin/tests

@@ -4,4 +4,4 @@ poetry run isort . --check --diff
 poetry run black . --check --diff
 poetry run flake8 .
 
-manage test
+manage test etna

docker-compose.yml

@@ -42,6 +42,9 @@ services:
       - DATABASE_HOST=db
       - DATABASE_NAME=postgres
       - DATABASE_USER=postgres
+      - DJANGO_SUPERUSER_PASSWORD=admin
+      - DJANGO_SUPERUSER_USERNAME=admin
+      - DJANGO_SUPERUSER_EMAIL=admin@tna-development
       - KEEP_ALIVE=30
 
   cli:

docs/developer-guide/docker.md

@@ -10,7 +10,7 @@ The `Dockerfile` is used to create containers for both local development and dep
 1. Sets the active user to `app`
 1. Sets a [pipefail](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#using-pipes) to ensure one failed command in a piped statment breaks the pipe
 1. Installs Poetry in the `app` user's home directory (`/home/app`)
-1. Copies in the dependency requirment files (`pyproject.toml` and `poetry.lock`)
+1. Copies in the dependency requirement files (`pyproject.toml` and `poetry.lock`)
 1. Installs the project dependencies with Poetry
 1. Copies in all the application code
 1. Ensures the run script can be executed by the current user

docs/developer-guide/project-conventions.md

@@ -1,8 +1,11 @@
 # Etna project conventions
 
+At TNA we follow a set of conventions for our projects to ensure consistency and quality across our codebases. These can be found in our [developer handbook](https://nationalarchives.github.io/developer-handbook/) and should be followed when contributing to the Etna project, as well as the guidance below.
+
 ## Docker for local development
 
 Etna uses Docker to create a consistent environment for local development.
+For more information on Docker inside our project see the [Docker guide](docker.md).
 
 On macOS and Windows, Docker requires [Docker
 Desktop](https://www.docker.com/products/docker-desktop) to be installed. Linux
@@ -30,6 +33,12 @@ Build steps for the `web` container are defined in the project's `Dockerfile`.
 
 Our `web` container has a [dependency](https://docs.docker.com/compose/compose-file/compose-file-v3/#depends_on) on the `db` container, as Django/Wagtail will not run without a database to connect to.
 
+### `dev`
+
+This container is used to run development commands on the `web` container, such as `manage.py` commands and `poetry` commands.
+
+Build steps for the `dev` container are defined in the project's `dev/Dockerfile`.
+
 ### `cli`
 
 A service with the Platform.sh CLI, PHP and few other packages to help with copying of data and media from the environments running in Platform.sh.
@@ -52,19 +61,29 @@ Another option is to run the `format` Fabric command from your console to apply
 $ fab format
 ```
 
+`flake8` will just flag things in the terminal, it will not update any code for you like `isort` or `Black`.
+
 Compliance checks are also built in to the `test` Fabric command - you just need to use the ``--lint`` option to activate them. For example:
 
 ```console
 $ fab test --lint
 ```
 
-### SASS/CSS
+This will be checked by CI on every commit, so it's a good idea to run this locally before pushing your changes.
+
+### SASS/CSS and Javascript
 
-TBC
+The Etna project uses a few tools to improve the consistency and quality of SASS/CSS and JavaScript code:
 
-### Javascript
+- [``Prettier``](https://prettier.io/): An opinionated front-end code formatter that takes care of code formatting (so we don't have to think about it).
+
+To run `Prettier` against your code, simply run the `format` Fabric command from your console:
+
+```console
+$ fab format
+```
 
-TBC
+This will be checked by CI on every commit, so it's a good idea to run this locally before pushing your changes.
 
 ## Git/Github conventions
 

docs/features/analytics-tracking.md

@@ -0,0 +1,5 @@
+# Analytics/Tracking
+
+Analytics and tracking should always be added to new components and features where possible. This prevents us from having to go back and add it retrospectively.
+
+Guidance around analytics and tracking to be written here, TBC.

etna/ciim/constants.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import StrEnum
 from typing import Any, List
 
@@ -47,17 +47,10 @@ class Aggregation(StrEnum):
     LOCATION = "location"
 
 
-DEFAULT_AGGREGATIONS = (
-    Aggregation.COLLECTION,
-    Aggregation.LEVEL,
-    Aggregation.TOPIC,
-    Aggregation.CLOSURE,
-    Aggregation.HELD_BY,
-    Aggregation.CATALOGUE_SOURCE,
+DEFAULT_AGGREGATIONS = [
     Aggregation.GROUP
     + ":30",  # Fetch more 'groups' so that we receive counts for any bucket/tab options we might be showing.
-    Aggregation.TYPE,
-)
+]
 
 
 @dataclass
@@ -70,7 +63,7 @@ class Bucket:
     results: List[Any] = None
 
     # By default, 10 items of each aggregation are requested from the API. This can be overridden by using a string in the format '{name}:{number_of_items}'
-    aggregations: List[str] = DEFAULT_AGGREGATIONS
+    aggregations: List[str] = field(default_factory=lambda: DEFAULT_AGGREGATIONS)
 
     @cached_property
     def aggregations_normalised(self) -> List[str]:
@@ -127,62 +120,72 @@ def __iter__(self):
             key="tna",
             label="Records at The National Archives",
             description="Results for records held at The National Archives that match your search term.",
+            aggregations=DEFAULT_AGGREGATIONS
+            + [Aggregation.COLLECTION, Aggregation.LEVEL, Aggregation.CLOSURE],
         ),
         Bucket(
             key="digitised",
             label="Online records at The National Archives",
             description="Results for records available to download and held at The National Archives that match your search term.",
+            aggregations=DEFAULT_AGGREGATIONS
+            + [Aggregation.COLLECTION, Aggregation.LEVEL, Aggregation.CLOSURE],
         ),
         Bucket(
             key="nonTna",
             label="Records at other UK archives",
             description="Results for records held at other archives in the UK (and not at The National Archives) that match your search term.",
+            aggregations=DEFAULT_AGGREGATIONS
+            + [
+                Aggregation.COLLECTION,
+                Aggregation.CLOSURE,
+                Aggregation.HELD_BY,
+                Aggregation.CATALOGUE_SOURCE,
+            ],
         ),
         Bucket(
             key="creator",
             label="Record creators",
             description="Results for original creators of records (for example organisations, businesses, people, diaries and manors) that match your search term.",
-            aggregations=(
-                Aggregation.GROUP + ":30",
-                Aggregation.TYPE,
-                Aggregation.COUNTRY,
-            ),
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.TYPE, Aggregation.COUNTRY],
         ),
         Bucket(
             key="archive",
             label="Find an archive",
             description="Results for archives in the UK and from across the world that match your search term.",
-            aggregations=(
-                Aggregation.GROUP + ":30",
-                Aggregation.LOCATION,
-            ),
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.LOCATION],
         ),
     ]
 )
 
+
 WEBSITE_BUCKETS = BucketList(
     [
         Bucket(
             key="blog",
             label="Blog posts",
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.TOPIC],
         ),
         Bucket(
             key="researchGuide",
             label="Research Guides",
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.TOPIC],
         ),
         Bucket(
             key=BucketKeys.INSIGHT.value,
             label="Insights",
+            aggregations=DEFAULT_AGGREGATIONS,
         ),
         # TODO: Restore when we are succesfully indexing new highlight pages
         # Bucket(key=BucketKeys.HIGHLIGHT.value, label="Highlights"),
         Bucket(
             key="audio",
             label="Audio",
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.TOPIC],
         ),
         Bucket(
             key="video",
             label="Video",
+            aggregations=DEFAULT_AGGREGATIONS + [Aggregation.TOPIC],
         ),
     ]
 )

etna/feedback/forms.py

@@ -29,6 +29,8 @@ class FeedbackForm(forms.Form):
     page_revision = forms.ModelChoiceField(
         Revision.objects.all(), required=False, widget=forms.HiddenInput
     )
+    page_type = forms.CharField(required=False, widget=forms.HiddenInput)
+    page_title = forms.CharField(required=False, widget=forms.HiddenInput)
 
     def __init__(
         self,
@@ -90,6 +92,7 @@ def clean_url(self) -> Union[str, None]:
             path=normalize_path(parse_result.path),
             query_params=query_params,
         )
+
         return value
 
     def clean_response(self) -> Union[uuid.UUID, None]:

etna/feedback/migrations/0011_feedbacksubmission_page_title_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.4 on 2023-08-30 13:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("feedback", "0010_make_submission_fields_non_editable"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="feedbacksubmission",
+            name="page_title",
+            field=models.TextField(
+                editable=False, null=True, verbose_name="page title"
+            ),
+        ),
+        migrations.AddField(
+            model_name="feedbacksubmission",
+            name="page_type",
+            field=models.TextField(editable=False, null=True, verbose_name="page type"),
+        ),
+    ]

etna/feedback/models.py

@@ -243,6 +243,10 @@ class FeedbackSubmission(models.Model):
         on_delete=models.PROTECT,
         editable=False,
     )
+    page_type = models.TextField(verbose_name=_("page type"), null=True, editable=False)
+    page_title = models.TextField(
+        verbose_name=_("page title"), null=True, editable=False
+    )
     path = models.CharField(
         max_length=255, db_index=True, verbose_name=_("path"), editable=False
     )

etna/feedback/templates/feedback/widgets/response_submit_button_list.html

@@ -3,7 +3,7 @@
     <div{% if id %} id="{{ id }}" class="row justify-content-center"{% endif %}{% if widget.attrs.class %} class="{{ widget.attrs.class }}"{% endif %}>
         {% for group, options, index in widget.optgroups %}
             {% for option in options %}
-                <div class="feedback-response-option col-md-6">
+                <div class="feedback-response-option col-md-4">
                     {% include option.template_name with widget=option %}
                 </div>
             {% endfor %}