PR-Based Repro CI

.github/workflows/ci.yml

@@ -0,0 +1,167 @@
+name: CI
+run-name: CI (${{ github.event_name }}) for ${{ github.ref_name }}
+on:
+  # For config-pr-1-ci.yml
+  pull_request:
+    branches:
+      - 'release-*'
+      - 'dev-*'
+    paths-ignore:
+      # These are ignored because they don't have anything to do with the model itself
+      - .github/**
+      - tools/**
+      - doc/**
+      - config/**
+      - .*
+      - README.md
+  # For config-pr-2-confirm.yml
+  issue_comment:
+    types:
+      - created
+      - edited
+  # For config-pr-3-bump-tag.yml
+  create:
+  push:
+    branches:
+      - 'release-*'
+    paths:
+      - 'metadata.yaml'
+env:
+  # NOTE: This is also used in `jobs.pr.with.additional-artifact-content-paths` but we can't use `env` context in a `with` section.
+  PAYU_OUTPUT_DIR: archive/output000
+jobs:
+  ###########
+  # PR jobs #
+  ###########
+  pr:
+    name: PR
+    if: github.event_name == 'pull_request' && github.repository != 'ACCESS-NRI/model-configs-template'
+    uses: access-nri/model-config-tests/.github/workflows/config-pr-1-ci.yml@main
+    with:
+      additional-artifact-content-paths: |
+        archive/output000/MOM_parameter_doc.*
+    secrets: inherit
+    permissions:
+      contents: write
+      pull-requests: write  # For pull request comments denoting failure of the workflow
+      checks: write  # For results of tests as a check
+
+  pr-docs:
+    name: PR Docs
+    needs:
+      - pr
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACT_LOCAL_LOCATION: /opt/artifact
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.pr.outputs.repro-ci-artifact-name }}
+          path: ${{ env.ARTIFACT_LOCAL_LOCATION }}
+
+      - name: Diff docs/
+        id: diff
+        run: |
+          mv ${{ env.ARTIFACT_LOCAL_LOCATION }}/${{ env.PAYU_OUTPUT_DIR }}/* ./docs
+          git add .
+          if git diff --cached --exit-code ./docs; then
+            echo "::warning::Modification to 'docs/', need to commit with '!update docs'"
+            exit 1
+          fi
+
+      - name: Diff docs/ notifier
+        if: failure() && steps.diff.outcome == 'failure'
+        env:
+          REPO_TARGET_DOCS_URL: https://${{ github.server_url }}/${{ github.repository}}/tree/${{ github.head_ref }}/docs
+          REPO_SOURCE_ARTIFACT_URL: ${{ needs.pr.outputs.repro-ci-artifact-url }}
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          pr: ${{ github.event.issue.number }}
+          comment: |
+            The `docs` in ${{ env.REPO_TARGET_DOCS_URL }} differ from ${{ env.REPO_SOURCE_ARTIFACT_URL }} (in directory `${{ env.PAYU_OUTPUT_DIR }}`).
+
+            Comment `!update_docs` if you want these changes committed.
+
+  ###################
+  # PR Comment jobs #
+  ###################
+  pr-comment:
+    name: Comment
+    if: github.event_name == 'issue_comment' && github.repository != 'ACCESS-NRI/model-configs-template'
+    uses: access-nri/model-config-tests/.github/workflows/config-pr-2-confirm.yml@main
+    secrets: inherit
+    permissions:
+      contents: write  # For updating metadata.yaml version and committing checksums
+      pull-requests: write  # For commenting on PR
+
+  pr-comment-docs:
+    name: Comment Docs
+    if: github.event_name == 'issue_comment' && github.event.comment.body == '!update_docs' && github.repository != 'ACCESS-NRI/model-configs-template'
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACT_LOCAL_LOCATION: /opt/artifact
+      GH_TOKEN: ${{ github.token }}
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: access-nri/actions/.github/actions/react-to-comment@main
+        with:
+          reaction: rocket
+          token: ${{ github.token }}
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+
+      - name: Checkout Associated PR ${{ github.event.issue.number }}
+        # Since the trigger for this workflow was on.issue_comment, we need
+        # to do a bit more wrangling to checkout the pull request and get the branch name
+        id: pr
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Download Docs
+        # Given the PR branch, we need to find the latest associated workflow run
+        # on this branch we can then download the associated artifact
+        run: |
+          associated_run=$(gh run list \
+            --json='databaseId,headBranch,updatedAt,status' \
+            --jq='[.[] | select(.headBranch == "${{ steps.pr.outputs.branch }}" and .status == "completed")] | sort_by(.updatedAt) | last | .databaseId')
+          gh run download $associated_run -D ${{ env.ARTIFACT_LOCAL_LOCATION }}
+
+      - name: Import Commit-Signing Key
+        uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4  # v6.1.0
+        with:
+          gpg_private_key: ${{ secrets.GH_ACTIONS_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GH_ACTIONS_BOT_GPG_PASSPHRASE }}
+          git_config_global: true
+          git_committer_name: ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git_committer_email: ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
+      - name: Commit docs/
+        env:
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          mv ${{ env.ARTIFACT_LOCAL_LOCATION }}/${{ env.PAYU_OUTPUT_DIR }}/* ./docs
+          git add .
+          git commit -m "Updated docs as part of ${{ env.RUN_URL }}"
+          git push
+
+  #################
+  # Bump Tag jobs #
+  #################
+  bump-tag:
+    name: Tag Bump
+    if: (github.event_name == 'push' || github.event_name == 'create' && github.ref_type == 'branch' && startsWith(github.ref_name, 'release-')) && github.repository != 'ACCESS-NRI/model-configs-template'
+    uses: access-nri/model-config-tests/.github/workflows/config-pr-3-bump-tag.yml@main
+    secrets: inherit
+    permissions:
+      contents: write  # For creating a new release

.github/workflows/config.yml

@@ -0,0 +1,26 @@
+name: CI Config
+run-name: CI Config
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - config/**
+jobs:
+  validate:
+    name: Validate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Logging Validation Inputs
+        run: echo '::notice::Validating config/ci.json with ${{ vars.CI_JSON_SCHEMA_VERSION }}'
+
+      - name: Checkout CI Config
+        uses: actions/checkout@v4
+
+      - name: config/ci.json
+        uses: access-nri/schema/.github/actions/validate-with-schema@main
+        with:
+          schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }}
+          meta-schema-version: draft-2020-12
+          schema-location: au.org.access-nri/model/configuration/ci
+          data-location: config/ci.json

CONTRIBUTING.md

@@ -0,0 +1,77 @@
+# Contributing
+
+## Changes to the CI Infrastructure
+
+Changes to the CI Infrastructure are made to the `main` branch in this repository. Config branches use the `ci.yml` workflows to `workflow_call` the equivalent workflow that is in [`model-config-tests`](https://github.com/ACCESS-NRI/model-config-tests).
+
+## Dev and Release branches
+
+Each configuration has a `dev-*` and a `release-*` branch. They differ in the CI checks that are run when pull requests are made to update the branch. Any branch starting with either `dev-*` or  `release-*` are protected branches. You cannot (and should not) modify them directly or create new branches starting with either `dev-` or `release-`.
+
+### Dev
+
+The `dev-*` branch is where a configuration is updated. Temporary branches should be created and a pull request made to update the `dev-*` branch. Quality assurance (QA) CI checks are run on pull requests to `dev-*` branches, but not reproducibility checks. There is no requirement that the version be updated when changes are made to the `dev-` branch. So the `dev-` branch of a configuration allows for smaller changes that can be accumulated before a PR is made to the respective `release-*` branch.
+
+### Release
+
+Pull requests to the `release-*` branch should be made from the respective `dev-*` branch and are intended to create a new version of the configuration. These pull requests have CI quality assurance (QA) checks that ensure the model configuration is suitable for release. CI Model reproducibility checks are also conducted: a short test run of the configuration is checked for bitwise reproducibility. The success or otherwise of this check determines if a minor or major version bump is required.
+
+It is expected that the version *will* be updated before the pull request can be merged. This in turn creates a new tag for that configuration branch. It can be confusing for users if there are a large number of versions of a configuration and it is of little benefit to them. For this reason the atomicity of updates to a released configuration should be minimised, i.e.  updates should be meaningful.
+
+## Creation of a new ACCESS-OM2 Config
+
+Config branches are entirely separate from the `main` history in this repository, except for a few files in `.github`. Note, you may need to be an Administrator to commit to `release-*` or `dev-*` branches directly.
+
+### Brand new configuration
+
+If you are creating a brand new configuration, and don't have the config stored in another repository, just checkout a `dev-*` branch from `main` and delete everything except `.github/workflows/ci.yml`, then add your config.
+
+### Config is Stored in Another Repository
+
+Create a `dev-*` branch by adding the config repository as a remote and checking out the config branch:
+
+```bash
+git remote add <config_repo> <config_repo_url>  # ex. git remote add config [email protected]/my/configs.git
+git checkout <config_repo>/<config_branch> -b dev-<config_name>  # checkout config from new remote + add to branch, ex. git checkout config/main -b dev-1deg_abc_def
+git checkout main -- .github/workflows/ci.yml
+git add .
+git commit -m "Initial commit for config branch"
+git push  # might require admin permissions for pushes to dev-* branch
+```
+
+### Create a new release branch
+
+For a brand new configuration there is no existing `release-*` branch, so one needs to be created. Follow the pull request process outlined below to update the dev branch so that it is passing QA checks. At this point create a `release-*` branch from the `dev-` branch and `git push` it to the repository:
+
+```bash
+git checkout -b release-<config_name>
+git push release-<config_name>
+```
+
+For the CI workflows to work correctly the `release-` branch needs to have a version set, and a reproducibility checksum committed. There is a convenience workflow for this purpose: [Generate Initial Checksums](https://github.com/ACCESS-NRI/access-om2-configs/actions/workflows/generate-initial-checksums.yml). Click the "Run workflow" menu, fill in the fields and push the green "Run workflow" button.
+
+Once the workflow is completed there should be a new commit on the `release-*` branch, and a [tag](https://github.com/ACCESS-NRI/access-om2-configs/tags) for the specified version.
+
+Once the `release-*` branch has been updated those changes need to be merged **back** into the `dev-*` branch. This step is only necessary when the `release-*` branch is updated independently of the `dev-*` branch.
+
+## Pull Request Process
+
+### Update dev config
+
+1. Make your changes, test them, and open a PR from a feature/change branch (or fork) to the `dev-*` branch of a particular configuration.
+2. QA checks will run to ensure the configuration meets criteria for a released configuration, and to ensure consistency of released configurations.
+3. Fix the problems identified in the QA checks, commit and push to the PR branch.
+4. Once all checks pass the pull request branch can be merged.
+5. Consider making a PR to the equivalent `release-*` branch.
+
+Note: If this is a brand new configuration and there is no existing `release-*` branch you will [need to create one first](#create-a-new-release-branch).
+
+### Update release config
+
+1. Open a PR from the `dev-*` branch of a particular configuration to the equivalent `release-*` branch
+2. QA checks will run to ensure the configuration meets criteria for a released configuration, and to ensure consistency of released configurations.
+3. Checks will also run to test if changes break reproducibility with the current major version config tag on the target branch. For example, if you are opening a PR on the `release-025deg_jra55do_ryf` branch, and the last tagged version on this branch is `release-025deg_jra55do_ryf-1.2`, the checksums between the config in your PR and the checksum in the config tag are compared.
+4. A comment will be posted on the PR when this is completed, notifying you whether the checksums match (in this example meaning a minor bump to `*-1.3`), or are different (meaning a major bump to `*-2.0`).
+5. Optionally, you can now modify your PR and get more reproducibility checks. Particularly in the case where bitwise reproducibility should be retained this is an opportunity to modify the configuration to enable this.
+6. Bump the version using the `!bump [major|minor]` command depending on the result of the reproducibility check. Additionally, if the checksums are different, the updated checksum will be automatically committed to the PR. Bumping the version in some way is a requirement before the PR will be mergable.
+7. Merge the PR

README.md

@@ -22,12 +22,12 @@ according to the following naming scheme:
 Additional required information, like if the configuration includes
 biogeochemistry, is appended to the name.
 
-Currently the following configurations are available:
+Currently the following development configurations are available:
 
-- [`1deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/1deg_jra55do_ryf)
-- [`1deg_jra55do_iaf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/1deg_jra55do_iaf)
-- [`1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/tree/1deg_jra55do_ryf_wombatlite)
-- [`025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/025deg_jra55do_ryf)
+- [`dev-1deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/dev-1deg_jra55do_ryf)
+- [`dev-1deg_jra55do_iaf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/dev-1deg_jra55do_iaf)
+- [`dev-1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/tree/dev-1deg_jra55do_ryf_wombatlite)
+- [`dev-025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/tree/dev-025deg_jra55do_ryf)
 
 **Note that the [`main`](https://github.com/ACCESS-NRI/access-om3-configs/tree/main) branch
 does not store any configuration, only some documentation.**
@@ -43,12 +43,12 @@ These configurations should **not** be used for production runs.
 
 ## Comparison table
 
-- [`1deg_jra55do_ryf`➡️`1deg_jra55do_iaf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_ryf..1deg_jra55do_iaf)
-- [`1deg_jra55do_ryf`➡️`1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_ryf..1deg_jra55do_ryf_wombatlite)
-- [`1deg_jra55do_ryf`➡️`025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_ryf..025deg_jra55do_ryf)
-- [`1deg_jra55do_iaf`➡️`1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_iaf..1deg_jra55do_ryf_wombatlite)
-- [`1deg_jra55do_iaf`➡️`025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_iaf..025deg_jra55do_ryf)
-- [`1deg_jra55do_ryf_wombatlite`➡️`025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_ryf_wombatlite..025deg_jra55do_ryf)
+- [`dev-1deg_jra55do_ryf`➡️`dev-1deg_jra55do_iaf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_ryf..dev-dev-1deg_jra55do_iaf)
+- [`dev-1deg_jra55do_ryf`➡️`dev-1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_ryf..dev-1deg_jra55do_ryf_wombatlite)
+- [`dev-1deg_jra55do_ryf`➡️`dev-025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_ryf..dev-025deg_jra55do_ryf)
+- [`dev-1deg_jra55do_iaf`➡️`dev-1deg_jra55do_ryf_wombatlite`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_iaf..dev-1deg_jra55do_ryf_wombatlite)
+- [`dev-1deg_jra55do_iaf`➡️`dev-025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_iaf..dev-025deg_jra55do_ryf)
+- [`dev-1deg_jra55do_ryf_wombatlite`➡️`dev-025deg_jra55do_ryf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/dev-1deg_jra55do_ryf_wombatlite..dev-025deg_jra55do_ryf)
 
 ## Setting up an experiment
 
@@ -115,6 +115,10 @@ branch in your fork to the branch it originated from in [ACCESS-NRI/access-om3-c
 
 ## Configuration CI
 
+### PR Repro CI
+
+This Pipeline compares configurations modified in a PR (the `source` branch) against a 'ground truth' checksum, usually the `target` branch. It also verifies that commons mistakes in configurations are not made. This allows developers to know if the changes they are about to commit lead to valid and reproducible results. Either way, if the PR is merged, the new commit is tagged in such a way that we know how reproducible it is against past configurations.
+
 ### User-Dispatchable Repro-CI Workflow
 
 This repository contains a user-dispatchable workflow (minimum `Write` role required) for the generation of reproducibility checksums on a given Config Branch. The workflow requires sign off from [@ACCESS-NRI/ocean](https://github.com/orgs/ACCESS-NRI/teams/ocean) to run on Gadi.