From 5a82b213df00d8b0c94eb04b63c579e60a28ee64 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 10 Sep 2024 15:07:28 +1000 Subject: [PATCH] Add User-Dispatchable Repro Tests (#121) * Added initial ci.json configuration file * Added initial checksum generation workflow * README.md: Updated to include reference to `config/ci.json` and user dispatch workflows * README.md: Fixed markdown linting errors * Update README.md Co-authored-by: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> --------- Co-authored-by: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> --- .../workflows/generate-initial-checksums.yml | 77 +++++++++++++++++++ README.md | 51 +++++++++--- config/ci.json | 23 ++++++ 3 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/generate-initial-checksums.yml create mode 100644 config/ci.json diff --git a/.github/workflows/generate-initial-checksums.yml b/.github/workflows/generate-initial-checksums.yml new file mode 100644 index 0000000..4d921fa --- /dev/null +++ b/.github/workflows/generate-initial-checksums.yml @@ -0,0 +1,77 @@ +name: Generate Initial Checksums +on: + workflow_dispatch: + inputs: + config-branch-name: + type: string + required: true + description: The configuration branch that will be run that will generate the checksums. + commit-checksums: + type: boolean + required: true + description: Commit the checksums to the target branch once generated. They will still be stored as a workflow artifact. + committed-checksum-location: + type: string + required: false + default: ./testing/checksum + description: "If checksums are being committed: Where in the repository the generated checksums should be committed to." + committed-checksum-tag-version: + type: string + required: false + description: "If checksums are being committed: The initial version for the git tag associated with the committed checksums." +jobs: + config: + name: Read Testing Configuration + runs-on: ubuntu-latest + outputs: + payu-version: ${{ steps.repro-config.outputs.payu-version }} + model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Validate + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }} + meta-schema-version: draft-2020-12 + schema-location: au.org.access-nri/model/configuration/ci + data-location: config/ci.json + + - name: Read reproducibility tests config + id: repro-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: reproducibility + branch-or-tag: ${{ inputs.config-branch-name }} + config-filepath: "config/ci.json" + + log-inputs: + name: Log Inputs + needs: + - config + runs-on: ubuntu-latest + steps: + - run: | + echo '::notice::This deployment is using the following inputs: `config-branch-name`=`${{ inputs.config-branch-name }}`, `commit-checksums`=`${{ inputs.commit-checksums }}`, `committed-checksum-location`=`${{ inputs.committed-checksum-location }}`,`committed-checksum-tag-version`=`${{ inputs.committed-checksum-tag-version }}`.' + echo '::notice::This deployment is using Payu Version ${{ needs.config.outputs.payu-version }} and Model Config Tests Version ${{ needs.config.outputs.model-config-tests-version }}' + + generate-checksums: + name: Generate Checksums + needs: + - log-inputs + - config + uses: access-nri/model-config-tests/.github/workflows/generate-checksums.yml@main + with: + config-branch-name: ${{ inputs.config-branch-name }} + commit-checksums: ${{ inputs.commit-checksums }} + committed-checksum-location: ${{ inputs.committed-checksum-location }} + committed-checksum-tag: ${{ inputs.committed-checksum-tag-version != '' && format('{0}-{1}', inputs.config-branch-name, inputs.committed-checksum-tag-version) || '' }} + environment-name: "Gadi Initial Checksum" + model-config-tests-version: ${{ needs.config.outputs.model-config-tests-version }} + payu-version: ${{ needs.config.outputs.payu-version }} + permissions: + contents: write + secrets: inherit diff --git a/README.md b/README.md index 3ef719c..878b389 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ following components: All the configurations use the [payu](https://payu.readthedocs.io/en/latest/) workflow management tool. - ## Repository structure Each configuration is stored as a git branch. Most of the branches are named @@ -42,7 +41,6 @@ for testing ACCESS-OM3: These configurations should **not** be used for production runs. - ## Comparison table - [`1deg_jra55do_ryf`➡️`1deg_jra55do_iaf`](https://github.com/ACCESS-NRI/access-om3-configs/compare/1deg_jra55do_ryf..1deg_jra55do_iaf) @@ -75,10 +73,10 @@ Here is an step-by-step example of how to set up a `1deg_jra55do_ryf` experiment (called `my_1deg_jra55do_ryf_experiment_name`) after setting up your fork: ```bash -$ git clone git@github.com:/access-om3-configs.git my_1deg_jra55do_ryf_experiment_name -$ cd my_1deg_jra55do_ryf_experiment_name -$ git checkout 1deg_jra55do_ryf -$ git checkout -b my_1deg_jra55do_ryf_experiment_name +git clone git@github.com:/access-om3-configs.git my_1deg_jra55do_ryf_experiment_name +cd my_1deg_jra55do_ryf_experiment_name +git checkout 1deg_jra55do_ryf +git checkout -b my_1deg_jra55do_ryf_experiment_name ``` Here `` should be your GitHub user name. @@ -90,24 +88,23 @@ the `config.yaml` file and change the following line: ```yaml runlog: false ``` + to + ```yaml runlog: true ``` - ## Customising your experiment See [this section of the quick start instructions in the ACCESS-OM3 wiki](https://github.com/COSIMA/access-om3/wiki/Quick-start#customising-your-experiment). - ## Running your experiment See [this section of the quick start instructions in the ACCESS-OM3 wiki](https://github.com/COSIMA/access-om3/wiki/Quick-start#running). - ## Pull requests We welcome contributions from users of these configurations. If you make a @@ -115,3 +112,39 @@ configuration improvement which you think should be included in the [ACCESS-NRI/ repository, push it to your fork and then do a pull request from the relevant branch in your fork to the branch it originated from in [ACCESS-NRI/access-om3-configs](https://github.com/ACCESS-NRI/access-om3-configs) (not `main`). + +## Configuration CI + +### User-Dispatchable Repro-CI Workflow + +This repository contains a user-dispatchable workflow (minimum `Write` role required) for the generation of reproducibility checksums on a given Config Branch. The workflow requires sign off from [@ACCESS-NRI/ocean](https://github.com/orgs/ACCESS-NRI/teams/ocean) to run on Gadi. + +Workflow inputs : + +| Input | Type | Required | Default | Description | Example | Notes | +| ----- | ---- | -------- | ------- | ----------- | ------- | ----- | +| `config-branch-name` | `string` | `true` | N/A | The configuration branch that will be run that will generate the checksums | `dev-025deg_jra55do_ryf` | This can be any branch - not just `release` or `dev` branches | +| `commit-checksums` | `boolean` | `true` | `false` | Whether to commit the checksums to the target branch once generated | `true` | If unchecked, the checksums are still accessible as a workflow run artifact | +| `committed-checksum-location` | `string` | `false` | `./testing/checksum` | If checksums are being committed: Where in the repository the generated checksums should be committed | `./some/dir` | Requires the path starting with `.` | +| `committed-checksum-tag-version` | `string` | `false` | N/A | If checksums are being committed: An optional initial version for the committed checksums as a `git tag` of the form `{config-branch-name}-{version}` | `1.0` | If left blank, no tag will be added | + +### Configuring the CI: `config/ci.json` + +This is the `config/ci.json` configuration file for specifying different test markers, or test versions based on type of the test to run, and the name of the git branch or tag. The different types of test are defined as: + +- `scheduled`: Scheduled monthly reproducibility tests. The keys under these tests represent released config tags to run scheduled checks on. +- `reproducibility`: Reproducibility tests that are run as part of pull requests. The keys under these tests represent the target branches into which pull requests are being merged. +- `qa` - Quick quality assurance tests that are run as part of pull requests. The keys under these tests represent the target branches into which pull requests are being merged. + +The configuration properties needed to run the tests are: + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | -------- | +| markers | `string` | Markers used for the pytest checks, in the python format | `checksum` | +| model-config-tests-version | `string` | The version of the model-config-tests | `0.0.1` | +| python-version | `string` | The python version used to create test virtual environment on Github hosted tests | `3.11.0` | +| payu-version | `string` | The Payu version used to run the model | `1.1.5` | + +As most of the tests use the same test and python versions, and similar markers, there are two levels of defaults. There's a default at test type level which is useful for defining test markers - this selects certain pytests to run in `model-config-tests`. There is an outer global default, which is used if a property is not defined for a given branch/tag, and it is not defined for the test default. The `parse-ci-config` action applies the fall-back default logic. For more information on using this action see [`ACCESS-NRI/model-config-tests`](https://github.com/ACCESS-NRI/model-config-tests/). + +The CI for this file (in [`config.yml`](./.github/workflows/config.yml)) validates modifications to the `ci.json` against it's schema, found in [`ACCESS-NRI/schema`](https://github.com/ACCESS-NRI/schema). It does not yet verify that modifications make sense. diff --git a/config/ci.json b/config/ci.json new file mode 100644 index 0000000..838d28e --- /dev/null +++ b/config/ci.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://github.com/ACCESS-NRI/schema/tree/main/au.org.access-nri/model/configuration/ci/2-0-0.json", + "scheduled": { + "default": { + "markers": "checksum" + } + }, + "reproducibility": { + "default": { + "markers": "checksum" + } + }, + "qa": { + "default": { + "markers": "access_om3 or config" + } + }, + "default": { + "model-config-tests-version": "0.0.7", + "python-version": "3.11.0", + "payu-version": "1.1.5" + } +}