Merge pull request #38 from ecmwf/develop

Release preparation

.github/ci-config.yml

@@ -0,0 +1,3 @@
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages

.github/workflows/changelog-pr-update.yml

@@ -0,0 +1,12 @@
+name: Check Changelog Update on PR
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened, labeled, unlabeled]
+jobs:
+  Check-Changelog:
+    name: Check Changelog Action
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: tarides/changelog-check-action@v2
+        with:
+          changelog: CHANGELOG.md

.github/workflows/changelog-release-update.yml

@@ -0,0 +1,34 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.release.target_commitish }}
+
+    - name: Update Changelog
+      uses: stefanzweifel/changelog-updater-action@v1
+      with:
+        latest-version: ${{ github.event.release.tag_name }}
+        heading-text: ${{ github.event.release.name }}
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        add-paths: |
+          CHANGELOG.md

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+    paths:
+      - "src/**"
+      - "tests/**"
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manuallyp instals
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-models: ecmwf/anemoi-models@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+   # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-models: ecmwf/anemoi-models@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

.github/workflows/python-publish.yml

@@ -35,50 +35,19 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install
       run: |
-        pip install pytest
-        pip install -e .[all]
-        pip install -r tests/requirements.txt
+        pip install -e .[all,tests]
         pip freeze
 
     - name: Tests
       run: pytest
 
   deploy:
-
-    if: ${{ github.event_name == 'release' }}
-    runs-on: ubuntu-latest
     needs: [checks, quality]
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.10'
-
-    - name: Check that tag version matches code version
-      run: |
-        tag=${GITHUB_REF#refs/tags/}
-        version=$(python setup.py --version)
-        echo 'tag='$tag
-        echo "version file="$version
-        test "$tag" == "$version"
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-      run: |
-        python setup.py sdist
-        twine upload dist/*
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

.github/workflows/readthedocs-pr-update.yml

@@ -0,0 +1,22 @@
+name: Read the Docs PR Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    # Execute this action only on PRs that touch
+    # documentation files.
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "anemoi-models"

.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/
@@ -184,3 +185,4 @@ _build/
 ?.*
 ~*
 *.sync
+_version.py

.pre-commit-config.yaml

@@ -1,19 +1,16 @@
 repos:
-
 # Empty notebookds
 - repo: local
   hooks:
-    - id: clear-notebooks-output
-      name: clear-notebooks-output
-      files: tools/.*\.ipynb$
-      stages: [commit]
-      language: python
-      entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
-      additional_dependencies: [jupyter]
-
-
+  - id: clear-notebooks-output
+    name: clear-notebooks-output
+    files: tools/.*\.ipynb$
+    stages: [commit]
+    language: python
+    entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+    additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: check-yaml # Check YAML files for syntax errors only
     args: [--unsafe, --allow-multiple-documents]
@@ -23,13 +20,11 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
-
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.1.1
+  rev: 24.8.0
   hooks:
-    - id: black
-      args: [--line-length=120]
-
+  - id: black
+    args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
   rev: 5.13.2
   hooks:
@@ -38,26 +33,38 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
-
-
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.0
+  rev: v0.6.3
   hooks:
   - id: ruff
-    exclude: '(dev/.*|.*_)\.py$'
+    # Next line if for documenation cod snippets
+    exclude: '^[^_].*_\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
-
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v0.9.1
   hooks:
-    - id: sphinx-lint
-
+  - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
 - repo: https://github.com/dzhu/rstfmt
   rev: v0.0.14
   hooks:
-    - id: rstfmt
+  - id: rstfmt
+    exclude: 'cli/.*' # Because we use argparse
+- repo: https://github.com/b8raoult/pre-commit-docconvert
+  rev: "0.1.5"
+  hooks:
+  - id: docconvert
+    args: ["numpy"]
+- repo: https://github.com/b8raoult/optional-dependencies-all
+  rev: "0.0.6"
+  hooks:
+  - id: optional-dependencies-all
+    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: "2.2.1"
+  hooks:
+  - id: pyproject-fmt

.readthedocs.yaml

@@ -3,11 +3,14 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.11"
 
 sphinx:
   configuration: docs/conf.py
 
 python:
    install:
-     - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

CHANGELOG.md

@@ -0,0 +1,57 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
+
+## [Unreleased]
+
+### Added
+ - CI workflow to update the changelog on release
+ - Remapper: Preprocessor for remapping one variable to multiple ones. Includes changes to the data indices since the remapper changes the number of variables. With optional config keywords.
+
+### Changed
+
+ - Update CI to inherit from common infrastructue reusable workflows
+ - run downstream-ci only when src and tests folders have changed
+ - New error messages for wrongs graphs.
+
+### Removed
+
+## [0.2.1] - Dependency update
+
+### Added
+
+- downstream-ci pipeline
+- readthedocs PR update check action
+
+### Removed
+
+- anemoi-datasets dependency
+
+## [0.2.0] - Support Heterodata
+
+### Added
+
+- Option to choose the edge attributes
+
+### Changed
+
+- Updated to support new PyTorch Geometric HeteroData structure (defined by `anemoi-graphs` package).
+
+## [0.1.0] - Initial Release
+
+### Added
+- Documentation
+- Initial code release with models, layers, distributed, preprocessing, and data_indices
+- Added Changelog
+
+<!-- Add Git Diffs for Links above -->
+[unreleased]: https://github.com/ecmwf/anemoi-models/compare/0.2.1...HEAD
+[0.2.1]: https://github.com/ecmwf/anemoi-models/compare/0.2.0...0.2.1
+[0.2.0]: https://github.com/ecmwf/anemoi-models/compare/0.1.0...0.2.0
+[0.1.0]: https://github.com/ecmwf/anemoi-models/releases/tag/0.1.0

README.md

@@ -1,4 +1,4 @@
-# anemoi-utils
+# anemoi-models
 
 **DISCLAIMER**
 This project is **BETA** and will be **Experimental** for the foreseeable future.