use pyproject

.github/workflows/python-publish.yml

@@ -15,18 +15,14 @@ jobs:
     name: Code QA
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - run: pip install black flake8 isort
-    - run: black --version
-    - run: isort --version
-    - run: flake8 --version
-    - run: isort --check .
-    - run: black --check .
-    - run: flake8 .
+    # - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+    - uses: pre-commit/[email protected]
 
   checks:
-    if: ${{ github.event_name == 'release' }}
-
     strategy:
       fail-fast: false
       matrix:
@@ -35,10 +31,9 @@ jobs:
 
     name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
-    needs: quality
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-python@v2
       with:
@@ -58,32 +53,24 @@ jobs:
 
     if: ${{ github.event_name == 'release' }}
     runs-on: ubuntu-latest
-    needs: checks
+    needs: [checks, quality]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10'
-
-    - name: Check that tag version matches code version
-      run: |
-        tag=${GITHUB_REF#refs/tags/}
-        version=$(python setup.py --version)
-        echo 'tag='$tag
-        echo "version file="$version
-        test "$tag" == "$version"
+        python-version: 3.x
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install build wheel twine
     - name: Build and publish
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
       run: |
-        python setup.py sdist
+        python -m build
         twine upload dist/*

.gitignore

@@ -176,3 +176,4 @@ bar
 *.req
 dev/
 *.out
+_version.py

.pre-commit-config.yaml

@@ -0,0 +1,69 @@
+repos:
+
+# Empty notebookds
+- repo: local
+  hooks:
+    - id: clear-notebooks-output
+      name: clear-notebooks-output
+      files: tools/.*\.ipynb$
+      stages: [commit]
+      language: python
+      entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+      additional_dependencies: [jupyter]
+
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+  - id: end-of-file-fixer # Ensure files end in a newline
+  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.1.1
+  hooks:
+    - id: black
+      args: [--line-length=120]
+
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.3.0
+  hooks:
+  - id: ruff
+    exclude: '(dev/.*|.*_)\.py$'
+    args:
+    - --line-length=120
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview
+
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v0.9.1
+  hooks:
+    - id: sphinx-lint
+
+# For now, we use it. But it does not support a lot of sphinx features
+- repo: https://github.com/dzhu/rstfmt
+  rev: v0.0.14
+  hooks:
+    - id: rstfmt
+
+- repo: https://github.com/b8raoult/pre-commit-docconvert
+  rev: "0.1.4"
+  hooks:
+  - id: docconvert
+    args: ["numpy"]

pyproject.toml

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
+
+[build-system]
+requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+
+[project]
+description = "A package to run AI weather models."
+name = "ai-models"
+
+dynamic = ["version"]
+license = { file = "LICENSE" }
+requires-python = ">=3.9"
+
+authors = [
+    { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "[email protected]" },
+]
+
+keywords = ["tools", "ai"]
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    "entrypoints",
+    "requests",
+    "climetlab>=0.20.11",
+    "multiurl",
+    "ecmwflibs>=0.6.1",
+    "gputil",
+    "earthkit-meteo",
+    "pyyaml",
+    "tqdm",
+]
+
+
+[project.urls]
+Homepage = "https://github.com/ecmwf/ai-models/"
+Repository = "https://github.com/ecmwf/ai-models/"
+Issues = "https://github.com/ecmwf/ai-models/issues"
+
+[project.scripts]
+anemoi-datasets = "ai_models.__main__:main"
+
+[tool.setuptools_scm]
+version_file = "src/ai_models/_version.py"

ai_models/__main__.py → src/ai_models/__main__.py

@@ -12,7 +12,9 @@
 import sys
 
 from .inputs import available_inputs
-from .model import Timer, available_models, load_model
+from .model import Timer
+from .model import available_models
+from .model import load_model
 from .outputs import available_outputs
 
 LOG = logging.getLogger(__name__)
@@ -46,27 +48,19 @@ def _main(argv):
 
     parser.add_argument(
         "--retrieve-requests",
-        help=(
-            "Print mars requests to stdout."
-            "Use --requests-extra to extend or overide the requests. "
-        ),
+        help=("Print mars requests to stdout." "Use --requests-extra to extend or overide the requests. "),
         action="store_true",
     )
 
     parser.add_argument(
         "--archive-requests",
-        help=(
-            "Save mars archive requests to FILE."
-            "Use --requests-extra to extend or overide the requests. "
-        ),
+        help=("Save mars archive requests to FILE." "Use --requests-extra to extend or overide the requests. "),
         metavar="FILE",
     )
 
     parser.add_argument(
         "--requests-extra",
-        help=(
-            "Extends the retrieve or archive requests with a list of key1=value1,key2=value."
-        ),
+        help=("Extends the retrieve or archive requests with a list of key1=value1,key2=value."),
     )
 
     parser.add_argument(
@@ -293,9 +287,7 @@ def _main(argv):
 
     if args.requests_extra:
         if not args.retrieve_requests and not args.archive_requests:
-            parser.error(
-                "You need to specify --retrieve-requests or --archive-requests"
-            )
+            parser.error("You need to specify --retrieve-requests or --archive-requests")
 
     run(vars(args), unknownargs)
 
@@ -313,9 +305,7 @@ def run(cfg: dict, model_args: list):
         sys.exit(0)
 
     # This logic is a bit convoluted, but it is for backwards compatibility.
-    if cfg["retrieve_requests"] or (
-        cfg["requests_extra"] and not cfg["archive_requests"]
-    ):
+    if cfg["retrieve_requests"] or (cfg["requests_extra"] and not cfg["archive_requests"]):
         model.print_requests()
         sys.exit(0)
 

ai_models/checkpoint.py → src/ai_models/checkpoint.py

@@ -55,9 +55,7 @@ def peek(path):
         for b in f.namelist():
             if os.path.basename(b) == "data.pkl":
                 if data_pkl is not None:
-                    raise Exception(
-                        f"Found two data.pkl files in {path}: {data_pkl} and {b}"
-                    )
+                    raise Exception(f"Found two data.pkl files in {path}: {data_pkl} and {b}")
                 data_pkl = b
 
     LOG.info(f"Found data.pkl at {data_pkl}")

ai_models/model.py → src/ai_models/model.py

@@ -54,9 +54,7 @@ def add(self, field):
             self.request[k].add(str(v))
             if k in self.UNIQUE:
                 if len(self.request[k]) > 1:
-                    raise ValueError(
-                        f"Field {field} has different values for {k}: {self.request[k]}"
-                    )
+                    raise ValueError(f"Field {field} has different values for {k}: {self.request[k]}")
 
 
 class Model: