CI: Azure w/ Pixi

Pixi is a very fast package manager that can install conda-forge and pypi dependencies.
EZoni · Sep 2, 2024 · 8b7c101 · 8b7c101
1 parent ac9ff7f
commit 8b7c101
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 89 deletions.
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -1,24 +1,18 @@
 # -*- mode: yaml -*-
 
 pool:
-  vmImage: 'ubuntu-20.04'
+  vmImage: 'ubuntu-22.04'
 
 pr:
   autoCancel: true
   drafts: false
 
 jobs:
 - job:
-  # FIXME remove unused variables
   variables:
-    BLASPP_HOME: '/usr/local'
-    CEI_SUDO: 'sudo'
-    CEI_TMP: '/tmp/cei'
-    CMAKE_GENERATOR: 'Ninja'
-    FFTW_HOME: '/usr'
-    LAPACKPP_HOME: '/usr/local'
-    WARPX_CI_CCACHE: 'TRUE'
-    #WARPX_OPENPMD: 'TRUE'
+    AMReX_CMAKE_FLAGS: -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON
+    CMAKE_BUILD_PARALLEL_LEVEL: 2
+    CMAKE_GENERATOR: Ninja
 
   strategy:
     matrix:
@@ -31,19 +25,18 @@ jobs:
       # Cartesian 3D
       cartesian_3d:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON
-        WARPX_HEFFTE: 'TRUE'
+        WARPX_HEFFTE: TRUE
       # Cylindrical RZ
       cylindrical_rz:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON
-        WARPX_RZ_FFT: 'TRUE'
+        WARPX_RZ_FFT: TRUE
       # embedded boundaries
       embedded_boundaries:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_EB=ON
-        WARPX_RZ_FFT: 'TRUE'
+        WARPX_RZ_FFT: TRUE
       # single precision
       #single_precision:
       #  WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE
-      #  WARPX_RZ_FFT: 'TRUE'
 
   # default: 60; maximum: 360
   timeoutInMinutes: 240
@@ -61,101 +54,52 @@ jobs:
          Ccache | "$(System.JobName)" | .azure-pipelines.yml
       path: /home/vsts/.ccache
       cacheHitVar: CCACHE_CACHE_RESTORED
-    displayName: Cache Ccache Objects
-
-  - task: Cache@2
-    continueOnError: true
-    inputs:
-      key: 'Python3 | "$(System.JobName)" | .azure-pipelines.yml'
-      restoreKeys: |
-         Python3 | "$(System.JobName)" | .azure-pipelines.yml
-      path: /home/vsts/.local/lib/python3.8
-      cacheHitVar: PYTHON38_CACHE_RESTORED
-    displayName: Cache Python Libraries
+    displayName: Cache CCache Objects
 
   - bash: |
       set -eu -o pipefail
       cat /proc/cpuinfo | grep "model name" | sort -u
       df -h
-      echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
-      sudo apt update
-      sudo apt install -y ccache curl gcc gfortran git g++ ninja-build \
-        openmpi-bin libopenmpi-dev \
-        libfftw3-dev libfftw3-mpi-dev libhdf5-openmpi-dev pkg-config make \
-        python3 python3-pandas python3-pip python3-venv python3-setuptools libblas-dev liblapack-dev
-      ccache --set-config=max_size=10.0G
-      python3 -m pip install --upgrade pip
-      python3 -m pip install --upgrade build
-      python3 -m pip install --upgrade packaging
-      python3 -m pip install --upgrade setuptools
-      python3 -m pip install --upgrade wheel
-      python3 -m pip install --upgrade virtualenv
-      python3 -m pip install --upgrade pipx
-      python3 -m pipx install cmake
-      python3 -m pipx ensurepath
-      export PATH="$HOME/.local/bin:$PATH"
-      sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
-      sudo chmod a+x /usr/local/bin/cmake-easyinstall
-      #if [ "${WARPX_OPENPMD:-FALSE}" == "TRUE" ]; then
-      #  cmake-easyinstall --prefix=/usr/local                   \
-      #    git+https://github.com/openPMD/[email protected] \
-      #    -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)         \
-      #    -DCMAKE_VERBOSE_MAKEFILE=ON                           \
-      #    -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
-      #  #python3 -m pip install --upgrade openpmd-api
-      #fi
-      if [ "${WARPX_RZ_FFT:-FALSE}" == "TRUE" ]; then
-        # BLAS++
-        cmake-easyinstall --prefix=/usr/local           \
-          git+https://github.com/icl-utk-edu/blaspp.git \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
-          -DCMAKE_CXX_STANDARD=17                       \
-          -Duse_openmp=OFF -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
-        # LAPACK++
-        cmake-easyinstall --prefix=/usr/local             \
-          git+https://github.com/icl-utk-edu/lapackpp.git \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)   \
-          -DCMAKE_CXX_STANDARD=17                         \
-          -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
-      fi
-      if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/[email protected] \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
-          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
-          -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF    \
-          -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF      \
-          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
-          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
-          -DHeffte_ENABLE_MAGMA=OFF                              \
-          -DCMAKE_VERBOSE_MAKEFILE=ON
-      fi
-      # Python modules required for test analysis
-      python3 -m pip install --upgrade -r Regression/requirements.txt
-      python3 -m pip cache purge
+
+      # Setup pixi for Azure Pipelines
+      curl -fsSL https://pixi.sh/install.sh | bash
+      export PATH=$HOME/.pixi/bin:$PATH
+      echo "##vso[task.setvariable variable=PATH]$PATH"
+
+      # configure in pyproject.toml [tool.pixi.dependencies]
+      pixi install
+      pixi list
+    displayName: Install dependencies
+
+  - bash: |
+      set -eu -o pipefail
+
       # external repositories required for test analysis
-      cd ..
-      git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
+      git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git ../warpx-data
       # TODO select only specific datasets?
-      git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git
-      cd -
-      rm -rf ${CEI_TMP}
+      git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git ../openPMD-example-datasets
+
       df -h
-    displayName: 'Install dependencies'
+    displayName: Download test data
 
   - bash: |
+      eval "$(pixi shell-hook)"
+
       set -eu -o pipefail
-      df -h
+
       # configure
-      export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON"
       cmake -S . -B build       \
         ${AMReX_CMAKE_FLAGS}    \
         ${WARPX_CMAKE_FLAGS}    \
         -DWarpX_TEST_CLEANUP=ON \
         -DWarpX_TEST_FPETRAP=ON
+
       # build
       cmake --build build -j 2
+
       # run tests (exclude pytest.AMReX when running Python tests)
       ctest --test-dir build --output-on-failure -E AMReX
+
       rm -rf build
       df -h
-    displayName: 'Build & test'
+    displayName: Build & test
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true
diff --git a/.gitignore b/.gitignore
@@ -43,6 +43,9 @@ Docs/source/_static/doxyhtml
 ####################
 # Package Managers #
 ####################
+# Pixi
+# https://pixi.sh
+pixi.lock
 # anonymous Spack environments
 # https://spack.readthedocs.io/en/latest/environments.html#anonymous-environments
 .spack-env/
@@ -70,3 +73,6 @@ cmake-build-*/
 .DS_Store
 .AppleDouble
 .LSOverride
+# pixi environments
+.pixi
+*.egg-info
diff --git a/pyproject.toml b/pyproject.toml
@@ -20,3 +20,57 @@ known-first-party = ["amrex", "picmistandard", "pywarpx", "warpx"]
 [tool.isort]
 known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"]
 profile = "black"
+
+[tool.pixi.project]
+name = "warpx-omp-dev"
+channels = ["conda-forge"]
+platforms = ["linux-64"]
+
+[tool.pixi.dependencies]
+boost = ">=1.85.0"
+cmake = ">=3.24.0"
+compilers = "*"
+dill = "*"
+fftw = ">=3.3"
+git = ">=2.18"
+heffte = { version = ">=2.4.0", build = "mpi_mpich_*" }
+lapackpp = ">=2022.5.0"
+lasy = ">=0.5.0"
+matplotlib = "*"
+mpi = { build = "mpich" }
+mpi4py = "*"
+ninja = "*"
+numpy = "*"
+openpmd-api = { version = ">=0.15.2", build = "mpi_mpich_*" }
+openpmd-viewer = "*"
+packaging = "*"
+pandas = "*"
+periodictable = "*"
+picmistandard = "==0.30.0"  # see requirements.txt
+pip = ">=24.2"
+pkg-config = "*"
+pybind11 = ">=2.12.0"
+python = "*"
+setuptools = ">=42"
+scipy ="*"
+virtualenv = "*"
+wheel = ">=0.44.0"
+yt = ">=4.3.1"
+
+# For packages not yet available on Conda-Forge
+# [tool.pixi.pypi-dependencies]
+# lasy = ">=0.5.0"
+# picmistandard = "==0.30.0"  # see requirements.txt
+# pywarpx = { path = ".", editable = true }
+
+[tool.linux-64.dependencies]
+libgomp = "*"
+
+[tool.win-64.dependencies]
+llvm-openmp = "*"
+
+[tool.osx-64.dependencies]
+llvm-openmp = "*"
+
+[tool.osx-arm64.dependencies]
+llvm-openmp = "*"