Skip to content

Commit

Permalink
CI: Azure w/ Pixi
Browse files Browse the repository at this point in the history
Pixi is a very fast package manager that can install
conda-forge and pypi dependencies.
  • Loading branch information
ax3l committed Sep 2, 2024
1 parent ac9ff7f commit 3d94226
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 84 deletions.
107 changes: 23 additions & 84 deletions .azure-pipelines.yml
Original file line number Diff line number Diff line change
@@ -1,24 +1,17 @@
# -*- mode: yaml -*-

pool:
vmImage: 'ubuntu-20.04'
vmImage: 'ubuntu-22.04'

pr:
autoCancel: true
drafts: false

jobs:
- job:
# FIXME remove unused variables
variables:
BLASPP_HOME: '/usr/local'
CEI_SUDO: 'sudo'
CEI_TMP: '/tmp/cei'
CMAKE_BUILD_PARALLEL_LEVEL: 2
CMAKE_GENERATOR: 'Ninja'
FFTW_HOME: '/usr'
LAPACKPP_HOME: '/usr/local'
WARPX_CI_CCACHE: 'TRUE'
#WARPX_OPENPMD: 'TRUE'

strategy:
matrix:
Expand All @@ -43,7 +36,6 @@ jobs:
# single precision
#single_precision:
# WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE
# WARPX_RZ_FFT: 'TRUE'

# default: 60; maximum: 360
timeoutInMinutes: 240
Expand All @@ -61,101 +53,48 @@ jobs:
Ccache | "$(System.JobName)" | .azure-pipelines.yml
path: /home/vsts/.ccache
cacheHitVar: CCACHE_CACHE_RESTORED
displayName: Cache Ccache Objects

- task: Cache@2
continueOnError: true
inputs:
key: 'Python3 | "$(System.JobName)" | .azure-pipelines.yml'
restoreKeys: |
Python3 | "$(System.JobName)" | .azure-pipelines.yml
path: /home/vsts/.local/lib/python3.8
cacheHitVar: PYTHON38_CACHE_RESTORED
displayName: Cache Python Libraries
displayName: Cache CCache Objects

- bash: |
set -eu -o pipefail
cat /proc/cpuinfo | grep "model name" | sort -u
df -h
echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
sudo apt update
sudo apt install -y ccache curl gcc gfortran git g++ ninja-build \
openmpi-bin libopenmpi-dev \
libfftw3-dev libfftw3-mpi-dev libhdf5-openmpi-dev pkg-config make \
python3 python3-pandas python3-pip python3-venv python3-setuptools libblas-dev liblapack-dev
ccache --set-config=max_size=10.0G
python3 -m pip install --upgrade pip
python3 -m pip install --upgrade build
python3 -m pip install --upgrade packaging
python3 -m pip install --upgrade setuptools
python3 -m pip install --upgrade wheel
python3 -m pip install --upgrade virtualenv
python3 -m pip install --upgrade pipx
python3 -m pipx install cmake
python3 -m pipx ensurepath
export PATH="$HOME/.local/bin:$PATH"
sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
sudo chmod a+x /usr/local/bin/cmake-easyinstall
#if [ "${WARPX_OPENPMD:-FALSE}" == "TRUE" ]; then
# cmake-easyinstall --prefix=/usr/local \
# git+https://github.com/openPMD/[email protected] \
# -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
# -DCMAKE_VERBOSE_MAKEFILE=ON \
# -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
# #python3 -m pip install --upgrade openpmd-api
#fi
if [ "${WARPX_RZ_FFT:-FALSE}" == "TRUE" ]; then
# BLAS++
cmake-easyinstall --prefix=/usr/local \
git+https://github.com/icl-utk-edu/blaspp.git \
-DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
-DCMAKE_CXX_STANDARD=17 \
-Duse_openmp=OFF -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
# LAPACK++
cmake-easyinstall --prefix=/usr/local \
git+https://github.com/icl-utk-edu/lapackpp.git \
-DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
-DCMAKE_CXX_STANDARD=17 \
-Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
fi
if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then
cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/[email protected] \
-DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
-DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \
-DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \
-DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \
-DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \
-DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
-DHeffte_ENABLE_MAGMA=OFF \
-DCMAKE_VERBOSE_MAKEFILE=ON
fi
# Python modules required for test analysis
python3 -m pip install --upgrade -r Regression/requirements.txt
python3 -m pip cache purge
# configure in pyproject.toml [tool.pixi.dependencies]
curl -fsSL https://pixi.sh/install.sh | bash
export PATH=$HOME/.pixi/bin:$PATH
pixi install
displayName: Install dependencies
- bash: |
set -eu -o pipefail
export PATH=$HOME/.pixi/bin:$PATH
# external repositories required for test analysis
cd ..
git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git ..
# TODO select only specific datasets?
git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git
cd -
rm -rf ${CEI_TMP}
git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git ..
df -h
displayName: 'Install dependencies'
displayName: Download test data
- bash: |
set -eu -o pipefail
df -h
# configure
export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON"
cmake -S . -B build \
${AMReX_CMAKE_FLAGS} \
${WARPX_CMAKE_FLAGS} \
-DWarpX_TEST_CLEANUP=ON \
-DWarpX_TEST_FPETRAP=ON
# build
cmake --build build -j 2
# run tests (exclude pytest.AMReX when running Python tests)
ctest --test-dir build --output-on-failure -E AMReX
rm -rf build
df -h
displayName: 'Build & test'
displayName: Build & test
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# GitHub syntax highlighting
pixi.lock linguist-language=YAML linguist-generated=true
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ Docs/source/_static/doxyhtml
####################
# Package Managers #
####################
# Pixi
# https://pixi.sh
pixi.lock
# anonymous Spack environments
# https://spack.readthedocs.io/en/latest/environments.html#anonymous-environments
.spack-env/
Expand Down Expand Up @@ -70,3 +73,6 @@ cmake-build-*/
.DS_Store
.AppleDouble
.LSOverride
# pixi environments
.pixi
*.egg-info
54 changes: 54 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,57 @@ known-first-party = ["amrex", "picmistandard", "pywarpx", "warpx"]
[tool.isort]
known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"]
profile = "black"

[tool.pixi.project]
name = "warpx-omp-dev"
channels = ["conda-forge"]
platforms = ["linux-64"]

[tool.pixi.dependencies]
boost = ">=1.85.0"
cmake = ">=3.24.0"
compilers = "*"
dill = "*"
fftw = ">=3.3"
git = ">=2.18"
heffte = { version = ">=2.4.0", build = "mpi_mpich_*" }
lapackpp = ">=2022.5.0"
lasy = ">=0.5.0"
matplotlib = "*"
mpi = { build = "mpich" }
mpi4py = "*"
ninja = "*"
numpy = "*"
openpmd-api = { version = ">=0.15.2", build = "mpi_mpich_*" }
openpmd-viewer = "*"
packaging = "*"
pandas = "*"
periodictable = "*"
picmistandard = "==0.30.0" # see requirements.txt
pip = ">=24.2"
pkg-config = "*"
pybind11 = ">=2.12.0"
python = "*"
setuptools = ">=42"
scipy ="*"
virtualenv = "*"
wheel = ">=0.44.0"
yt = ">=4.3.1"

# For packages not yet available on Conda-Forge
# [tool.pixi.pypi-dependencies]
# lasy = ">=0.5.0"
# picmistandard = "==0.30.0" # see requirements.txt
# pywarpx = { path = ".", editable = true }

[tool.linux-64.dependencies]
libgomp = "*"

[tool.win-64.dependencies]
llvm-openmp = "*"

[tool.osx-64.dependencies]
llvm-openmp = "*"

[tool.osx-arm64.dependencies]
llvm-openmp = "*"

0 comments on commit 3d94226

Please sign in to comment.