Skip to content

Commit

Permalink
Merge pull request #229 from traversaro/rebuild-christmas-2024-linux-…
Browse files Browse the repository at this point in the history
…64-rattler-build

Rebuild January 2025 - switch to rattler-build
  • Loading branch information
Tobias-Fischer authored Jan 6, 2025
2 parents 71d4b56 + 0218b40 commit aa18162
Show file tree
Hide file tree
Showing 105 changed files with 5,298 additions and 2,646 deletions.
847 changes: 0 additions & 847 deletions .ci_support/conda_forge_pinnings.yaml

This file was deleted.

2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# GitHub syntax highlighting
pixi.lock linguist-language=YAML linguist-generated=true
15 changes: 0 additions & 15 deletions .github/testpr_environment.yml

This file was deleted.

4 changes: 3 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
name: Generate CI pipelines for ROS 2 packages

permissions:
actions: write
contents: write
Expand All @@ -18,7 +20,7 @@ jobs:
python-version: '3.11' # Version range or exact version of a Python version to use, using SemVer's version range syntax
- name: Install vinca
run: |
pip install git+https://github.com/RoboStack/vinca.git
pip install git+https://github.com/RoboStack/vinca.git@rattler-build-humble
- name: Generate recipes for linux-64
run: |
Expand Down
164 changes: 66 additions & 98 deletions .github/workflows/testpr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,71 +8,51 @@ env:

jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, macos-14, windows-2019]
platform: [linux-64, linux-aarch64, osx-64, win-64, osx-arm64]
exclude:
- os: ubuntu-latest
platform: win-64
- os: ubuntu-latest
platform: osx-64
include:
- os: ubuntu-latest
platform: osx-arm64
- os: macos-14
platform: linux-64
- os: macos-14
- os: cirun-linux-aarch64--${{ github.run_id }}
platform: linux-aarch64
- os: macos-14
- os: macos-13
platform: osx-64
- os: macos-14
platform: win-64
- os: macos-latest
platform: linux-64
- os: macos-latest
platform: linux-aarch64
- os: macos-latest
platform: osx-arm64
- os: macos-latest
platform: win-64
- os: windows-2019
platform: linux-64
- os: windows-2019
platform: linux-aarch64
- os: windows-2019
platform: osx-64
- os: windows-2019
platform: osx-arm64
platform: win-64

runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v4
with:
persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
fetch-depth: 0 # otherwise, you will failed to push refs to dest repo

- uses: prefix-dev/[email protected]
with:
environments: beta
frozen: true

# Workaround for https://github.com/RoboStack/ros-humble/pull/141#issuecomment-1941919816
- name: Clean up PATH
if: contains(matrix.os, 'windows')
uses: egor-tensin/cleanup-path@v4
with:
# cygpath in C:\Program Files\Git\usr\bin is used by install micromamba
# git in C:\Program Files\Git\bin is used by pip install git+
# git in C:\Program Files\Git\bin is used by pip install git+
dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin'

- uses: actions/checkout@master
- name: install micromamba
uses: mamba-org/setup-micromamba@main
with:
environment-file: .github/testpr_environment.yml
condarc: |
channels:
- robostack-staging
- conda-forge
- shell: bash -l {0}
if: matrix.platform == 'linux-aarch64'
run: |
echo "::group::Configure binfmt_misc"
docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
export GIT_BRANCH="$(basename $GITHUB_REF)"
echo "::endgroup::"
# - shell: bash -l {0}
# if: matrix.platform == 'linux-aarch64'
# run: |
# echo "::group::Configure binfmt_misc"
# docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
# export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
# export GIT_BRANCH="$(basename $GITHUB_REF)"
# echo "::endgroup::"

- name: Install libgl1-mesa-dev (only for linux-aarch64)
if: matrix.platform == 'linux-aarch64'
Expand All @@ -81,24 +61,15 @@ jobs:
sudo apt-get update
sudo apt-get install -y libgl1-mesa-dev libglu1-mesa-dev libx11-dev libxi-dev libxmu-dev
- shell: bash -l {0}
run: |
conda config --show channels | grep defaults && conda config --remove channels defaults
conda info
conda config --show
micromamba install -y pip rospkg networkx "catkin_pkg>=0.4.16" "ruamel.yaml>=0.16.6" "rosdistro>=0.8.0" "empy>=3.3.4" boa
- name: install vinca
shell: bash -l {0}
run: |
# use no-deps for now, otherwise problems with ruamel.
pip install git+https://github.com/RoboStack/vinca.git --no-deps
# For some reason, the Strawberry perl's pkg-config is found
# instead of the conda's one, so let's delete the /c/Strawberry directory
# Furthermore, we also need to remove an older SDK that is used and can result in compilation problems
- name: Debug pkg-config problem
if: contains(matrix.os, 'windows')
shell: bash -l {0}
run: |
rm -rf /c/Strawberry
rm -rf "/c/Program Files (x86)/Windows Kits/10/Include/10.0.17763.0/"
- name: Check what files have changed
id: filecheck
Expand All @@ -119,85 +90,82 @@ jobs:
echo "::set-output name=WIN_YAML_CHANGED::${?}"
- name: Generate recipes for linux-64
shell: bash -l {0}
if: steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-64'
if: steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.platform == 'linux-64'
run: |
git clean -fdx
cp vinca_linux_64.yaml vinca.yaml
vinca --platform linux-64
ls
cat recipe.yaml
mkdir -p recipes
$HOME/.pixi/bin/pixi run -e beta -v vinca --platform linux-64 -m -n
ls -la recipes
- name: Generate recipes for linux-aarch64
shell: bash -l {0}
if: steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-aarch64'
if: steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.platform == 'linux-aarch64'
run: |
git clean -fdx
cp vinca_linux_aarch64.yaml vinca.yaml
vinca --platform linux-aarch64
ls
cat recipe.yaml
mkdir -p recipes
$HOME/.pixi/bin/pixi run -e beta -v vinca --platform linux-aarch64 -m -n
ls -la recipes
- name: Generate recipes for osx-64
shell: bash -l {0}
if: steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.os == 'macos-latest' && matrix.platform == 'osx-64'
if: steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.platform == 'osx-64'
run: |
git clean -fdx
cp vinca_osx.yaml vinca.yaml
vinca --platform osx-64
ls
cat recipe.yaml
mkdir -p recipes
$HOME/.pixi/bin/pixi run -e beta -v vinca --platform osx-64 -m -n
ls -la recipes
- name: Generate recipes for osx-arm64
shell: bash -l {0}
if: steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.os == 'macos-14' && matrix.platform == 'osx-arm64'
if: steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.platform == 'osx-arm64'
run: |
git clean -fdx
cp vinca_osx_arm64.yaml vinca.yaml
vinca --platform osx-arm64
ls
cat recipe.yaml
mkdir -p recipes
$HOME/.pixi/bin/pixi run -e beta -v vinca --platform osx-arm64 -m -n
ls -la recipes
- name: Generate recipes for win-64
shell: bash -l {0}
if: steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.os == 'windows-2019' && matrix.platform == 'win-64'
if: steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.platform == 'win-64'
run: |
# Workaround for problem related to long paths
echo "CONDA_BLD_PATH=C:\\bld\\" >> $GITHUB_ENV
git clean -fdx
mkdir /c/bld
cp vinca_win.yaml vinca.yaml
vinca --platform win-64
ls
cat recipe.yaml
mkdir -p recipes
$HOME/.pixi/bin/pixi run -e beta -v vinca --platform win-64 -m -n
ls -la recipes
- name: Check if there are packages to be built
id: newrecipecheck
shell: bash -l {0}
run: |
# continue on error
set +e
grep -q "outputs: \[\]" recipe.yaml > /dev/null
test ! -d recipes
echo "::set-output name=RECIPE_CREATED::${?}"
- name: Build recipes for linux-64
shell: bash -l {0}
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.platform == 'linux-64'
run: |
boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-64'
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform linux-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform linux-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
- name: Build recipes for linux-aarch64
shell: bash -l {0}
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-aarch64'
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.platform == 'linux-aarch64'
run: |
boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=linux-aarch64
boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=linux-aarch64
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform linux-aarch64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform linux-aarch64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
- name: Build recipes for osx-64
shell: bash -l {0}
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.os == 'macos-latest' && matrix.platform == 'osx-64'
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.platform == 'osx-64'
run: |
boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform osx-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform osx-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
- name: Build recipes for osx-arm64
shell: bash -l {0}
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.os == 'macos-14' && matrix.platform == 'osx-arm64'
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.platform == 'osx-arm64'
run: |
boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=osx-arm64
boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform osx-arm64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform osx-arm64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
- name: Build recipes for win-64
shell: bash -l {0}
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.os == 'windows-2019' && matrix.platform == 'win-64'
if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.platform == 'win-64'
run: |
boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
$HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform win-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
$HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes -m ./conda_build_config.yaml --target-platform win-64 -c robostack-staging -c conda-forge --skip-existing
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,7 @@ recipes/
*.ps1
*.json
.DS_Store
# pixi environments
.pixi
*.egg-info
output/
68 changes: 13 additions & 55 deletions .scripts/build_linux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,67 +9,25 @@ set -xeuo pipefail
export PYTHONUNBUFFERED=1
export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}"
export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}"
# export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support"
# export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml"

cat >~/.condarc <<CONDARC
conda-build:
root-dir: /opt/conda/build_artifacts
CONDARC
curl -fsSL https://pixi.sh/install.sh | bash
export PATH="$HOME/.pixi/bin:$PATH"

# This doesn't work, see e.g. https://dev.azure.com/robostack/ros_pipelines/_build/results?buildId=188&view=logs&j=4e20d398-0572-5e54-89c7-6bdb9c00a59a&t=f5885ff8-badf-54b3-1543-35164851bdf3
# if grep -q libgl "recipes/${CURRENT_BUILD_PKG_NAME}.yaml"; then
# sudo yum install -y install mesa-libGL-devel
# fi

export "CONDA_BLD_PATH=/opt/conda/build_artifacts"

mkdir -p $CONDA_BLD_PATH
conda index $CONDA_BLD_PATH

conda config --set remote_max_retries 5
conda config --add channels conda-forge
conda config --add channels robostack-staging
conda config --add channels $CONDA_BLD_PATH
conda config --remove channels defaults
# conda config --set channel_priority strict

mamba update conda --yes --quiet -c conda-forge
mamba install --yes --quiet pip conda-build anaconda-client mamba boa

# setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
# export PATH="$HOME/miniconda/bin:$PATH"
conda config --set anaconda_upload yes
conda config --set show_channel_urls true
conda config --set auto_update_conda false
conda config --set add_pip_as_python_dependency false

conda info
conda config --show-sources
conda list --show-channel-urls
export CONDA_BLD_PATH="/opt/conda/build_artifacts"

pwd

for recipe in ${CURRENT_RECIPES[@]}; do
cd ${FEEDSTOCK_ROOT}/recipes/${recipe}
boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml
done

anaconda -t ${ANACONDA_API_TOKEN} upload /opt/conda/build_artifacts/linux-*/*.tar.bz2 --force
# quetz-client "${QUETZ_URL}" /opt/conda/build_artifacts --force

# set up the condarc
cd ${FEEDSTOCK_ROOT}

# source run_conda_forge_build_setup

# # make the build number clobber
# make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
for recipe in ${CURRENT_RECIPES[@]}; do
pixi run -v rattler-build build \
--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
-m ${FEEDSTOCK_ROOT}/conda_build_config.yaml \
-c robostack-staging -c conda-forge \
--output-dir $CONDA_BLD_PATH

# conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \
# --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml"
# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \

# if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then
# upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
# fi
done

# touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}"
pixi run upload ${CONDA_BLD_PATH}/linux-*/*.conda --force
Loading

0 comments on commit aa18162

Please sign in to comment.