feat: add initial support for arm builds (#40550)

bioconda · Mar 4, 2024 · 36731b7 · 36731b7
1 parent a6d2d89
commit 36731b7
Show file tree

Hide file tree

Showing 3 changed files with 315 additions and 2 deletions.
diff --git a/.circleci/check-for-additional-platforms.sh b/.circleci/check-for-additional-platforms.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Check to see if any changed recipes have specified the key
+# extra:additional-platforms, and if so, if they match the platform of the
+# currently-running machine.
+
+# arguments
+git_range=$1
+job_name=$2
+
+# Download ARM version of yq
+yq_platform=$(uname)
+yq_arch=$(uname -m)
+[[ $yq_arch = "aarch64" ]] && yq_arch="arm64"
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_${yq_platform}_${yq_arch} -O /usr/local/bin/yq
+
+# Find recipes changed from this merge
+files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$' `
+build=0
+
+for file in $files; do
+    echo $file
+    # To create a properly-formatted yaml that yq can parse, comment out jinja2
+    # variable setting with {% %} and remove variable use with {{ }}.
+    additional_platforms=$(cat $file \
+    | sed -E 's/(.*)\{%(.*)%\}(.*)/# \1\2\3/g' \
+    | sed -E 's/(.*)\{\{(.*)\}\}(.*)/\1\2\3/g' \
+    | yq '.extra.additional-platforms[]')
+    # Check if any additional platforms match this job
+    for additional_platform in $additional_platforms; do
+    if [ "${CIRCLE_JOB}" = "${job_name}-${additional_platform}" ]
+    then
+        build=1
+        break
+    fi
+    done
+done
+
+# If no changed recipes apply to this platform, skip remaining steps
+if [[ build -lt 1 ]]
+then
+    echo "No recipes using this platform, skipping rest of job."
+    circleci-agent step halt
+fi
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,260 @@
+# .circleci/config.yml
+version: 2.1
+
+executors:
+  # osx-arm64:
+  #   macos:
+  #     xcode: 14.2.0 # indicate your selected version of Xcode
+  #   resource_class: macos.m1.large.gen1
+  linux-aarch64:
+    machine:
+      image: ubuntu-2204:current
+    resource_class: arm.medium
+
+jobs: # a basic unit of work in a run
+  build_and_test:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and test
+          command: |
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils build recipes config.yml \
+                --lint --docker --mulled-test \
+                --docker-base-image quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch) \
+                --git-range origin/master HEAD
+
+      - run:
+          name: Prepare artifacts
+          command: |
+            (
+            mkdir -p /tmp/artifacts/packages
+            cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0
+            find -name .cache | xargs rm -rf || true
+            for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 noarch; do
+              cp -rv $n /tmp/artifacts/packages || true
+            done
+            if command -V docker >/dev/null; then
+              mkdir -p /tmp/artifacts/images
+              cd /tmp/artifacts/images
+              docker image ls --format='{{.Repository}}:{{.Tag}}' | \
+                { grep biocontainers || true ; } | \
+                xargs -n1 -P4 bash -c '
+                  test -n "${1+x}" || exit 0
+                  echo "Start compressing docker image ${1} ..."
+                  docker save "${1}" | gzip -c > "${1##*/}.tar.gz"
+                  echo "Finished compressing docker image ${1} ."
+                  ' --
+            fi
+            ) || true
+
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  build_and_upload:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and push
+          command: |
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils handle-merged-pr recipes config.yml \
+              --repo bioconda/bioconda-recipes \
+              --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \
+              --fallback build \
+              --artifact-source circleci
+
+  bulk_build:
+    parameters:
+      os:
+        type: executor
+      runner:
+        type: integer
+    executor: << parameters.os >>
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - 1e:85:74:42:35:5f:c5:a2:35:c2:ec:b7:80:c0:7c:40
+
+      - checkout
+
+      - run:
+          name: Check for [ci run]
+          command: |
+            commit_message="$(git log --format=oneline -n 1 $CIRCLE_SHA1)"
+            if [[ $commit_message  =~ "[ci run]" ]]; then
+              echo "[ci run] found, continuing."
+            else
+              echo "[ci run] not found, exiting."
+              circleci-agent step halt
+            fi
+
+      - run:
+          name: set git user
+          command: |
+            git config user.name BiocondaBot
+            git config user.email [email protected]
+            git branch --set-upstream-to=origin/$CIRCLE_BRANCH $CIRCLE_BRANCH
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      # For now, do not upload ARM images to biocontainers: --mulled-upload-target biocontainers
+      - run:
+          name: Build and upload
+          command: |
+            set -e
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            echo '============'
+            conda info --all
+            conda config --show-sources
+            python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
+            echo '============'
+            bioconda-utils build recipes config.yml \
+              --worker-offset << parameters.runner >> --n-workers 6 \
+              --docker --mulled-test --docker-base-image quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch) \
+              --anaconda-upload \
+              --mulled-upload-target biocontainers \
+              --record-build-failures \
+              --skiplist-leafs
+            conda clean -y --all
+
+
+workflows:
+  build and test (ARM):
+    jobs:
+      - build_and_test:
+          filters:
+            branches:
+              ignore:
+                - master
+                - bulk
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+  build and upload (ARM):
+    jobs:
+      - build_and_upload:
+          filters:
+            branches:
+              only: master
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+  Bulk branch (ARM):
+    jobs:
+      - bulk_build:
+          filters:
+            branches:
+              only: bulk
+          matrix:
+            parameters:
+              os: 
+                #- osx-arm64
+                - linux-aarch64
+              runner: [0, 1, 2, 3, 4, 5]
diff --git a/recipes/abra2/meta.yaml b/recipes/abra2/meta.yaml
@@ -10,21 +10,30 @@ source:
   sha256: edb35fb6ff318239353a573d1abad148073b8ac0bb2cd44a18e561abe08aef32
 
 build:
-  number: 2
+  number: 3
   skip: true  # [osx]
+  run_exports:
+    - pin_subpackage(name, max_pin="x")
 
 requirements:
   build:
     - maven
     - make
     - {{ compiler('cxx') }}
+  host:
+    - zlib
   run:
     - openjdk >=8,<10
     - coreutils
+    - zlib
 
 test:
   commands:
-     - 'abra2 2>&1 | grep "Abra version"'
+     - 'abra2 2>&1 | grep "Abra version" || (abra2 && exit 1)'
+
+extra:
+  additional-platforms:
+    - linux-aarch64
 
 about:
   home: https://github.com/mozack/abra2