feat: download aarch64 artifacts from CircleCI (#921)

- Support for downloading artifacts from CircleCI instead of Azure.
- Added option `--artifact-source` with `choices=['azure', 'circleci']`
to `handle_merged_pr`
- Check architecture when returning platform
- See PR for the ARM builds
bioconda/bioconda-recipes#40550

bioconda_utils/artifacts.py

@@ -9,6 +9,8 @@
 
 import requests
 import backoff
+import json
+from pathlib import Path
 from bioconda_utils import utils
 from bioconda_utils.upload import anaconda_upload, skopeo_upload
 
@@ -18,7 +20,7 @@
 IMAGE_RE = re.compile(r"(.+)(?::|%3A)(.+)\.tar\.gz$")
 
 
-def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None) -> bool:
+def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None, artifact_source="azure") -> bool:
     _config = utils.load_config(config)
     repodata = utils.RepoData()
 
@@ -32,7 +34,7 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
         # no PR found for the commit
         return True
     pr = prs[0]
-    artifacts = set(fetch_artifacts(pr))
+    artifacts = set(fetch_artifacts(pr, artifact_source))
     if not artifacts:
         # no artifacts found, fail and rebuild packages
         logger.info("No artifacts found.")
@@ -41,13 +43,19 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
         for artifact in artifacts:
             with tempfile.TemporaryDirectory() as tmpdir:
                 # download the artifact
-                artifact_path = os.path.join(tmpdir, os.path.basename(artifact))
-                download_artifact(artifact, artifact_path)
-                zipfile.ZipFile(artifact_path).extractall(tmpdir)
+                if artifact_source == "azure":
+                    artifact_path = os.path.join(tmpdir, os.path.basename(artifact))
+                    download_artifact(artifact, artifact_path)
+                    zipfile.ZipFile(artifact_path).extractall(tmpdir)
+                elif artifact_source == "circleci":
+                    artifact_dir = os.path.join(tmpdir, *(artifact.split("/")[-4:-1]))
+                    artifact_path = os.path.join(tmpdir, artifact_dir, os.path.basename(artifact))
+                    Path(artifact_dir).mkdir(parents=True, exist_ok=True) 
+                    download_artifact(artifact, artifact_path)
 
                 # get all the contained packages and images and upload them
                 platform_patterns = [repodata.platform2subdir(repodata.native_platform())]
-                if repodata.native_platform() == "linux":
+                if repodata.native_platform().startswith("linux"):
                     platform_patterns.append("noarch")
 
                 for platform_pattern in platform_patterns:
@@ -100,13 +108,14 @@ def download_artifact(url, to_path):
                 f.write(chunk)
 
 
-def fetch_artifacts(pr):
+def fetch_artifacts(pr, artifact_source):
     """
     Fetch artifacts from a PR.
 
     Parameters
     ----------
     pr: PR number
+    artifact_source: application hosting build artifacts (e.g., Azure or Circle CI)
 
     Returns
     -------
@@ -119,10 +128,22 @@ def fetch_artifacts(pr):
     repodata = utils.RepoData()
     platform = repodata.native_platform()
     for check_run in check_runs:
-        if check_run.name.startswith(f"bioconda.bioconda-recipes (test_{platform}"):
+        if (
+            artifact_source == "azure" and 
+            check_run.app.slug == "azure-pipelines" and
+            check_run.name.startswith(f"bioconda.bioconda-recipes (test_{platform}")
+        ):
             # azure builds
             artifact_url = get_azure_artifacts(check_run)
             yield from artifact_url
+        elif (
+            artifact_source == "circleci" and
+            check_run.app.slug == "circleci-checks"
+        ):
+            # Circle CI builds
+            artifact_url = get_circleci_artifacts(check_run, platform)
+            yield from artifact_url
+
 
 def get_azure_artifacts(check_run):
     azure_build_id = parse_azure_build_id(check_run.details_url)
@@ -138,3 +159,29 @@ def get_azure_artifacts(check_run):
 
 def parse_azure_build_id(url: str) -> str:
     return re.search("buildId=(\d+)", url).group(1)
+
+
+def get_circleci_artifacts(check_run, platform):
+    circleci_workflow_id = json.loads(check_run.external_id)["workflow-id"]
+    url_wf = f"https://circleci.com/api/v2/workflow/{circleci_workflow_id}/job"
+    res_wf = requests.get(url_wf)
+    json_wf = json.loads(res_wf.text)
+
+    if len(json_wf["items"]) == 0:
+        raise ValueError("No jobs found!")
+    else:
+        for job in json_wf["items"]:
+            if job["name"].startswith(f"build_and_test-{platform}"):
+                circleci_job_num = job["job_number"]
+                url = f"https://circleci.com/api/v2/project/gh/bioconda/bioconda-recipes/{circleci_job_num}/artifacts"
+                res = requests.get(url)
+                json_job = json.loads(res.text)
+                if len(json_job["items"]) == 0:
+                    raise ValueError("No artifacts found!")
+                else:
+                    for artifact in json_job["items"]:
+                        artifact_url = artifact["url"]
+                        if artifact_url.endswith(".html") or artifact_url.endswith(".json") or artifact_url.endswith(".json.bz2"):
+                            continue
+                        else:
+                            yield artifact_url

bioconda_utils/cli.py

@@ -517,6 +517,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
 @arg('--dryrun', action='store_true', help='''Do not actually upload anything.''')
 @arg('--fallback', choices=['build', 'ignore'], default='build', help="What to do if no artifacts are found in the PR.")
 @arg('--quay-upload-target', help="Provide a quay.io target to push docker images to.")
+@arg('--artifact-source', choices=['azure', 'circleci'], default='azure', help="Application hosting build artifacts (e.g., Azure or Circle CI).")
 @enable_logging()
 def handle_merged_pr(
     recipe_folder,
@@ -525,12 +526,13 @@ def handle_merged_pr(
     git_range=None,
     dryrun=False,
     fallback='build',
-    quay_upload_target=None
+    quay_upload_target=None,
+    artifact_source='azure'
 ):
     label = os.getenv('BIOCONDA_LABEL', None) or None
 
     success = upload_pr_artifacts(
-        config, repo, git_range[1], dryrun=dryrun, mulled_upload_target=quay_upload_target, label=label
+        config, repo, git_range[1], dryrun=dryrun, mulled_upload_target=quay_upload_target, label=label, artifact_source=artifact_source
     )
     if not success and fallback == 'build':
         success = build(

bioconda_utils/utils.py

@@ -11,6 +11,7 @@
 import glob
 import logging
 import os
+import platform
 import re
 import subprocess as sp
 import sys
@@ -1514,6 +1515,9 @@ def to_dataframe(json_data, meta_data):
 
     @staticmethod
     def native_platform():
+        arch = platform.machine()
+        if sys.platform.startswith("linux") and arch == "aarch64":
+            return "linux-aarch64"
         if sys.platform.startswith("linux"):
             return "linux"
         if sys.platform.startswith("darwin"):
@@ -1524,14 +1528,15 @@ def native_platform():
     def platform2subdir(platform):
         if platform == 'linux':
             return 'linux-64'
+        elif platform == 'linux-aarch64':
+            return 'linux-aarch64'
         elif platform == 'osx':
             return 'osx-64'
         elif platform == 'noarch':
             return 'noarch'
         else:
             raise ValueError(
-                'Unsupported platform: bioconda only supports linux, osx and noarch.')
-
+                'Unsupported platform: bioconda only supports linux, linux-aarch64, osx and noarch.')
 
 
     def get_versions(self, name):