diff --git a/bioconda_utils/artifacts.py b/bioconda_utils/artifacts.py index e1594cb369..eef49f5dfc 100644 --- a/bioconda_utils/artifacts.py +++ b/bioconda_utils/artifacts.py @@ -9,6 +9,8 @@ import requests import backoff +import json +from pathlib import Path from bioconda_utils import utils from bioconda_utils.upload import anaconda_upload, skopeo_upload @@ -18,7 +20,7 @@ IMAGE_RE = re.compile(r"(.+)(?::|%3A)(.+)\.tar\.gz$") -def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None) -> bool: +def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None, artifact_source="azure") -> bool: _config = utils.load_config(config) repodata = utils.RepoData() @@ -32,7 +34,7 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe # no PR found for the commit return True pr = prs[0] - artifacts = set(fetch_artifacts(pr)) + artifacts = set(fetch_artifacts(pr, artifact_source)) if not artifacts: # no artifacts found, fail and rebuild packages logger.info("No artifacts found.") @@ -41,13 +43,19 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe for artifact in artifacts: with tempfile.TemporaryDirectory() as tmpdir: # download the artifact - artifact_path = os.path.join(tmpdir, os.path.basename(artifact)) - download_artifact(artifact, artifact_path) - zipfile.ZipFile(artifact_path).extractall(tmpdir) + if artifact_source == "azure": + artifact_path = os.path.join(tmpdir, os.path.basename(artifact)) + download_artifact(artifact, artifact_path) + zipfile.ZipFile(artifact_path).extractall(tmpdir) + elif artifact_source == "circleci": + artifact_dir = os.path.join(tmpdir, *(artifact.split("/")[-4:-1])) + artifact_path = os.path.join(tmpdir, artifact_dir, os.path.basename(artifact)) + Path(artifact_dir).mkdir(parents=True, exist_ok=True) + download_artifact(artifact, artifact_path) # get all the contained packages and images and upload them platform_patterns = [repodata.platform2subdir(repodata.native_platform())] - if repodata.native_platform() == "linux": + if repodata.native_platform().startswith("linux"): platform_patterns.append("noarch") for platform_pattern in platform_patterns: @@ -100,13 +108,14 @@ def download_artifact(url, to_path): f.write(chunk) -def fetch_artifacts(pr): +def fetch_artifacts(pr, artifact_source): """ Fetch artifacts from a PR. Parameters ---------- pr: PR number + artifact_source: application hosting build artifacts (e.g., Azure or Circle CI) Returns ------- @@ -119,10 +128,22 @@ def fetch_artifacts(pr): repodata = utils.RepoData() platform = repodata.native_platform() for check_run in check_runs: - if check_run.name.startswith(f"bioconda.bioconda-recipes (test_{platform}"): + if ( + artifact_source == "azure" and + check_run.app.slug == "azure-pipelines" and + check_run.name.startswith(f"bioconda.bioconda-recipes (test_{platform}") + ): # azure builds artifact_url = get_azure_artifacts(check_run) yield from artifact_url + elif ( + artifact_source == "circleci" and + check_run.app.slug == "circleci-checks" + ): + # Circle CI builds + artifact_url = get_circleci_artifacts(check_run, platform) + yield from artifact_url + def get_azure_artifacts(check_run): azure_build_id = parse_azure_build_id(check_run.details_url) @@ -138,3 +159,29 @@ def get_azure_artifacts(check_run): def parse_azure_build_id(url: str) -> str: return re.search("buildId=(\d+)", url).group(1) + + +def get_circleci_artifacts(check_run, platform): + circleci_workflow_id = json.loads(check_run.external_id)["workflow-id"] + url_wf = f"https://circleci.com/api/v2/workflow/{circleci_workflow_id}/job" + res_wf = requests.get(url_wf) + json_wf = json.loads(res_wf.text) + + if len(json_wf["items"]) == 0: + raise ValueError("No jobs found!") + else: + for job in json_wf["items"]: + if job["name"].startswith(f"build_and_test-{platform}"): + circleci_job_num = job["job_number"] + url = f"https://circleci.com/api/v2/project/gh/bioconda/bioconda-recipes/{circleci_job_num}/artifacts" + res = requests.get(url) + json_job = json.loads(res.text) + if len(json_job["items"]) == 0: + raise ValueError("No artifacts found!") + else: + for artifact in json_job["items"]: + artifact_url = artifact["url"] + if artifact_url.endswith(".html") or artifact_url.endswith(".json") or artifact_url.endswith(".json.bz2"): + continue + else: + yield artifact_url diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 35b55b938b..24e3ab877d 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -517,6 +517,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, @arg('--dryrun', action='store_true', help='''Do not actually upload anything.''') @arg('--fallback', choices=['build', 'ignore'], default='build', help="What to do if no artifacts are found in the PR.") @arg('--quay-upload-target', help="Provide a quay.io target to push docker images to.") +@arg('--artifact-source', choices=['azure', 'circleci'], default='azure', help="Application hosting build artifacts (e.g., Azure or Circle CI).") @enable_logging() def handle_merged_pr( recipe_folder, @@ -525,12 +526,13 @@ def handle_merged_pr( git_range=None, dryrun=False, fallback='build', - quay_upload_target=None + quay_upload_target=None, + artifact_source='azure' ): label = os.getenv('BIOCONDA_LABEL', None) or None success = upload_pr_artifacts( - config, repo, git_range[1], dryrun=dryrun, mulled_upload_target=quay_upload_target, label=label + config, repo, git_range[1], dryrun=dryrun, mulled_upload_target=quay_upload_target, label=label, artifact_source=artifact_source ) if not success and fallback == 'build': success = build( diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 24b8acf8be..0e8539bee9 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -11,6 +11,7 @@ import glob import logging import os +import platform import re import subprocess as sp import sys @@ -1514,6 +1515,9 @@ def to_dataframe(json_data, meta_data): @staticmethod def native_platform(): + arch = platform.machine() + if sys.platform.startswith("linux") and arch == "aarch64": + return "linux-aarch64" if sys.platform.startswith("linux"): return "linux" if sys.platform.startswith("darwin"): @@ -1524,14 +1528,15 @@ def native_platform(): def platform2subdir(platform): if platform == 'linux': return 'linux-64' + elif platform == 'linux-aarch64': + return 'linux-aarch64' elif platform == 'osx': return 'osx-64' elif platform == 'noarch': return 'noarch' else: raise ValueError( - 'Unsupported platform: bioconda only supports linux, osx and noarch.') - + 'Unsupported platform: bioconda only supports linux, linux-aarch64, osx and noarch.') def get_versions(self, name):