From dfa8fc6b7201c7a86b8ddf816c10f61d0c7ca5ed Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 10 Oct 2024 10:21:42 +0200 Subject: [PATCH 1/4] fix bug with test: none --- conda_forge_tick/feedstock_parser.py | 37 ++++++++++++++++++++++++++-- conda_forge_tick/utils.py | 3 +-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 2b1d00d79..072779a48 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -27,8 +27,7 @@ from .migrators_types import PackageName, RequirementsTypedDict from conda_forge_tick.lazy_json_backends import LazyJson, dumps, loads - -from .utils import as_iterable, parse_meta_yaml, parse_recipe_yaml +from conda_forge_tick.utils import as_iterable, parse_meta_yaml, parse_recipe_yaml logger = logging.getLogger(__name__) @@ -154,9 +153,23 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None): requirements_dict[section].update( list(as_iterable(req.get(section, []) or [])), ) + test: "TestTypedDict" = block.get("test", {}) requirements_dict["test"].update(test.get("requirements", []) or []) requirements_dict["test"].update(test.get("requires", []) or []) + + if "tests" in block: + for test in block.get("tests", []): + # only script tests have requirements + if "requirements" in test: + run_reqs = test["requirements"].get("run", []) + build_reqs = test["requirements"].get("build", []) + requirements_dict["test"].update(run_reqs + build_reqs) + if "python" in test: + # if pip_check is unset or True, we need pip + if test.get("pip_check", True): + requirements_dict["test"].add("pip") + run_exports = (block.get("build", {}) or {}).get("run_exports", {}) if isinstance(run_exports, dict) and run_exports.get("strong"): strong_exports = True @@ -686,3 +699,23 @@ def load_feedstock( conda_forge_yaml=conda_forge_yaml, mark_not_archived=mark_not_archived, ) + + +if __name__ == "__main__": + graph = {} + load_feedstock_local( + "carma", + graph, + ) + + import json + import pprint + + class SetEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + + pprint.pprint(graph) + print(json.dumps(graph, indent=4, cls=SetEncoder)) diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index d03cc7132..2d920a506 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -509,8 +509,8 @@ def _parse_recipes( { "name": None if package_output is None else package_output.get("name"), "requirements": requirements_output_data, - "test": None, "build": build_output_data, + "tests": recipe.get("tests", []), } ) @@ -520,7 +520,6 @@ def _parse_recipes( "package": package_data, "requirements": requirements_data, "source": source_data, - "test": None, "outputs": output_data, "extra": first.get("extra"), } From da079d37e694f636f6c71c0c311f7549f1e09fc6 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Thu, 10 Oct 2024 08:21:10 -0500 Subject: [PATCH 2/4] Update conda_forge_tick/feedstock_parser.py --- conda_forge_tick/feedstock_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 072779a48..444597113 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -154,7 +154,7 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None): list(as_iterable(req.get(section, []) or [])), ) - test: "TestTypedDict" = block.get("test", {}) + test: "TestTypedDict" = block.get("test", {}) or {} requirements_dict["test"].update(test.get("requirements", []) or []) requirements_dict["test"].update(test.get("requires", []) or []) From d84fbee237e0d61798c16f26dcd266cdad9d2240 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 11 Oct 2024 10:40:09 +0200 Subject: [PATCH 3/4] use stub functionss --- conda_forge_tick/feedstock_parser.py | 4 ++++ conda_forge_tick/utils.py | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 072779a48..bf984b74a 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -719,3 +719,7 @@ def default(self, obj): pprint.pprint(graph) print(json.dumps(graph, indent=4, cls=SetEncoder)) + + from pathlib import Path + + Path("out.json").write_text(json.dumps(graph, indent=4, cls=SetEncoder)) diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index 2d920a506..b1f029b26 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -7,6 +7,7 @@ import logging import os import pprint +import re import subprocess import sys import tempfile @@ -349,6 +350,28 @@ def parse_recipe_yaml_local( return parsed_recipes +def replace_compiler_with_stub(text: str) -> str: + """ + Replace compiler function calls with a stub function call to match the conda-build + output. + + Parameters + ---------- + text : str + + Returns + ------- + str + """ + pattern = r'\$\{\{\s*compiler\((["\'])(.*?)\1\)\s*\}\}' + text = re.sub(pattern, lambda m: f"{m.group(2)}_compiler_stub", text) + + pattern = r'\$\{\{\s*stdlib\((["\'])(.*?)\1\)\s*\}\}' + text = re.sub(pattern, lambda m: f"{m.group(2)}_stdlib_stub", text) + + return text + + def _render_recipe_yaml( text: str, platform_arch: str | None = None, @@ -375,13 +398,16 @@ def _render_recipe_yaml( build_platform_flags = ( [] if platform_arch is None else ["--build-platform", platform_arch] ) + + prepared_text = replace_compiler_with_stub(text) + res = subprocess.run( ["rattler-build", "build", "--render-only"] + variant_config_flags + build_platform_flags, stdout=subprocess.PIPE, text=True, - input=text, + input=prepared_text, check=True, ) return [output["recipe"] for output in json.loads(res.stdout)] From 404629c415f276ba67498813e3bf734c6aadb394 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 11 Oct 2024 11:38:40 +0200 Subject: [PATCH 4/4] make debug nicer --- conda_forge_tick/feedstock_parser.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index b8cafff56..698b3dcea 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -702,24 +702,28 @@ def load_feedstock( if __name__ == "__main__": + import json + import os + import sys + + # Do not use docker when debugging + os.environ["CF_FEEDSTOCK_OPS_IN_CONTAINER"] = "true" + + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} feedstock_name") + sys.exit(1) + feedstock_name = sys.argv[1] + graph = {} load_feedstock_local( "carma", graph, ) - import json - import pprint - class SetEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, set): return list(obj) return json.JSONEncoder.default(self, obj) - pprint.pprint(graph) print(json.dumps(graph, indent=4, cls=SetEncoder)) - - from pathlib import Path - - Path("out.json").write_text(json.dumps(graph, indent=4, cls=SetEncoder))