diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index e1da77301..c9b968e5d 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -4,16 +4,12 @@ import logging import os import random +import textwrap import time import traceback import typing -from subprocess import CalledProcessError -from textwrap import dedent -from typing import MutableMapping, Tuple, cast - -if typing.TYPE_CHECKING: - from .migrators_types import MigrationUidTypedDict - +from dataclasses import dataclass +from typing import Literal, cast from urllib.error import URLError from uuid import uuid4 @@ -24,16 +20,22 @@ from conda.models.version import VersionOrder from conda_forge_tick.cli_context import CliContext -from conda_forge_tick.contexts import FeedstockContext, MigratorSessionContext +from conda_forge_tick.contexts import ( + ClonedFeedstockContext, + FeedstockContext, + MigratorSessionContext, +) from conda_forge_tick.deploy import deploy from conda_forge_tick.feedstock_parser import BOOTSTRAP_MAPPINGS from conda_forge_tick.git_utils import ( - GIT_CLONE_DIR, - comment_on_pr, - get_repo, + DryRunBackend, + DuplicatePullRequestError, + GitCli, + GitCliError, + GitPlatformBackend, + RepositoryNotFoundError, github_backend, is_github_api_limit_reached, - push_repo, ) from conda_forge_tick.lazy_json_backends import ( LazyJson, @@ -49,7 +51,7 @@ from conda_forge_tick.migration_runner import run_migration from conda_forge_tick.migrators import MigrationYaml, Migrator, Version from conda_forge_tick.migrators.version import VersionMigrationError -from conda_forge_tick.os_utils import eval_cmd, pushd +from conda_forge_tick.os_utils import eval_cmd from conda_forge_tick.rerender_feedstock import rerender_feedstock from conda_forge_tick.solver_checks import is_recipe_solvable from conda_forge_tick.utils import ( @@ -66,6 +68,10 @@ sanitize_string, ) +from .migrators_types import MigrationUidTypedDict +from .models.pr_info import PullRequestInfoSpecial +from .models.pr_json import PullRequestData, PullRequestState + logger = logging.getLogger(__name__) BOT_HOME_DIR: str = os.getcwd() @@ -139,35 +145,346 @@ def _get_pre_pr_migrator_attempts(attrs, migrator_name, *, is_version): return pri.get("pre_pr_migrator_attempts", {}).get(migrator_name, 0) +def _prepare_feedstock_repository( + backend: GitPlatformBackend, + context: ClonedFeedstockContext, + branch: str, + base_branch: str, +) -> bool: + """ + Prepare a feedstock repository for migration by forking and cloning it. The local clone will be present in + context.local_clone_dir. + + Any errors are written to the pr_info attribute of the feedstock context and logged. + + :param backend: The GitPlatformBackend instance to use. + :param context: The FeedstockContext instance. + :param branch: The branch to create in the forked repository. + :param base_branch: The base branch to branch from. + :return: True if the repository was successfully prepared, False otherwise. + """ + try: + backend.fork(context.git_repo_owner, context.git_repo_name) + except RepositoryNotFoundError: + logger.warning( + f"Could not fork {context.git_repo_owner}/{context.git_repo_name}: Not Found" + ) + + error_message = f"{context.feedstock_name}: Git repository not found." + logger.critical( + f"Failed to migrate {context.feedstock_name}, {error_message}", + ) + + with context.attrs["pr_info"] as pri: + pri["bad"] = error_message + + return False + + backend.clone_fork_and_branch( + upstream_owner=context.git_repo_owner, + repo_name=context.git_repo_name, + target_dir=context.local_clone_dir, + new_branch=branch, + base_branch=base_branch, + ) + return True + + +def _commit_migration( + cli: GitCli, + context: ClonedFeedstockContext, + commit_message: str, + allow_empty_commits: bool = False, + raise_commit_errors: bool = True, +) -> None: + """ + Commit a migration that has been run in the local clone of a feedstock repository. + If an error occurs during the commit, it is logged. + + :param cli: The GitCli instance to use. + :param context: The FeedstockContext instance. + :param commit_message: The commit message to use. + :param allow_empty_commits: Whether the migrator allows empty commits. + :param raise_commit_errors: Whether to raise an exception if an error occurs during the commit. + + :raises GitCliError: If an error occurs during the commit and raise_commit_errors is True. + """ + cli.add( + context.local_clone_dir, + all_=True, + ) + + try: + cli.commit( + context.local_clone_dir, commit_message, allow_empty=allow_empty_commits + ) + except GitCliError as e: + logger.info("could not commit to feedstock - likely no changes", exc_info=e) + + if raise_commit_errors: + raise + + +@dataclass(frozen=True) +class _RerenderInfo: + """ + Additional information about a rerender operation. + """ + + nontrivial_migration_yaml_changes: bool + """ + True if any files which are not in the following list were changed during the rerender, False otherwise: + 1. anything in the recipe directory + 2. anything in the migrators directory + 3. the README file + + This is useful to discard MigrationYaml migrations that only drop a file in the migrations directory. + """ + rerender_comment: str | None = None + """ + If requested, a comment to be added to the PR to indicate an issue with the rerender. + None if no comment should be added. + """ + + +def _run_rerender( + git_cli: GitCli, context: ClonedFeedstockContext, suppress_errors: bool = False +) -> _RerenderInfo: + logger.info("Rerendering the feedstock") + + try: + rerender_msg = rerender_feedstock(str(context.local_clone_dir), timeout=900) + except Exception as e: + logger.error("RERENDER ERROR", exc_info=e) + + if not suppress_errors: + raise + + rerender_comment = textwrap.dedent( + """ + Hi! This feedstock was not able to be rerendered after the version update changes. I + have pushed the version update changes anyways and am trying to rerender again with this + comment. Hopefully you all can fix this! + + @conda-forge-admin rerender + """ + ) + + return _RerenderInfo( + nontrivial_migration_yaml_changes=False, rerender_comment=rerender_comment + ) + + if rerender_msg is None: + return _RerenderInfo(nontrivial_migration_yaml_changes=False) + + git_cli.commit(context.local_clone_dir, rerender_msg, all_=True, allow_empty=True) + + # HEAD~ is the state before the last commit + changed_files = git_cli.diffed_files(context.local_clone_dir, "HEAD~") + + recipe_dir = context.local_clone_dir / "recipe" + + nontrivial_migration_yaml_changes = any( + not file.is_relative_to(recipe_dir) and not file.name.startswith("README") + for file in changed_files + ) + + return _RerenderInfo(nontrivial_migration_yaml_changes) + + +def _has_automerge(migrator: Migrator, context: FeedstockContext) -> bool: + """ + Determine if a migration should be auto merged based on the feedstock and migrator settings. + + :param migrator: The migrator to check. + :param context: The feedstock context. + + :return: True if the migrator should be auto merged, False otherwise. + """ + if isinstance(migrator, Version): + return context.automerge in [True, "version"] + else: + return getattr(migrator, "automerge", False) and context.automerge in [ + True, + "migration", + ] + + +def _is_solvability_check_needed( + migrator: Migrator, context: FeedstockContext, base_branch: str +) -> bool: + migrator_check_solvable = getattr(migrator, "check_solvable", True) + pr_attempts = _get_pre_pr_migrator_attempts( + context.attrs, + migrator_name=get_migrator_name(migrator), + is_version=isinstance(migrator, Version), + ) + max_pr_attempts = getattr( + migrator, "force_pr_after_solver_attempts", MAX_SOLVER_ATTEMPTS * 2 + ) + + logger.info( + textwrap.dedent( + f""" + automerge and check_solvable status/settings: + automerge: + feedstock_automerge: {context.automerge} + migrator_automerge: {getattr(migrator, 'automerge', False)} + has_automerge: {_has_automerge(migrator, context)} (only considers feedstock if version migration) + check_solvable: + feedstock_check_solvable: {context.check_solvable} + migrator_check_solvable: {migrator_check_solvable} + pre_pr_migrator_attempts: {pr_attempts} + force_pr_after_solver_attempts: {max_pr_attempts} + """ + ) + ) + + return ( + context.feedstock_name != "conda-forge-pinning" + and (base_branch == "master" or base_branch == "main") + # feedstocks that have problematic bootstrapping will not always be solvable + and context.feedstock_name not in BOOTSTRAP_MAPPINGS + # stuff in cycles always goes + and context.attrs["name"] not in getattr(migrator, "cycles", set()) + # stuff at the top always goes + and context.attrs["name"] not in getattr(migrator, "top_level", set()) + # either the migrator or the feedstock has to request solver checks + and (migrator_check_solvable or context.check_solvable) + # we try up to MAX_SOLVER_ATTEMPTS times, and then we just skip + # the solver check and issue the PR if automerge is off + and (_has_automerge(migrator, context) or (pr_attempts < max_pr_attempts)) + ) + + +def _handle_solvability_error( + errors: list[str], context: FeedstockContext, migrator: Migrator, base_branch: str +) -> None: + ci_url = get_bot_run_url() + ci_url = f"(bot CI job)" if ci_url else "" + _solver_err_str = textwrap.dedent( + f""" + not solvable {ci_url} @ {base_branch} +
+
+
+        {'
'.join(sorted(set(errors)))}
+        
+
+
+ """, + ).strip() + + _set_pre_pr_migrator_error( + context.attrs, + get_migrator_name(migrator), + _solver_err_str, + is_version=isinstance(migrator, Version), + ) + + # remove part of a try for solver errors to make those slightly + # higher priority next time the bot runs + if isinstance(migrator, Version): + with context.attrs["version_pr_info"] as vpri: + _new_ver = vpri["new_version"] + vpri["new_version_attempts"][_new_ver] -= 0.8 + + +def _check_and_process_solvability( + migrator: Migrator, context: ClonedFeedstockContext, base_branch: str +) -> bool: + """ + If the migration needs a solvability check, perform the check. If the recipe is not solvable, handle the error + by setting the corresponding fields in the feedstock attributes. + If the recipe is solvable, reset the fields that track the solvability check status. + + :param migrator: The migrator that was run + :param context: The current FeedstockContext of the feedstock that was migrated + :param base_branch: The branch of the feedstock repository that is the migration target + + :returns: True if the migration can proceed normally, False if a required solvability check failed and the migration + needs to be aborted + """ + if not _is_solvability_check_needed(migrator, context, base_branch): + return True + + solvable, solvability_errors, _ = is_recipe_solvable( + str(context.local_clone_dir), + build_platform=context.attrs["conda-forge.yml"].get( + "build_platform", + None, + ), + ) + if solvable: + _reset_pre_pr_migrator_fields( + context.attrs, + get_migrator_name(migrator), + is_version=isinstance(migrator, Version), + ) + return True + + _handle_solvability_error(solvability_errors, context, migrator, base_branch) + return False + + +def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: + return PullRequestInfoSpecial( + id=str(uuid4()), + merged_at="never issued", + state="closed", + ) + + +def run_with_tmpdir( + context: FeedstockContext, + migrator: Migrator, + git_backend: GitPlatformBackend, + rerender: bool = True, + base_branch: str = "main", + **kwargs: typing.Any, +) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: + """ + For a given feedstock and migration run the migration in a temporary directory that will be deleted after the + migration is complete. + + The parameters are the same as for the `run` function. The only difference is that you pass a FeedstockContext + instance instead of a ClonedFeedstockContext instance. + + The exceptions are the same as for the `run` function. + """ + with context.reserve_clone_directory() as cloned_context: + return run( + context=cloned_context, + migrator=migrator, + git_backend=git_backend, + rerender=rerender, + base_branch=base_branch, + **kwargs, + ) + + def run( - feedstock_ctx: FeedstockContext, + context: ClonedFeedstockContext, migrator: Migrator, - protocol: str = "ssh", - pull_request: bool = True, + git_backend: GitPlatformBackend, rerender: bool = True, - fork: bool = True, base_branch: str = "main", - dry_run: bool = False, **kwargs: typing.Any, -) -> Tuple["MigrationUidTypedDict", dict]: +) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration Parameters ---------- - feedstock_ctx: FeedstockContext - The node attributes + context: ClonedFeedstockContext + The current feedstock context, already containing information about a temporary directory for the feedstock. migrator: Migrator instance The migrator to run on the feedstock - protocol : str, optional - The git protocol to use, defaults to ``ssh`` - pull_request : bool, optional - If true issue pull request, defaults to true + git_backend: GitPlatformBackend + The git backend to use. Use the DryRunBackend for testing. rerender : bool Whether to rerender - fork : bool - If true create a fork, defaults to true base_branch : str, optional - The base branch to which the PR will be targeted. Defaults to "main". + The base branch to which the PR will be targeted. kwargs: dict The keyword arguments to pass to the migrator. @@ -177,301 +494,138 @@ def run( The migration return dict used for tracking finished migrations pr_json: dict The PR json object for recreating the PR as needed - """ + Exceptions + ---------- + GitCliError + If an error occurs during a git command which is not suppressed + """ # sometimes we get weird directory issues so make sure we reset os.chdir(BOT_HOME_DIR) - # get the repo - branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[0:6] - migrator_name = get_migrator_name(migrator) is_version_migration = isinstance(migrator, Version) _increment_pre_pr_migrator_attempt( - feedstock_ctx.attrs, + context.attrs, migrator_name, is_version=is_version_migration, ) - # TODO: run this in parallel - feedstock_dir, repo = get_repo( - fctx=feedstock_ctx, branch=branch_name, base_branch=base_branch - ) - if not feedstock_dir or not repo: - logger.critical( - "Failed to migrate %s, %s", - feedstock_ctx.feedstock_name, - feedstock_ctx.attrs.get("pr_info", {}).get("bad"), - ) + branch_name = migrator.remote_branch(context) + "_h" + uuid4().hex[0:6] + if not _prepare_feedstock_repository( + git_backend, context, branch_name, base_branch + ): + # something went wrong during forking or cloning return False, False - # need to use an absolute path here - feedstock_dir = os.path.abspath(feedstock_dir) - + # feedstock_dir must be absolute migration_run_data = run_migration( migrator=migrator, - feedstock_dir=feedstock_dir, - feedstock_name=feedstock_ctx.feedstock_name, - node_attrs=feedstock_ctx.attrs, - default_branch=feedstock_ctx.default_branch, + feedstock_dir=str(context.local_clone_dir.resolve()), + feedstock_name=context.feedstock_name, + node_attrs=context.attrs, + default_branch=context.default_branch, **kwargs, ) if not migration_run_data["migrate_return_value"]: logger.critical( - "Failed to migrate %s, %s", - feedstock_ctx.feedstock_name, - feedstock_ctx.attrs.get("pr_info", {}).get("bad"), + f"Failed to migrate {context.feedstock_name}, {context.attrs.get('pr_info', {}).get('bad')}", ) - eval_cmd(["rm", "-rf", feedstock_dir]) return False, False - # rerender, maybe - diffed_files: typing.List[str] = [] - with pushd(feedstock_dir): - msg = migration_run_data["commit_message"] - try: - eval_cmd(["git", "add", "--all", "."]) - if migrator.allow_empty_commits: - eval_cmd(["git", "commit", "--allow-empty", "-am", msg]) - else: - eval_cmd(["git", "commit", "-am", msg]) - except CalledProcessError as e: - logger.info( - "could not commit to feedstock - " - "likely no changes - error is '%s'" % (repr(e)), - ) - # we bail here if we do not plan to rerender and we wanted an empty - # commit - # this prevents PRs that don't actually get made from getting marked as done - if migrator.allow_empty_commits and not rerender: - raise e - - if rerender: - head_ref = eval_cmd(["git", "rev-parse", "HEAD"]).strip() - logger.info("Rerendering the feedstock") - - try: - rerender_msg = rerender_feedstock(feedstock_dir, timeout=900) - if rerender_msg is not None: - eval_cmd(["git", "commit", "--allow-empty", "-am", rerender_msg]) - - make_rerender_comment = False - except Exception as e: - # I am trying this bit of code to force these errors - # to be surfaced in the logs at the right time. - print(f"RERENDER ERROR: {e}", flush=True) - if not isinstance(migrator, Version): - raise - else: - # for check solvable or automerge, we always raise rerender errors - if get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "check_solvable"], - {}, - False, - ) or get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "automerge"], - {}, - False, - ): - raise - else: - make_rerender_comment = True - - # If we tried to run the MigrationYaml and rerender did nothing (we only - # bumped the build number and dropped a yaml file in migrations) bail - # for instance platform specific migrations - gdiff = eval_cmd( - ["git", "diff", "--name-only", f"{head_ref.strip()}...HEAD"] - ) - - diffed_files = [ - _ - for _ in gdiff.split() - if not ( - _.startswith("recipe") - or _.startswith("migrators") - or _.startswith("README") - ) - ] - else: - make_rerender_comment = False - - feedstock_automerge = get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "automerge"], - {}, - False, - ) - if isinstance(migrator, Version): - has_automerge = feedstock_automerge in [True, "version"] - else: - has_automerge = getattr( - migrator, "automerge", False - ) and feedstock_automerge in [True, "migration"] - - migrator_check_solvable = getattr(migrator, "check_solvable", True) - feedstock_check_solvable = get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "check_solvable"], - {}, - False, - ) - pr_attempts = _get_pre_pr_migrator_attempts( - feedstock_ctx.attrs, - migrator_name, - is_version=is_version_migration, - ) - max_pr_attempts = getattr( - migrator, "force_pr_after_solver_attempts", MAX_SOLVER_ATTEMPTS * 2 - ) - - logger.info( - f"""automerge and check_solvable status/settings: - automerge: - feedstock_automerge: {feedstock_automerge} - migratror_automerge: {getattr(migrator, 'automerge', False)} - has_automerge: {has_automerge} (only considers feedstock if version migration) - check_solvable: - feedstock_checksolvable: {feedstock_check_solvable} - migrator_check_solvable: {migrator_check_solvable} - pre_pr_migrator_attempts: {pr_attempts} - force_pr_after_solver_attempts: {max_pr_attempts} -""" + # We raise an exception if we don't plan to rerender and wanted an empty commit. + # This prevents PRs that don't actually get made from getting marked as done. + _commit_migration( + cli=git_backend.cli, + context=context, + commit_message=migration_run_data["commit_message"], + allow_empty_commits=migrator.allow_empty_commits, + raise_commit_errors=migrator.allow_empty_commits and not rerender, ) - if ( - feedstock_ctx.feedstock_name != "conda-forge-pinning" - and (base_branch == "master" or base_branch == "main") - # feedstocks that have problematic bootstrapping will not always be solvable - and feedstock_ctx.feedstock_name not in BOOTSTRAP_MAPPINGS - # stuff in cycles always goes - and feedstock_ctx.attrs["name"] not in getattr(migrator, "cycles", set()) - # stuff at the top always goes - and feedstock_ctx.attrs["name"] not in getattr(migrator, "top_level", set()) - # either the migrator or the feedstock has to request solver checks - and (migrator_check_solvable or feedstock_check_solvable) - # we try up to MAX_SOLVER_ATTEMPTS times and then we just skip - # the solver check and issue the PR if automerge is off - and (has_automerge or (pr_attempts < max_pr_attempts)) - ): - solvable, errors, _ = is_recipe_solvable( - feedstock_dir, - build_platform=feedstock_ctx.attrs["conda-forge.yml"].get( - "build_platform", - None, - ), + if rerender: + # for version migrations, check solvable or automerge, we always raise rerender errors + suppress_errors = ( + not is_version_migration + and not context.check_solvable + and not context.automerge ) - if not solvable: - ci_url = get_bot_run_url() - ci_url = f"(bot CI job)" if ci_url else "" - _solver_err_str = dedent( - f""" - not solvable {ci_url} @ {base_branch} -
-
-
-                {'
'.join(sorted(set(errors)))}
-                
-
-
- """, - ).strip() - - _set_pre_pr_migrator_error( - feedstock_ctx.attrs, - migrator_name, - _solver_err_str, - is_version=is_version_migration, - ) - # remove part of a try for solver errors to make those slightly - # higher priority next time the bot runs - if isinstance(migrator, Version): - with feedstock_ctx.attrs["version_pr_info"] as vpri: - _new_ver = vpri["new_version"] - vpri["new_version_attempts"][_new_ver] -= 0.8 + rerender_info = _run_rerender(git_backend.cli, context, suppress_errors) + else: + rerender_info = _RerenderInfo(nontrivial_migration_yaml_changes=False) - eval_cmd(["rm", "-rf", feedstock_dir]) - return False, False - else: - _reset_pre_pr_migrator_fields( - feedstock_ctx.attrs, migrator_name, is_version=is_version_migration - ) + if not _check_and_process_solvability(migrator, context, base_branch): + logger.warning("Skipping migration due to solvability check failure") + return False, False - # TODO: Better annotation here - pr_json: typing.Union[MutableMapping, None, bool] + pr_data: PullRequestData | PullRequestInfoSpecial | None = None + """ + The PR data for the PR that was created. The contents of this variable will be stored in the bot's database. + None means: We don't update the PR data. + """ if ( isinstance(migrator, MigrationYaml) - and not diffed_files - and feedstock_ctx.attrs["name"] != "conda-forge-pinning" + and not rerender_info.nontrivial_migration_yaml_changes + and context.attrs["name"] != "conda-forge-pinning" ): # spoof this so it looks like the package is done - pr_json = { - "state": "closed", - "merged_at": "never issued", - "id": str(uuid4()), - } + pr_data = get_spoofed_closed_pr_info() else: - # push up + # push and PR + git_backend.push_to_repository( + owner=git_backend.user, + repo_name=context.git_repo_name, + git_dir=context.local_clone_dir, + branch=branch_name, + ) try: - # TODO: remove this hack, but for now this is the only way to get - # the feedstock dir into pr_body - feedstock_ctx.feedstock_dir = feedstock_dir - pr_json = push_repo( - fctx=feedstock_ctx, - feedstock_dir=feedstock_dir, - body=migration_run_data["pr_body"], - repo=repo, - title=migration_run_data["pr_title"], - branch=branch_name, + pr_data = git_backend.create_pull_request( + target_owner=context.git_repo_owner, + target_repo=context.git_repo_name, base_branch=base_branch, - dry_run=dry_run, + head_branch=branch_name, + title=migration_run_data["pr_title"], + body=migration_run_data["pr_body"], + ) + except DuplicatePullRequestError: + # This shouldn't happen too often anymore since we won't double PR + logger.warning( + f"Attempted to create a duplicate PR for merging {git_backend.user}:{branch_name} " + f"into {context.git_repo_owner}:{base_branch}. Ignoring." ) + # Don't update the PR data (keep pr_data as None) - # This shouldn't happen too often any more since we won't double PR - except github3.GitHubError as e: - if e.msg != "Validation Failed": - raise - else: - print(f"Error during push {e}") - # If we just push to the existing PR then do nothing to the json - pr_json = False - ljpr = False - - if pr_json and pr_json["state"] != "closed" and make_rerender_comment: - comment_on_pr( - pr_json, - """\ -Hi! This feedstock was not able to be rerendered after the version update changes. I -have pushed the version update changes anyways and am trying to rerender again with this -comment. Hopefully you all can fix this! - -@conda-forge-admin rerender""", - repo, + if ( + pr_data + and pr_data.state != PullRequestState.CLOSED + and rerender_info.rerender_comment + ): + git_backend.comment_on_pull_request( + repo_owner=context.git_repo_owner, + repo_name=context.git_repo_name, + pr_number=pr_data.number, + comment=rerender_info.rerender_comment, ) - if pr_json: - ljpr = LazyJson( - os.path.join("pr_json", str(pr_json["id"]) + ".json"), + if pr_data: + pr_lazy_json = LazyJson( + os.path.join("pr_json", f"{pr_data.id}.json"), ) - with ljpr as __ljpr: - __ljpr.update(**pr_json) + with pr_lazy_json as __edit_pr_lazy_json: + __edit_pr_lazy_json.update(**pr_data.model_dump(mode="json")) else: - ljpr = False + pr_lazy_json = False # If we've gotten this far then the node is good - with feedstock_ctx.attrs["pr_info"] as pri: + with context.attrs["pr_info"] as pri: pri["bad"] = False _reset_pre_pr_migrator_fields( - feedstock_ctx.attrs, migrator_name, is_version=is_version_migration + context.attrs, migrator_name, is_version=is_version_migration ) - logger.info("Removing feedstock dir") - eval_cmd(["rm", "-rf", feedstock_dir]) - return migration_run_data["migrate_return_value"], ljpr + return migration_run_data["migrate_return_value"], pr_lazy_json def _compute_time_per_migrator(mctx, migrators): @@ -552,8 +706,8 @@ def _run_migrator_on_feedstock_branch( attrs, base_branch, migrator, - fctx, - dry_run, + fctx: FeedstockContext, + git_backend: GitPlatformBackend, mctx, migrator_name, good_prs, @@ -564,14 +718,13 @@ def _run_migrator_on_feedstock_branch( fctx.attrs["new_version"] = attrs.get("version_pr_info", {}).get( "new_version", None ) - migrator_uid, pr_json = run( - feedstock_ctx=fctx, + migrator_uid, pr_json = run_with_tmpdir( + context=fctx, migrator=migrator, + git_backend=git_backend, rerender=migrator.rerender, - protocol="https", - hash_type=attrs.get("hash_type", "sha256"), base_branch=base_branch, - dry_run=dry_run, + hash_type=attrs.get("hash_type", "sha256"), ) finally: fctx.attrs.pop("new_version", None) @@ -604,6 +757,7 @@ def _run_migrator_on_feedstock_branch( ) except (github3.GitHubError, github.GithubException) as e: + # TODO: pull this down into run() - also check the other exceptions if hasattr(e, "msg") and e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: @@ -730,7 +884,7 @@ def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): return False -def _run_migrator(migrator, mctx, temp, time_per, dry_run): +def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBackend): _mg_start = time.time() migrator_name = get_migrator_name(migrator) @@ -848,14 +1002,14 @@ def _run_migrator(migrator, mctx, temp, time_per, dry_run): ) ): good_prs, break_loop = _run_migrator_on_feedstock_branch( - attrs, - base_branch, - migrator, - fctx, - dry_run, - mctx, - migrator_name, - good_prs, + attrs=attrs, + base_branch=base_branch, + migrator=migrator, + fctx=fctx, + git_backend=git_backend, + mctx=mctx, + migrator_name=migrator_name, + good_prs=good_prs, ) if break_loop: break @@ -871,11 +1025,9 @@ def _run_migrator(migrator, mctx, temp, time_per, dry_run): os.chdir(BOT_HOME_DIR) # Write graph partially through - if not dry_run: - dump_graph(mctx.graph) + dump_graph(mctx.graph) with filter_reprinted_lines("rm-tmp"): - eval_cmd(["rm", "-rf", f"{GIT_CLONE_DIR}/*"]) for f in glob.glob("/tmp/*"): if f not in temp: try: @@ -1106,13 +1258,15 @@ def main(ctx: CliContext) -> None: flush=True, ) + git_backend = github_backend() if not ctx.dry_run else DryRunBackend() + for mg_ind, migrator in enumerate(migrators): good_prs = _run_migrator( migrator, mctx, temp, time_per_migrator[mg_ind], - ctx.dry_run, + git_backend, ) if good_prs > 0: pass diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index 5cc545831..873d3f18e 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -1,10 +1,17 @@ +from __future__ import annotations + import os +import tempfile import typing +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass +from pathlib import Path from networkx import DiGraph from conda_forge_tick.lazy_json_backends import load +from conda_forge_tick.utils import get_keys_default if typing.TYPE_CHECKING: from conda_forge_tick.migrators_types import AttrsTypedDict @@ -27,10 +34,10 @@ class MigratorSessionContext: dry_run: bool = True -@dataclass +@dataclass(frozen=True) class FeedstockContext: feedstock_name: str - attrs: "AttrsTypedDict" + attrs: AttrsTypedDict _default_branch: str = None @property @@ -40,6 +47,73 @@ def default_branch(self): else: return self._default_branch - @default_branch.setter - def default_branch(self, v): - self._default_branch = v + @property + def git_repo_owner(self) -> str: + return "conda-forge" + + @property + def git_repo_name(self) -> str: + return f"{self.feedstock_name}-feedstock" + + @property + def git_href(self) -> str: + """ + A link to the feedstocks GitHub repository. + """ + return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}" + + @property + def automerge(self) -> bool | str: + """ + Get the automerge setting of the feedstock. + + Note: A better solution to implement this is to use the NodeAttributes Pydantic + model for the attrs field. This will be done in the future. + """ + return get_keys_default( + self.attrs, + ["conda-forge.yml", "bot", "automerge"], + {}, + False, + ) + + @property + def check_solvable(self) -> bool: + """ + Get the check_solvable setting of the feedstock. + + Note: A better solution to implement this is to use the NodeAttributes Pydantic + model for the attrs field. This will be done in the future. + """ + return get_keys_default( + self.attrs, + ["conda-forge.yml", "bot", "check_solvable"], + {}, + False, + ) + + @contextmanager + def reserve_clone_directory(self) -> Iterator[ClonedFeedstockContext]: + """ + Reserve a temporary directory for the feedstock repository that will be available within the context manager. + The returned context object will contain the path to the feedstock repository in local_clone_dir. + After the context manager exits, the temporary directory will be deleted. + """ + with tempfile.TemporaryDirectory() as tmpdir: + local_clone_dir = Path(tmpdir) / self.git_repo_name + local_clone_dir.mkdir() + yield ClonedFeedstockContext( + **self.__dict__, + local_clone_dir=local_clone_dir, + ) + + +@dataclass(frozen=True, kw_only=True) +class ClonedFeedstockContext(FeedstockContext): + """ + A FeedstockContext object that has reserved a temporary directory for the feedstock repository. + """ + + # Implementation Note: Keep this class frozen or there will be consistency issues if someone modifies + # a ClonedFeedstockContext object in place - it will not be reflected in the original FeedstockContext object. + local_clone_dir: Path diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 17fe6a086..14a7e3a8a 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -5,13 +5,16 @@ import logging import math import subprocess +import sys +import textwrap import threading import time from abc import ABC, abstractmethod from datetime import datetime +from email import utils from functools import cached_property from pathlib import Path -from typing import Dict, Literal, Optional, Tuple, Union +from typing import Dict, Iterator, Optional, Union import backoff import github @@ -20,7 +23,9 @@ import github3.pulls import github3.repos import requests +from github3.session import GitHubSession from requests.exceptions import RequestException, Timeout +from requests.structures import CaseInsensitiveDict from conda_forge_tick import sensitive_env @@ -28,10 +33,16 @@ # and pull all the needed info from the various source classes) from conda_forge_tick.lazy_json_backends import LazyJson -from .contexts import FeedstockContext from .executors import lock_git_operation -from .os_utils import pushd -from .utils import get_bot_run_url, run_command_hiding_token +from .models.pr_json import ( + GithubPullRequestBase, + GithubPullRequestMergeableState, + GithubRepository, + PullRequestData, + PullRequestInfoHead, + PullRequestState, +) +from .utils import get_bot_run_url, replace_tokens, run_command_hiding_token logger = logging.getLogger(__name__) @@ -42,8 +53,6 @@ MAX_GITHUB_TIMEOUT = 60 -GIT_CLONE_DIR = "./feedstocks/" - BOT_RERUN_LABEL = { "name": "bot-rerun", } @@ -52,7 +61,7 @@ # these keys are kept from github PR json blobs # to add more keys to keep, put them in the right spot in the dict and -# set them to None. Also add them to the PullRequestInfo Pydantic model! +# set them to None. Also add them to the PullRequestData Pydantic model! PR_KEYS_TO_KEEP = { "ETag": None, "Last-Modified": None, @@ -74,13 +83,17 @@ } +def get_bot_token(): + with sensitive_env() as env: + return env["BOT_TOKEN"] + + def github3_client() -> github3.GitHub: """ This will be removed in the future, use the GitHubBackend class instead. """ if not hasattr(GITHUB3_CLIENT, "client"): - with sensitive_env() as env: - GITHUB3_CLIENT.client = github3.login(token=env["BOT_TOKEN"]) + GITHUB3_CLIENT.client = github3.login(token=get_bot_token()) return GITHUB3_CLIENT.client @@ -89,11 +102,10 @@ def github_client() -> github.Github: This will be removed in the future, use the GitHubBackend class instead. """ if not hasattr(GITHUB_CLIENT, "client"): - with sensitive_env() as env: - GITHUB_CLIENT.client = github.Github( - auth=github.Auth.Token(env["BOT_TOKEN"]), - per_page=100, - ) + GITHUB_CLIENT.client = github.Github( + auth=github.Auth.Token(get_bot_token()), + per_page=100, + ) return GITHUB_CLIENT.client @@ -117,6 +129,26 @@ class GitConnectionMode(enum.StrEnum): class GitCliError(Exception): + """ + A generic error that occurred while running a git CLI command. + """ + + pass + + +class GitPlatformError(Exception): + """ + A generic error that occurred while interacting with a git platform. + """ + + pass + + +class DuplicatePullRequestError(GitPlatformError): + """ + Raised if a pull request already exists. + """ + pass @@ -136,22 +168,25 @@ class GitCli: If this does impact performance too much, we can consider a per-repository locking strategy. """ - @staticmethod + def __init__(self): + self.__hidden_tokens: list[str] = [] + @lock_git_operation() def _run_git_command( + self, cmd: list[str | Path], working_directory: Path | None = None, check_error: bool = True, - capture_text: bool = False, ) -> subprocess.CompletedProcess: """ - Run a git command. + Run a git command. stdout is only printed if the command fails. stderr is always printed. + If outputs are captured, they are never printed. + stdout is always captured, we capture stderr only if tokens are hidden. + :param cmd: The command to run, as a list of strings. :param working_directory: The directory to run the command in. If None, the command will be run in the current working directory. :param check_error: If True, raise a GitCliError if the git command fails. - :param capture_text: If True, capture the output of the git command as text. The output will be in the - returned result object. :return: The result of the git command. :raises GitCliError: If the git command fails and check_error is True. :raises FileNotFoundError: If the working directory does not exist. @@ -159,14 +194,94 @@ def _run_git_command( git_command = ["git"] + cmd logger.debug(f"Running git command: {git_command}") - capture_args = {"capture_output": True, "text": True} if capture_text else {} + + # we only need to capture stderr if we want to hide tokens + stderr_args = {"stderr": subprocess.PIPE} if self.__hidden_tokens else {} try: - return subprocess.run( - git_command, check=check_error, cwd=working_directory, **capture_args + p = subprocess.run( + git_command, + check=check_error, + cwd=working_directory, + stdout=subprocess.PIPE, + **stderr_args, + text=True, ) except subprocess.CalledProcessError as e: - raise GitCliError("Error running git command.") from e + e.stdout = replace_tokens(e.stdout, self.__hidden_tokens) + e.stderr = replace_tokens(e.stderr, self.__hidden_tokens) + logger.info( + f"Command '{' '.join(git_command)}' failed.\nstdout:\n{e.stdout}\nend of stdout" + ) + if self.__hidden_tokens: + logger.info(f"stderr:\n{e.stderr}\nend of stderr") + raise GitCliError(f"Error running git command: {repr(e)}") from e + + p.stdout = replace_tokens(p.stdout, self.__hidden_tokens) + p.stderr = replace_tokens(p.stderr, self.__hidden_tokens) + + if self.__hidden_tokens: + # we suppressed stderr, so we need to print it here + print(p.stderr, file=sys.stderr, end="") + + return p + + def add_hidden_token(self, token: str) -> None: + """ + Permanently hide a token in the logs. + + :param token: The token to hide. + """ + self.__hidden_tokens.append(token) + + @lock_git_operation() + def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): + """ + Add files to the git index with `git add`. + + :param git_dir: The directory of the git repository. + :param pathspec: The files to add. + :param all_: If True, not only add the files in pathspec, but also where the index already has an entry. + If _all is set with empty pathspec, all files in the entire working tree are updated. + :raises ValueError: If pathspec is empty and all_ is False. + :raises GitCliError: If the git command fails. + """ + if not pathspec and not all_: + raise ValueError("Either pathspec or all_ must be set.") + + all_arg = ["--all"] if all_ else [] + + self._run_git_command(["add", *all_arg, *pathspec], git_dir) + + @lock_git_operation() + def commit( + self, git_dir: Path, message: str, all_: bool = False, allow_empty: bool = False + ): + """ + Commit changes to the git repository with `git commit`. + :param git_dir: The directory of the git repository. + :param message: The commit message. + :param allow_empty: If True, allow an empty commit. + :param all_: Automatically stage files that have been modified and deleted, but new files are not affected. + :raises GitCliError: If the git command fails. + """ + all_arg = ["-a"] if all_ else [] + allow_empty_arg = ["--allow-empty"] if allow_empty else [] + + self._run_git_command( + ["commit", *all_arg, *allow_empty_arg, "-m", message], git_dir + ) + + def rev_parse_head(self, git_dir: Path) -> str: + """ + Get the commit hash of HEAD with `git rev-parse HEAD`. + :param git_dir: The directory of the git repository. + :return: The commit hash of HEAD. + :raises GitCliError: If the git command fails. + """ + ret = self._run_git_command(["rev-parse", "HEAD"], git_dir) + + return ret.stdout.strip() @lock_git_operation() def reset_hard(self, git_dir: Path, to_treeish: str = "HEAD"): @@ -210,6 +325,18 @@ def add_remote(self, git_dir: Path, remote_name: str, remote_url: str): """ self._run_git_command(["remote", "add", remote_name, remote_url], git_dir) + def push_to_url(self, git_dir: Path, remote_url: str, branch: str): + """ + Push changes to a remote URL. + + :param git_dir: The directory of the git repository. + :param remote_url: The URL of the remote. + :param branch: The branch to push to. + :raises GitCliError: If the git command fails. + """ + + self._run_git_command(["push", remote_url, branch], git_dir) + @lock_git_operation() def fetch_all(self, git_dir: Path): """ @@ -270,10 +397,7 @@ def checkout_branch( :raises FileNotFoundError: If git_dir does not exist """ track_flag = ["--track"] if track else [] - self._run_git_command( - ["checkout", "--quiet"] + track_flag + [branch], - git_dir, - ) + self._run_git_command(["checkout", "--quiet"] + track_flag + [branch], git_dir) @lock_git_operation() def checkout_new_branch( @@ -292,6 +416,25 @@ def checkout_new_branch( ["checkout", "--quiet", "-b", branch] + start_point_option, git_dir ) + def diffed_files( + self, git_dir: Path, commit_a: str, commit_b: str = "HEAD" + ) -> Iterator[Path]: + """ + Get the files that are different between two commits. + :param git_dir: The directory of the git repository. This should be the root of the repository. + If it is a subdirectory, only the files in that subdirectory will be returned. + :param commit_a: The first commit. + :param commit_b: The second commit. + :return: An iterator over the files that are different between the two commits. + """ + + # --relative ensures that we do not assemble invalid paths below if git_dir is a subdirectory + ret = self._run_git_command( + ["diff", "--name-only", "--relative", commit_a, commit_b], git_dir + ) + + return (git_dir / line for line in ret.stdout.splitlines()) + @lock_git_operation() def clone_fork_and_branch( self, @@ -406,26 +549,45 @@ def does_repository_exist(self, owner: str, repo_name: str) -> bool: """ pass - @staticmethod def get_remote_url( + self, owner: str, repo_name: str, connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, + token: str | None = None, ) -> str: """ Get the URL of a remote repository. :param owner: The owner of the repository. :param repo_name: The name of the repository. :param connection_mode: The connection mode to use. + :param token: A token to use for authentication. If falsy, no token is used. Use get_authenticated_remote_url + instead if you want to use the token of the current user. :raises ValueError: If the connection mode is not supported. + :raises RepositoryNotFoundError: If the repository does not exist. This is only raised if the backend relies on + the repository existing to generate the URL. """ # Currently we don't need any abstraction for other platforms than GitHub, so we don't build such abstractions. match connection_mode: case GitConnectionMode.HTTPS: - return f"https://github.com/{owner}/{repo_name}.git" + return f"https://{f'{token}@' if token else ''}github.com/{owner}/{repo_name}.git" case _: raise ValueError(f"Unsupported connection mode: {connection_mode}") + @abstractmethod + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + """ + Push changes to a repository. + :param owner: The owner of the repository. + :param repo_name: The name of the repository. + :param git_dir: The directory of the git repository. + :param branch: The branch to push to. + :raises GitPlatformError: If the push fails. + """ + pass + @abstractmethod def fork(self, owner: str, repo_name: str): """ @@ -505,6 +667,75 @@ def is_api_limit_reached(self) -> bool: """ return self.get_api_requests_left() in (0, None) + @abstractmethod + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + """ + Create a pull request from a forked repository. It is assumed that the forked repository is owned by the + current user and has the same name as the target repository. + + :param target_owner: The owner of the target repository. + :param target_repo: The name of the target repository. + :param base_branch: The base branch of the pull request, located in the target repository. + :param head_branch: The head branch of the pull request, located in the forked repository. + :param title: The title of the pull request. + :param body: The body of the pull request. + + :returns: The data of the created pull request. + + :raises GitPlatformError: If the pull request could not be created. + :raises DuplicatePullRequestError: If a pull request already exists and the backend checks for it. + """ + pass + + @abstractmethod + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ) -> None: + """ + Comment on an existing pull request. + :param repo_owner: The owner of the repository. + :param repo_name: The name of the repository. + :param pr_number: The number of the pull request. + :param comment: The comment to post. + + :raises RepositoryNotFoundError: If the repository does not exist. + :raises GitPlatformError: If the comment could not be posted, including if the pull request does not exist. + """ + pass + + +class _Github3SessionWrapper: + """ + This is a wrapper around the github3.session.GitHubSession that allows us to intercept the response headers. + """ + + def __init__(self, session: GitHubSession): + super().__init__() + self._session = session + self.last_response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict() + + def __getattr__(self, item): + return getattr(self._session, item) + + def _forward_request(self, method, *args, **kwargs): + response = method(*args, **kwargs) + self.last_response_headers = copy.deepcopy(response.headers) + return response + + def post(self, *args, **kwargs): + return self._forward_request(self._session.post, *args, **kwargs) + + def get(self, *args, **kwargs): + return self._forward_request(self._session.get, *args, **kwargs) + class GitHubBackend(GitPlatformBackend): """ @@ -520,9 +751,32 @@ class GitHubBackend(GitPlatformBackend): The number of items to fetch per page from the GitHub API. """ - def __init__(self, github3_client: github3.GitHub, pygithub_client: github.Github): - super().__init__(GitCli()) + def __init__( + self, + github3_client: github3.GitHub, + pygithub_client: github.Github, + token: str, + ): + """ + Create a new GitHubBackend. + + Note: Because we need additional response headers, we wrap the github3 session of the github3 client + with our own session wrapper and replace the github3 client's session with it. + + :param github3_client: The github3 client to use for interacting with the GitHub API. + :param pygithub_client: The PyGithub client to use for interacting with the GitHub API. + :param token: The token that will be hidden in CLI outputs and used for writing to git repositories. Note that + you need to authenticate github3 and PyGithub yourself. Use the `from_token` class method to create an instance + that has all necessary clients set up. + """ + cli = GitCli() + cli.add_hidden_token(token) + super().__init__(cli) + self.__token = token self.github3_client = github3_client + self._github3_session = _Github3SessionWrapper(self.github3_client.session) + self.github3_client.session = self._github3_session + self.pygithub_client = pygithub_client @classmethod @@ -530,12 +784,22 @@ def from_token(cls, token: str): return cls( github3.login(token=token), github.Github(auth=github.Auth.Token(token), per_page=cls._GITHUB_PER_PAGE), + token=token, ) def does_repository_exist(self, owner: str, repo_name: str) -> bool: repo = self.github3_client.repository(owner, repo_name) return repo is not None + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + # we need an authenticated URL with write access + remote_url = self.get_remote_url( + owner, repo_name, GitConnectionMode.HTTPS, self.__token + ) + self.cli.push_to_url(git_dir, remote_url, branch) + @lock_git_operation() def fork(self, owner: str, repo_name: str): if self.does_repository_exist(self.user, repo_name): @@ -612,6 +876,69 @@ def get_api_requests_left(self) -> int | None: return remaining_limit + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + repo: github3.repos.Repository = self.github3_client.repository( + target_owner, target_repo + ) + + try: + response: github3.pulls.ShortPullRequest | None = repo.create_pull( + title=title, + base=base_branch, + head=f"{self.user}:{head_branch}", + body=body, + ) + except github3.exceptions.UnprocessableEntity as e: + if any("already exists" in error.get("message", "") for error in e.errors): + raise DuplicatePullRequestError( + f"Pull request from {self.user}:{head_branch} to {target_owner}:{base_branch} already exists." + ) from e + raise + + if response is None: + raise GitPlatformError("Could not create pull request.") + + # fields like ETag and Last-Modified are stored in the response headers, we need to extract them + header_fields = { + k: self._github3_session.last_response_headers[k] + for k in PullRequestData.HEADER_FIELDS + } + + # note: this ignores extra fields in the response + return PullRequestData.model_validate(response.as_dict() | header_fields) + + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ) -> None: + try: + repo = self.github3_client.repository(repo_owner, repo_name) + except github3.exceptions.NotFoundError as e: + raise RepositoryNotFoundError( + f"Repository {repo_owner}/{repo_name} not found." + ) from e + + try: + pr = repo.pull_request(pr_number) + except github3.exceptions.NotFoundError as e: + raise GitPlatformError( + f"Pull request {repo_owner}/{repo_name}#{pr_number} not found." + ) from e + + try: + pr.create_comment(comment) + except github3.GitHubError as e: + raise GitPlatformError( + f"Could not comment on pull request {repo_owner}/{repo_name}#{pr_number}." + ) from e + class DryRunBackend(GitPlatformBackend): """ @@ -626,7 +953,12 @@ class DryRunBackend(GitPlatformBackend): def __init__(self): super().__init__(GitCli()) - self._repos: set[str] = set() + self._repos: dict[str, str] = {} + """ + _repos maps from repository name to the owner of the upstream repository. + If a remote URL of a fork is requested with get_remote_url, _USER (the virtual current user) is + replaced by the owner of the upstream repository. This allows cloning the forked repository. + """ def get_api_requests_left(self) -> Bound: return Bound.INFINITY @@ -640,15 +972,48 @@ def does_repository_exist(self, owner: str, repo_name: str) -> bool: self.get_remote_url(owner, repo_name, GitConnectionMode.HTTPS) ) + def get_remote_url( + self, + owner: str, + repo_name: str, + connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, + token: str | None = None, + ) -> str: + if owner != self._USER: + return super().get_remote_url(owner, repo_name, connection_mode, token) + # redirect to the upstream repository + try: + upstream_owner = self._repos[repo_name] + except KeyError: + raise RepositoryNotFoundError( + f"Repository {owner}/{repo_name} appears to be a virtual fork but does not exist. Note that dry-run " + "forks are persistent only for the duration of the backend instance." + ) + + return super().get_remote_url(upstream_owner, repo_name, connection_mode, token) + + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + logger.debug( + f"Dry Run: Pushing changes from {git_dir} to {owner}/{repo_name} on branch {branch}." + ) + @lock_git_operation() def fork(self, owner: str, repo_name: str): if repo_name in self._repos: - raise ValueError(f"Fork of {repo_name} already exists.") + logger.debug(f"Fork of {repo_name} already exists. Doing nothing.") + return + + if not self.does_repository_exist(owner, repo_name): + raise RepositoryNotFoundError( + f"Cannot fork non-existing repository {owner}/{repo_name}." + ) logger.debug( f"Dry Run: Creating fork of {owner}/{repo_name} for user {self._USER}." ) - self._repos.add(repo_name) + self._repos[repo_name] = owner def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): logger.debug( @@ -659,13 +1024,77 @@ def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): def user(self) -> str: return self._USER + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + logger.debug( + textwrap.dedent( + f""" + ============================================================== + Dry Run: Create Pull Request + Title: "{title}" + Target Repository: {target_owner}/{target_repo} + Branches: {self.user}:{head_branch} -> {target_owner}:{base_branch} + Body: + """ + ) + ) + + logger.debug(body) + logger.debug("==============================================================") + + now = datetime.now() + return PullRequestData.model_validate( + { + "ETag": "GITHUB_PR_ETAG", + "Last-Modified": utils.format_datetime(now), + "id": 13371337, + "html_url": f"https://github.com/{target_owner}/{target_repo}/pulls/1337", + "created_at": now, + "mergeable_state": GithubPullRequestMergeableState.CLEAN, + "mergeable": True, + "merged": False, + "draft": False, + "number": 1337, + "state": PullRequestState.OPEN, + "head": PullRequestInfoHead(ref=head_branch), + "base": GithubPullRequestBase(repo=GithubRepository(name=target_repo)), + } + ) + + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ): + if not self.does_repository_exist(repo_owner, repo_name): + raise RepositoryNotFoundError( + f"Repository {repo_owner}/{repo_name} not found." + ) + + logger.debug( + textwrap.dedent( + f""" + ============================================================== + Dry Run: Comment on Pull Request + Pull Request: {repo_owner}/{repo_name}#{pr_number} + Comment: + {comment} + ============================================================== + """ + ) + ) + def github_backend() -> GitHubBackend: """ This helper method will be removed in the future, use the GitHubBackend class directly. """ - with sensitive_env() as env: - return GitHubBackend.from_token(env["BOT_TOKEN"]) + return GitHubBackend.from_token(get_bot_token()) def is_github_api_limit_reached() -> bool: @@ -679,88 +1108,6 @@ def is_github_api_limit_reached() -> bool: return backend.is_api_limit_reached() -def feedstock_url(fctx: FeedstockContext, protocol: str = "ssh") -> str: - """Returns the URL for a conda-forge feedstock.""" - feedstock = fctx.feedstock_name + "-feedstock" - if feedstock.startswith("http://github.com/"): - return feedstock - elif feedstock.startswith("https://github.com/"): - return feedstock - elif feedstock.startswith("git@github.com:"): - return feedstock - protocol = protocol.lower() - if protocol == "http": - url = "http://github.com/conda-forge/" + feedstock + ".git" - elif protocol == "https": - url = "https://github.com/conda-forge/" + feedstock + ".git" - elif protocol == "ssh": - url = "git@github.com:conda-forge/" + feedstock + ".git" - else: - msg = f"Unrecognized github protocol {protocol}, must be ssh, http, or https." - raise ValueError(msg) - return url - - -def feedstock_repo(fctx: FeedstockContext) -> str: - """Gets the name of the feedstock repository.""" - return fctx.feedstock_name + "-feedstock" - - -@lock_git_operation() -def get_repo( - fctx: FeedstockContext, - branch: str, - base_branch: str = "main", -) -> Tuple[str, github3.repos.Repository] | Tuple[Literal[False], Literal[False]]: - """Get the feedstock repo - - Parameters - ---------- - fctx : FeedstockContext - Feedstock context used for constructing feedstock urls, etc. - branch : str - The branch to be made. - base_branch : str, optional - The base branch from which to make the new branch. - - Returns - ------- - recipe_dir : str - The recipe directory - repo : github3 repository - The github3 repository object. - """ - backend = github_backend() - feedstock_repo_name = feedstock_repo(fctx) - - try: - backend.fork("conda-forge", feedstock_repo_name) - except RepositoryNotFoundError: - logger.warning(f"Could not fork conda-forge/{feedstock_repo_name}") - with fctx.attrs["pr_info"] as pri: - pri["bad"] = f"{fctx.feedstock_name}: Git repository not found.\n" - return False, False - - feedstock_dir = Path(GIT_CLONE_DIR) / (fctx.feedstock_name + "-feedstock") - - backend.clone_fork_and_branch( - upstream_owner="conda-forge", - repo_name=feedstock_repo_name, - target_dir=feedstock_dir, - new_branch=branch, - base_branch=base_branch, - ) - - # This is needed because we want to migrate to the new backend step-by-step - repo: github3.repos.Repository | None = github3_client().repository( - "conda-forge", feedstock_repo_name - ) - - assert repo is not None - - return str(feedstock_dir), repo - - @lock_git_operation() def delete_branch(pr_json: LazyJson, dry_run: bool = False) -> None: ref = pr_json["head"]["ref"] @@ -772,17 +1119,18 @@ def delete_branch(pr_json: LazyJson, dry_run: bool = False) -> None: gh = github3_client() deploy_repo = gh.me().login + "/" + name - with sensitive_env() as env: - run_command_hiding_token( - [ - "git", - "push", - f"https://{env['BOT_TOKEN']}@github.com/{deploy_repo}.git", - "--delete", - ref, - ], - token=env["BOT_TOKEN"], - ) + token = get_bot_token() + + run_command_hiding_token( + [ + "git", + "push", + f"https://{token}@github.com/{deploy_repo}.git", + "--delete", + ref, + ], + token=token, + ) # Replace ref so we know not to try again pr_json["head"]["ref"] = "this_is_not_a_branch" @@ -849,11 +1197,10 @@ def lazy_update_pr_json( pr_json : dict-like A dict-like object with the current PR information. """ - with sensitive_env() as env: - hdrs = { - "Authorization": f"token {env['BOT_TOKEN']}", - "Accept": "application/vnd.github.v3+json", - } + hdrs = { + "Authorization": f"token {get_bot_token()}", + "Accept": "application/vnd.github.v3+json", + } if not force and "ETag" in pr_json: hdrs["If-None-Match"] = pr_json["ETag"] @@ -982,99 +1329,6 @@ def close_out_labels( return None -@lock_git_operation() -def push_repo( - fctx: FeedstockContext, - feedstock_dir: str, - body: str, - repo: github3.repos.Repository, - title: str, - branch: str, - base_branch: str = "main", - head: Optional[str] = None, - dry_run: bool = False, -) -> Union[dict, bool, None]: - """Push a repo up to github - - Parameters - ---------- - fctx : FeedstockContext - Feedstock context used for constructing feedstock urls, etc. - feedstock_dir : str - The feedstock directory - body : str - The PR body. - repo : github3.repos.Repository - The feedstock repo as a github3 object. - title : str - The title of the PR. - head : str, optional - The github head for the PR in the form `username:branch`. - branch : str - The head branch of the PR. - base_branch : str, optional - The base branch or target branch of the PR. - - Returns - ------- - pr_json: dict - The dict representing the PR, can be used with `from_json` - to create a PR instance. - """ - with sensitive_env() as env, pushd(feedstock_dir): - # Copyright (c) 2016 Aaron Meurer, Gil Forsyth - token = env["BOT_TOKEN"] - gh_username = github3_client().me().login - - if head is None: - head = gh_username + ":" + branch - - deploy_repo = gh_username + "/" + fctx.feedstock_name + "-feedstock" - if dry_run: - repo_url = f"https://github.com/{deploy_repo}.git" - print(f"dry run: adding remote and pushing up branch for {repo_url}") - else: - ecode = run_command_hiding_token( - [ - "git", - "remote", - "add", - "regro_remote", - f"https://{token}@github.com/{deploy_repo}.git", - ], - token=token, - ) - if ecode != 0: - print("Failed to add git remote!") - return False - - ecode = run_command_hiding_token( - ["git", "push", "--set-upstream", "regro_remote", branch], - token=token, - ) - if ecode != 0: - print("Failed to push to remote!") - return False - - # lastly make a PR for the feedstock - print("Creating conda-forge feedstock pull request...") - if dry_run: - print(f"dry run: create pr with title: {title}") - return False - else: - pr = repo.create_pull(title, base_branch, head, body=body) - if pr is None: - print("Failed to create pull request!") - return False - else: - print("Pull request created at " + pr.html_url) - - # Return a json object so we can remake the PR if needed - pr_dict: dict = pr.as_dict() - - return trim_pr_json_keys(pr_dict) - - def comment_on_pr(pr_json, comment, repo): """Make a comment on a PR. diff --git a/conda_forge_tick/migration_runner.py b/conda_forge_tick/migration_runner.py index c723ec589..651bcfc55 100644 --- a/conda_forge_tick/migration_runner.py +++ b/conda_forge_tick/migration_runner.py @@ -3,8 +3,9 @@ import os import shutil import tempfile +from pathlib import Path -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.lazy_json_backends import LazyJson, dumps from conda_forge_tick.os_utils import ( chmod_plus_rwX, @@ -221,12 +222,14 @@ def run_migration_local( - pr_body: The PR body for the migration. """ - feedstock_ctx = FeedstockContext( + # it would be better if we don't re-instantiate ClonedFeedstockContext ourselves and let + # FeedstockContext.reserve_clone_directory be the only way to create a ClonedFeedstockContext + feedstock_ctx = ClonedFeedstockContext( feedstock_name=feedstock_name, attrs=node_attrs, + _default_branch=default_branch, + local_clone_dir=Path(feedstock_dir), ) - feedstock_ctx.default_branch = default_branch - feedstock_ctx.feedstock_dir = feedstock_dir recipe_dir = os.path.join(feedstock_dir, "recipe") data = { diff --git a/conda_forge_tick/migrators/arch.py b/conda_forge_tick/migrators/arch.py index bfe81dd6f..274ed3ad2 100644 --- a/conda_forge_tick/migrators/arch.py +++ b/conda_forge_tick/migrators/arch.py @@ -4,7 +4,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.make_graph import ( get_deps_from_outputs_lut, make_outputs_lut_from_graph, @@ -213,7 +213,7 @@ def migrate( def pr_title(self, feedstock_ctx: FeedstockContext) -> str: return "Arch Migrator" - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( dedent( @@ -384,7 +384,7 @@ def migrate( def pr_title(self, feedstock_ctx: FeedstockContext) -> str: return "ARM OSX Migrator" - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( dedent( diff --git a/conda_forge_tick/migrators/broken_rebuild.py b/conda_forge_tick/migrators/broken_rebuild.py index 66e6b1303..fb3d13718 100644 --- a/conda_forge_tick/migrators/broken_rebuild.py +++ b/conda_forge_tick/migrators/broken_rebuild.py @@ -2,6 +2,7 @@ import networkx as nx +from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.migrators.core import Migrator BROKEN_PACKAGES = """\ @@ -380,7 +381,7 @@ def migrate(self, recipe_dir, attrs, **kwargs): self.set_build_number(os.path.join(recipe_dir, "meta.yaml")) return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( """\ diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 9f1227422..22b740426 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -10,7 +10,7 @@ import dateutil.parser import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort @@ -455,7 +455,9 @@ def migrate( """ return self.migrator_uid(attrs) - def pr_body(self, feedstock_ctx: FeedstockContext, add_label_text=True) -> str: + def pr_body( + self, feedstock_ctx: ClonedFeedstockContext, add_label_text=True + ) -> str: """Create a PR message body Returns diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index ac2c47779..ecdf06ede 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -10,7 +10,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.feedstock_parser import PIN_SEP_PAT from conda_forge_tick.make_graph import get_deps_from_outputs_lut from conda_forge_tick.migrators.core import GraphMigrator, Migrator, MiniMigrator @@ -280,7 +280,7 @@ def migrate( return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: "FeedstockContext") -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) if feedstock_ctx.feedstock_name == "conda-forge-pinning": additional_body = ( @@ -534,7 +534,7 @@ def migrate( return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: "FeedstockContext") -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = ( "This PR has been triggered in an effort to update the pin for" " **{name}**. The current pinned version is {current_pin}, " diff --git a/conda_forge_tick/migrators/replacement.py b/conda_forge_tick/migrators/replacement.py index 7df2be3f5..71ccf2dce 100644 --- a/conda_forge_tick/migrators/replacement.py +++ b/conda_forge_tick/migrators/replacement.py @@ -6,7 +6,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator if typing.TYPE_CHECKING: @@ -127,7 +127,7 @@ def migrate( self.set_build_number(os.path.join(recipe_dir, "meta.yaml")) return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( "I noticed that this recipe depends on `%s` instead of \n" diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index d860f7f90..056359cc9 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -11,7 +11,7 @@ import networkx as nx from conda.models.version import VersionOrder -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator from conda_forge_tick.models.pr_info import MigratorName from conda_forge_tick.os_utils import pushd @@ -219,7 +219,7 @@ def migrate( ) ) - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: if feedstock_ctx.feedstock_name in self.effective_graph.nodes: pred = [ ( @@ -319,7 +319,7 @@ def pr_body(self, feedstock_ctx: FeedstockContext) -> str: return super().pr_body(feedstock_ctx, add_label_text=False).format(body) - def _hint_and_maybe_update_deps(self, feedstock_ctx): + def _hint_and_maybe_update_deps(self, feedstock_ctx: ClonedFeedstockContext): update_deps = get_keys_default( feedstock_ctx.attrs, ["conda-forge.yml", "bot", "inspection"], @@ -340,7 +340,7 @@ def _hint_and_maybe_update_deps(self, feedstock_ctx): try: _, hint = get_dep_updates_and_hints( update_deps, - os.path.join(feedstock_ctx.feedstock_dir, "recipe"), + str(feedstock_ctx.local_clone_dir / "recipe"), feedstock_ctx.attrs, self.python_nodes, "new_version", diff --git a/conda_forge_tick/migrators_types.pyi b/conda_forge_tick/migrators_types.py similarity index 98% rename from conda_forge_tick/migrators_types.pyi rename to conda_forge_tick/migrators_types.py index 6d48eae6c..51d0281c1 100644 --- a/conda_forge_tick/migrators_types.pyi +++ b/conda_forge_tick/migrators_types.py @@ -5,6 +5,7 @@ PackageName = typing.NewType("PackageName", str) + class AboutTypedDict(TypedDict, total=False): description: str dev_url: str @@ -15,6 +16,7 @@ class AboutTypedDict(TypedDict, total=False): license_file: str summary: str + # PRStateOpen: Literal["open"] # PRStateClosed: Literal["closed"] # PRStateMerged: Literal["merged"] @@ -22,35 +24,43 @@ class AboutTypedDict(TypedDict, total=False): # PRState = Literal[PRStateClosed, PRStateMerged, PRStateOpen] PRState = typing.NewType("PRState", str) -class PRHead_TD(TypedDict, tota=False): + +class PRHead_TD(TypedDict, total=False): ref: str + class PR_TD(TypedDict, total=False): state: PRState head: PRHead_TD + class BlasRebuildMigrateTypedDict(TypedDict): bot_rerun: bool migrator_name: str migrator_version: int name: str + class BuildRunExportsDict(TypedDict, total=False): strong: List[PackageName] weak: List[PackageName] + class BuildTypedDict(TypedDict, total=False): noarch: str number: str script: str run_exports: Union[List[PackageName], BuildRunExportsDict] + ExtraTypedDict = TypedDict("ExtraTypedDict", {"recipe-maintainers": List[str]}) + # class HTypedDict(TypedDict): # data: 'DataTypedDict' # keys: List[str] + class MetaYamlOutputs(TypedDict, total=False): name: str requirements: "RequirementsTypedDict" @@ -58,6 +68,7 @@ class MetaYamlOutputs(TypedDict, total=False): # TODO: Not entirely sure this is right build: BuildRunExportsDict + class RecipeTypedDict(TypedDict, total=False): about: "AboutTypedDict" build: "BuildTypedDict" @@ -68,6 +79,7 @@ class RecipeTypedDict(TypedDict, total=False): test: "TestTypedDict" outputs: List[MetaYamlOutputs] + class MigrationUidTypedDict(TypedDict, total=False): bot_rerun: bool migrator_name: str @@ -77,31 +89,37 @@ class MigrationUidTypedDict(TypedDict, total=False): # Used by version migrators version: str + class PackageTypedDict(TypedDict): name: str version: str + class RequirementsTypedDict(TypedDict, total=False): build: List[str] host: List[str] run: List[str] + class SourceTypedDict(TypedDict, total=False): fn: str patches: List[str] sha256: str url: str + class TestTypedDict(TypedDict, total=False): commands: List[str] imports: List[str] requires: List[str] requirements: List[str] + class PRedElementTypedDict(TypedDict, total=False): data: MigrationUidTypedDict PR: PR_TD + class AttrsTypedDict_(TypedDict, total=False): about: AboutTypedDict build: BuildTypedDict @@ -124,12 +142,15 @@ class AttrsTypedDict_(TypedDict, total=False): # TODO: ADD in # "conda-forge.yml": + class CondaForgeYamlContents(TypedDict, total=False): provider: Dict[str, str] + CondaForgeYaml = TypedDict( "CondaForgeYaml", {"conda-forge.yml": CondaForgeYamlContents} ) + class AttrsTypedDict(AttrsTypedDict_, CondaForgeYaml): pass diff --git a/conda_forge_tick/models/common.py b/conda_forge_tick/models/common.py index beb445213..77b548fae 100644 --- a/conda_forge_tick/models/common.py +++ b/conda_forge_tick/models/common.py @@ -10,6 +10,7 @@ BeforeValidator, ConfigDict, Field, + PlainSerializer, UrlConstraints, ) from pydantic_core import Url @@ -25,7 +26,7 @@ class StrictBaseModel(BaseModel): class ValidatedBaseModel(BaseModel): - model_config = ConfigDict(validate_assignment=True, extra="allow") + model_config = ConfigDict(validate_assignment=True, extra="ignore") def before_validator_ensure_dict(value: Any) -> dict: @@ -77,7 +78,7 @@ def none_to_empty_dict(value: T | None) -> T | dict[Never]: return value -NoneIsEmptyDict = Annotated[dict[T], BeforeValidator(none_to_empty_dict)] +NoneIsEmptyDict = Annotated[dict[K, V], BeforeValidator(none_to_empty_dict)] """ A generic dict type that converts `None` to an empty dict. This should not be needed if this proper data model is used in production. @@ -151,22 +152,15 @@ def parse_rfc_2822_date(value: str) -> datetime: return email.utils.parsedate_to_datetime(value) -RFC2822Date = Annotated[datetime, BeforeValidator(parse_rfc_2822_date)] - - -def none_to_empty_dict(value: T | None) -> T | dict[Never, Never]: - """ - Convert `None` to an empty dictionary f, otherwise keep the value as is. - """ - if value is None: - return {} - return value +def serialize_rfc_2822_date(value: datetime) -> str: + return email.utils.format_datetime(value) -NoneIsEmptyDict = Annotated[dict[K, V], BeforeValidator(none_to_empty_dict)] -""" -A generic dict type that converts `None` to an empty dict. -""" +RFC2822Date = Annotated[ + datetime, + BeforeValidator(parse_rfc_2822_date), + PlainSerializer(serialize_rfc_2822_date), +] GitUrl = Annotated[Url, UrlConstraints(allowed_schemes=["git"])] diff --git a/conda_forge_tick/models/pr_json.py b/conda_forge_tick/models/pr_json.py index 261b3e8d7..4e1ef6f2f 100644 --- a/conda_forge_tick/models/pr_json.py +++ b/conda_forge_tick/models/pr_json.py @@ -1,11 +1,15 @@ from datetime import datetime from enum import StrEnum -from typing import Literal +from typing import ClassVar, Literal from pydantic import UUID4, AnyHttpUrl, Field, TypeAdapter from pydantic_extra_types.color import Color -from conda_forge_tick.models.common import RFC2822Date, StrictBaseModel +from conda_forge_tick.models.common import ( + RFC2822Date, + StrictBaseModel, + ValidatedBaseModel, +) class PullRequestLabelShort(StrictBaseModel): @@ -48,7 +52,7 @@ class PullRequestState(StrEnum): """ -class PullRequestInfoHead(StrictBaseModel): +class PullRequestInfoHead(ValidatedBaseModel): ref: str """ The head branch of the pull request. @@ -73,22 +77,31 @@ class GithubPullRequestMergeableState(StrEnum): CLEAN = "clean" -class GithubRepository(StrictBaseModel): +class GithubRepository(ValidatedBaseModel): name: str -class GithubPullRequestBase(StrictBaseModel): +class GithubPullRequestBase(ValidatedBaseModel): repo: GithubRepository -class PullRequestDataValid(StrictBaseModel): +class PullRequestDataValid(ValidatedBaseModel): """ Information about a pull request, as retrieved from the GitHub API. Refer to git_utils.PR_KEYS_TO_KEEP for the keys that are kept in the PR object. + ALSO UPDATE PR_KEYS_TO_KEEP IF YOU CHANGE THIS CLASS! GitHub documentation: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#get-a-pull-request """ + HEADER_FIELDS: ClassVar[set[str]] = { + "ETag", + "Last-Modified", + } + """ + A set of all header fields that are stored in the PR object. + """ + e_tag: str | None = Field(None, alias="ETag") """ HTTP ETag header field, allowing us to quickly check if the PR has changed. diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 77aebb673..54c2dae46 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -39,8 +39,6 @@ load_existing_graph, ) -from .git_utils import feedstock_url - GH_MERGE_STATE_STATUS = [ "behind", "blocked", @@ -273,7 +271,7 @@ def graph_migrator_status( .get("PR", {}) .get( "html_url", - feedstock_url(fctx=feedstock_ctx, protocol="https").strip(".git"), + feedstock_ctx.git_href, ), ) @@ -300,7 +298,7 @@ def graph_migrator_status( # I needed to fake some PRs they don't have html_urls though node_metadata["pr_url"] = pr_json["PR"].get( "html_url", - feedstock_url(fctx=feedstock_ctx, protocol="https").strip(".git"), + feedstock_ctx.git_href, ) node_metadata["pr_status"] = pr_json["PR"].get("mergeable_state", "") diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index 0356cfadb..0f0ceac47 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -14,7 +14,7 @@ import typing import warnings from collections import defaultdict -from typing import Any, Callable, Dict, Iterable, Optional, Set, Tuple, cast +from typing import Any, Callable, Dict, Iterable, Optional, Set, Tuple, cast, overload import jinja2 import jinja2.sandbox @@ -1298,12 +1298,38 @@ def change_log_level(logger, new_level): logger.setLevel(saved_logger_level) +@overload +def replace_tokens(s: str, tokens: Iterable[str]) -> str: ... + + +@overload +def replace_tokens(s: None, tokens: Iterable[str]) -> None: ... + + +def replace_tokens(s: str | None, tokens: Iterable[str]) -> str | None: + """ + Replace tokens in a string with asterisks of the same length. + + None values are passed through. + + :param s: The string to replace tokens in. + :param tokens: The tokens to replace. + + :return: The string with the tokens replaced. + """ + if not s: + return s + for token in tokens: + s = s.replace(token, "*" * len(token)) + return s + + def print_subprocess_output_strip_token( - completed_process: subprocess.CompletedProcess, token: str + completed_process: subprocess.CompletedProcess, *tokens: str ) -> None: """ Use this function to print the outputs (stdout and stderr) of a subprocess.CompletedProcess object - that may contain sensitive information. The token will be replaced with a string + that may contain sensitive information. The token or tokens will be replaced with a string of asterisks of the same length. This function assumes that you have called subprocess.run() with the arguments text=True, stdout=subprocess.PIPE, @@ -1315,7 +1341,7 @@ def print_subprocess_output_strip_token( :param completed_process: The subprocess.CompletedProcess object to print the outputs of. You have probably obtained this object by calling subprocess.run(). - :param token: The token to replace with asterisks. + :param tokens: The token or tokens to replace with asterisks. :raises ValueError: If the completed_process object does not contain str in stdout or stderr. """ @@ -1331,7 +1357,7 @@ def print_subprocess_output_strip_token( "text=True." ) - captured = captured.replace(token, "*" * len(token)) + replace_tokens(captured, tokens) print(captured, file=out_dev, end="") out_dev.flush() diff --git a/tests/fake_lazy_json.py b/tests/fake_lazy_json.py new file mode 100644 index 000000000..2880d8522 --- /dev/null +++ b/tests/fake_lazy_json.py @@ -0,0 +1,15 @@ +from types import TracebackType +from typing import Self + + +class FakeLazyJson(dict): + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + pass diff --git a/tests/github_api/create_issue_comment_pytest.json b/tests/github_api/create_issue_comment_pytest.json new file mode 100644 index 000000000..ba97b19f9 --- /dev/null +++ b/tests/github_api/create_issue_comment_pytest.json @@ -0,0 +1,33 @@ +{ + "id": 1, + "node_id": "MDEyOklzc3VlQ29tbWVudDE=", + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments/1", + "html_url": "https://github.com/conda-forge/pytest-feedstock/issues/1337#issuecomment-1", + "body": "ISSUE_COMMENT_BODY", + "body_html": "ISSUE_COMMENT_BODY_HTML", + "body_text": "ISSUE_COMMENT_BODY_TEXT", + "user": { + "login": "regro-cf-autotick-bot", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2011-04-14T16:00:49Z", + "updated_at": "2011-04-14T16:00:49Z", + "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337", + "author_association": "CONTRIBUTOR" +} diff --git a/tests/github_api/create_pull_duplicate.json b/tests/github_api/create_pull_duplicate.json new file mode 100644 index 000000000..aff82e963 --- /dev/null +++ b/tests/github_api/create_pull_duplicate.json @@ -0,0 +1,12 @@ +{ + "message": "Validation Failed", + "errors": [ + { + "resource": "PullRequest", + "code": "custom", + "message": "A pull request already exists for OWNER:BRANCH." + } + ], + "documentation_url": "https://docs.github.com/rest/pulls/pulls#create-a-pull-request", + "status": "422" +} diff --git a/tests/github_api/create_pull_validation_error.json b/tests/github_api/create_pull_validation_error.json new file mode 100644 index 000000000..ae6f3df17 --- /dev/null +++ b/tests/github_api/create_pull_validation_error.json @@ -0,0 +1,12 @@ +{ + "message": "Validation Failed", + "errors": [ + { + "resource": "PullRequest", + "field": "head", + "code": "invalid" + } + ], + "documentation_url": "https://docs.github.com/rest/pulls/pulls#create-a-pull-request", + "status": "422" +} diff --git a/tests/github_api/get_pull_pytest.json b/tests/github_api/get_pull_pytest.json new file mode 100644 index 000000000..9bb5754db --- /dev/null +++ b/tests/github_api/get_pull_pytest.json @@ -0,0 +1,360 @@ +{ + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337", + "id": 1853804278, + "node_id": "PR_kwDOAgM_Js5ufs72", + "html_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337", + "diff_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337.diff", + "patch_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337.patch", + "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337", + "number": 1337, + "state": "open", + "locked": false, + "title": "PR_TITLE", + "user": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "body": "PR_BODY", + "created_at": "2024-05-03T17:04:20Z", + "updated_at": "2024-05-27T13:31:50Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "351d0b862d129b53b8c7db2260d208d3a27fb204", + "assignee": null, + "assignees": [], + "requested_reviewers": [ + ], + "requested_teams": [], + "labels": [], + "milestone": null, + "draft": false, + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/commits", + "review_comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/comments", + "review_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095", + "head": { + "label": "regro-cf-autotick-bot:HEAD_BRANCH", + "ref": "HEAD_BRANCH", + "sha": "0eaa1de035b8720c8b85a7f435a0ae7037fe9095", + "user": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "repo": { + "id": 772632103, + "node_id": "R_kgDOLg1uJw", + "name": "pytest-feedstock", + "full_name": "regro-cf-autotick-bot/pytest-feedstock", + "private": false, + "owner": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock", + "description": "The tool for managing conda-forge feedstocks.", + "fork": true, + "url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock", + "forks_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/deployments", + "created_at": "2024-03-15T15:21:35Z", + "updated_at": "2024-05-16T15:11:20Z", + "pushed_at": "2024-05-16T15:20:39Z", + "git_url": "git://github.com/regro-cf-autotick-bot/pytest-feedstock.git", + "ssh_url": "git@github.com:regro-cf-autotick-bot/pytest-feedstock.git", + "clone_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git", + "svn_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock", + "homepage": "https://conda-forge.org/", + "size": 3959, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Python", + "has_issues": false, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "main" + } + }, + "base": { + "label": "conda-forge:main", + "ref": "main", + "sha": "59aa8df51b362904f0a8eb72274ad9458a5e4d8e", + "user": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 33767206, + "node_id": "MDEwOlJlcG9zaXRvcnkzMzc2NzIwNg==", + "name": "pytest-feedstock", + "full_name": "conda-forge/pytest-feedstock", + "private": false, + "owner": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/conda-forge/pytest-feedstock", + "description": "The tool for managing conda-forge feedstocks.", + "fork": false, + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock", + "forks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/deployments", + "created_at": "2015-04-11T07:38:36Z", + "updated_at": "2024-05-28T09:55:10Z", + "pushed_at": "2024-05-28T09:55:05Z", + "git_url": "git://github.com/conda-forge/pytest-feedstock.git", + "ssh_url": "git@github.com:conda-forge/pytest-feedstock.git", + "clone_url": "https://github.com/conda-forge/pytest-feedstock.git", + "svn_url": "https://github.com/conda-forge/pytest-feedstock", + "homepage": "https://conda-forge.org/", + "size": 3808, + "stargazers_count": 147, + "watchers_count": 147, + "language": "Python", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 166, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 334, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [ + "continuous-integration", + "hacktoberfest" + ], + "visibility": "public", + "forks": 166, + "open_issues": 334, + "watchers": 147, + "default_branch": "main" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + }, + "html": { + "href": "https://github.com/conda-forge/pytest-feedstock/pull/1337" + }, + "issue": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337" + }, + "comments": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095" + } + }, + "author_association": "MEMBER", + "auto_merge": null, + "body_html": "BODY_HTML", + "body_text": "BODY_TEXT", + "active_lock_reason": null, + "merged": false, + "mergeable": true, + "rebaseable": true, + "mergeable_state": "clean", + "merged_by": null, + "comments": 7, + "review_comments": 23, + "maintainer_can_modify": true, + "commits": 8, + "additions": 601, + "deletions": 752, + "changed_files": 32 +} diff --git a/tests/github_api/get_repo_pytest.json b/tests/github_api/get_repo_pytest.json new file mode 100644 index 000000000..4d4c25dc4 --- /dev/null +++ b/tests/github_api/get_repo_pytest.json @@ -0,0 +1,130 @@ +{ + "id": 62477336, + "node_id": "MDEwOlJlcG9zaXRvcnk2MjQ3NzMzNg==", + "name": "pytest-feedstock", + "full_name": "conda-forge/pytest-feedstock", + "private": false, + "owner": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/conda-forge/pytest-feedstock", + "description": "A conda-smithy repository for pytest.", + "fork": false, + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock", + "forks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/deployments", + "created_at": "2016-07-03T02:02:04Z", + "updated_at": "2024-05-20T16:07:25Z", + "pushed_at": "2024-05-20T16:07:21Z", + "git_url": "git://github.com/conda-forge/pytest-feedstock.git", + "ssh_url": "git@github.com:conda-forge/pytest-feedstock.git", + "clone_url": "https://github.com/conda-forge/pytest-feedstock.git", + "svn_url": "https://github.com/conda-forge/pytest-feedstock", + "homepage": null, + "size": 310, + "stargazers_count": 2, + "watchers_count": 2, + "language": null, + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 27, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 27, + "open_issues": 1, + "watchers": 2, + "default_branch": "main", + "temp_clone_token": null, + "custom_properties": {}, + "organization": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "network_count": 27, + "subscribers_count": 7 +} diff --git a/tests/github_api/github_response_headers.json b/tests/github_api/github_response_headers.json new file mode 100644 index 000000000..4ba208057 --- /dev/null +++ b/tests/github_api/github_response_headers.json @@ -0,0 +1,28 @@ +{ + "Server": "GitHub.com", + "Date": "Wed, 29 May 2024 12:07:38 GMT", + "Content-Type": "application/json; charset=utf-8", + "Cache-Control": "public, max-age=60, s-maxage=60", + "Vary": "Accept, Accept-Encoding, Accept, X-Requested-With", + "ETag": "W/\"7ba8c0b529b1303243a8c4636a95ce2e337591d152d69f8e90608c202a166483\"", + "Last-Modified": "Wed, 10 Apr 2024 13:15:22 GMT", + "X-GitHub-Media-Type": "github.v3; format=json", + "x-github-api-version-selected": "2022-11-28", + "Access-Control-Expose-Headers": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset", + "Access-Control-Allow-Origin": "*", + "Strict-Transport-Security": "max-age=31536000; includeSubdomains; preload", + "X-Frame-Options": "deny", + "X-Content-Type-Options": "nosniff", + "X-XSS-Protection": "0", + "Referrer-Policy": "origin-when-cross-origin, strict-origin-when-cross-origin", + "Content-Security-Policy": "default-src 'none'", + "Content-Encoding": "gzip", + "X-RateLimit-Limit": "60", + "X-RateLimit-Remaining": "51", + "X-RateLimit-Reset": "1716987025", + "X-RateLimit-Resource": "core", + "X-RateLimit-Used": "9", + "Accept-Ranges": "bytes", + "Content-Length": "4456", + "X-GitHub-Request-Id": "F7B3:2D05B1:3948EFC:3998343:74382A8F" +} diff --git a/tests/test_auto_tick.py b/tests/test_auto_tick.py new file mode 100644 index 000000000..c6b85c745 --- /dev/null +++ b/tests/test_auto_tick.py @@ -0,0 +1,292 @@ +import copy +import logging +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, create_autospec + +import pytest + +from conda_forge_tick.auto_tick import _commit_migration, _prepare_feedstock_repository +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext +from conda_forge_tick.git_utils import ( + DryRunBackend, + GitCli, + GitCliError, + GitPlatformBackend, + RepositoryNotFoundError, +) +from conda_forge_tick.migrators_types import AttrsTypedDict +from tests.fake_lazy_json import FakeLazyJson + +demo_attrs = AttrsTypedDict( + {"conda-forge.yml": {"provider": {"default_branch": "main"}, "bot": {}}} +) + + +def test_prepare_feedstock_repository_success(): + backend = create_autospec(GitPlatformBackend) + + with tempfile.TemporaryDirectory() as tmpdir_str: + tmpdir = Path(tmpdir_str) + cloned_context = ClonedFeedstockContext( + feedstock_name="pytest", + attrs=demo_attrs, + _default_branch="main", + local_clone_dir=tmpdir, + ) + + assert ( + _prepare_feedstock_repository( + backend, cloned_context, "new_branch", "base_branch" + ) + is True + ) + + backend.fork.assert_called_once_with("conda-forge", "pytest-feedstock") + + backend.clone_fork_and_branch.assert_called_once_with( + upstream_owner="conda-forge", + repo_name="pytest-feedstock", + target_dir=tmpdir, + new_branch="new_branch", + base_branch="base_branch", + ) + + +def test_prepare_feedstock_repository_repository_not_found(caplog): + backend = create_autospec(GitPlatformBackend) + + caplog.set_level(logging.WARNING) + + with tempfile.TemporaryDirectory() as tmpdir_str: + tmpdir = Path(tmpdir_str) + + demo_attrs_copy = copy.deepcopy(demo_attrs) + + attrs = FakeLazyJson() + pr_info = FakeLazyJson() + + with attrs: + for key, value in demo_attrs_copy.items(): + attrs[key] = value + attrs["pr_info"] = pr_info + + cloned_context = ClonedFeedstockContext( + feedstock_name="pytest", + attrs=attrs, # type: ignore + _default_branch="main", + local_clone_dir=tmpdir, + ) + + backend.fork.side_effect = RepositoryNotFoundError( + "Repository not found - MAGIC WORDS" + ) + + assert ( + _prepare_feedstock_repository( + backend, cloned_context, "new_branch", "base_branch" + ) + is False + ) + + backend.fork.assert_called_once_with("conda-forge", "pytest-feedstock") + + backend.clone_fork_and_branch.assert_not_called() + + assert "Not Found" in caplog.text + assert "pytest: Git repository not found." in attrs["pr_info"]["bad"] + + +def test_prepare_feedstock_repository_complete_dry_run(): + """ + This test really clones the repository using the DryRunBackend. + """ + + backend = DryRunBackend() + + context = FeedstockContext( + feedstock_name="pytest", + attrs=demo_attrs, + ) + + with context.reserve_clone_directory() as cloned_context: + assert ( + _prepare_feedstock_repository(backend, cloned_context, "new_branch", "main") + is True + ) + + assert cloned_context.local_clone_dir.joinpath("conda-forge.yml").exists() + + # new_branch should be checked out + assert ( + "new_branch" + in backend.cli._run_git_command( + ["status"], cloned_context.local_clone_dir + ).stdout + ) + + +def test_prepare_feedstock_repository_complete_fail(): + """ + This test really clones the repository using the DryRunBackend. + """ + + backend = DryRunBackend() + + context = FeedstockContext( + feedstock_name="this-repo-does-not-exist", + attrs=MagicMock(), + ) + + with context.reserve_clone_directory() as cloned_context: + assert ( + _prepare_feedstock_repository(backend, cloned_context, "new_branch", "main") + is False + ) + + +@pytest.mark.parametrize("raise_commit_errors", [True, False]) +@pytest.mark.parametrize("allow_empty_commits", [True, False]) +def test_commit_migration_nonempty( + raise_commit_errors: bool, allow_empty_commits: bool +): + backend = DryRunBackend() + + context = FeedstockContext( + feedstock_name="pytest", + attrs=demo_attrs, + ) + + with context.reserve_clone_directory() as cloned_context: + assert _prepare_feedstock_repository( + backend, cloned_context, "new_branch", "main" + ) + + # now we do our "migration" + cloned_context.local_clone_dir.joinpath("conda-forge.yml").unlink() + with cloned_context.local_clone_dir.joinpath("new_file.txt").open("w") as f: + f.write("Hello World!") + + _commit_migration( + backend.cli, + cloned_context, + "COMMIT_MESSAGE_1337", + allow_empty_commits, + raise_commit_errors, + ) + + # commit should be there + assert ( + "COMMIT_MESSAGE_1337" + in backend.cli._run_git_command( + ["log", "-1", "--pretty=%B"], cloned_context.local_clone_dir + ).stdout + ) + + +@pytest.mark.parametrize("raise_commit_errors", [True, False]) +@pytest.mark.parametrize("allow_empty_commits", [True, False]) +def test_commit_migration_empty(raise_commit_errors: bool, allow_empty_commits: bool): + backend = DryRunBackend() + + context = FeedstockContext( + feedstock_name="pytest", + attrs=demo_attrs, + ) + + with context.reserve_clone_directory() as cloned_context: + assert _prepare_feedstock_repository( + backend, cloned_context, "new_branch", "main" + ) + + if raise_commit_errors and not allow_empty_commits: + with pytest.raises(GitCliError): + _commit_migration( + backend.cli, + cloned_context, + "COMMIT_MESSAGE", + allow_empty_commits, + raise_commit_errors, + ) + return + else: + # everything should work normally + _commit_migration( + backend.cli, + cloned_context, + "COMMIT_MESSAGE_1337", + allow_empty_commits, + raise_commit_errors, + ) + + if not allow_empty_commits: + return + + # commit should be there + assert ( + "COMMIT_MESSAGE_1337" + in backend.cli._run_git_command( + ["log", "-1", "--pretty=%B"], cloned_context.local_clone_dir + ).stdout + ) + + +@pytest.mark.parametrize("raise_commit_errors", [True, False]) +@pytest.mark.parametrize("allow_empty_commits", [True, False]) +def test_commit_migration_mock_no_error( + allow_empty_commits: bool, raise_commit_errors: bool +): + cli = create_autospec(GitCli) + + clone_dir = Path("LOCAL_CLONE_DIR") + + context = ClonedFeedstockContext( + feedstock_name="pytest", attrs=demo_attrs, local_clone_dir=clone_dir + ) + + _commit_migration( + cli, context, "COMMIT MESSAGE 1337", allow_empty_commits, raise_commit_errors + ) + + cli.commit.assert_called_once_with( + clone_dir, "COMMIT MESSAGE 1337", allow_empty=allow_empty_commits + ) + + +@pytest.mark.parametrize("raise_commit_errors", [True, False]) +@pytest.mark.parametrize("allow_empty_commits", [True, False]) +def test_commit_migration_mock_error( + allow_empty_commits: bool, raise_commit_errors: bool +): + cli = create_autospec(GitCli) + + clone_dir = Path("LOCAL_CLONE_DIR") + + cli.commit.side_effect = GitCliError("Error") + + context = ClonedFeedstockContext( + feedstock_name="pytest", attrs=demo_attrs, local_clone_dir=clone_dir + ) + + if raise_commit_errors: + with pytest.raises(GitCliError): + _commit_migration( + cli, + context, + "COMMIT MESSAGE 1337", + allow_empty_commits, + raise_commit_errors, + ) + else: + _commit_migration( + cli, + context, + "COMMIT MESSAGE 1337", + allow_empty_commits, + raise_commit_errors, + ) + + cli.add.assert_called_once_with(clone_dir, all_=True) + cli.commit.assert_called_once_with( + clone_dir, "COMMIT MESSAGE 1337", allow_empty=allow_empty_commits + ) diff --git a/tests/test_contexts.py b/tests/test_contexts.py new file mode 100644 index 000000000..34b843642 --- /dev/null +++ b/tests/test_contexts.py @@ -0,0 +1,122 @@ +import pytest + +from conda_forge_tick.contexts import DEFAULT_BRANCHES, FeedstockContext +from conda_forge_tick.migrators_types import AttrsTypedDict + +# to make the typechecker happy, this satisfies the AttrsTypedDict type +demo_attrs = AttrsTypedDict( + {"conda-forge.yml": {"provider": {"default_branch": "main"}}} +) + +demo_attrs_automerge = AttrsTypedDict( + { + "conda-forge.yml": { + "provider": {"default_branch": "main"}, + "bot": {"automerge": True}, + } + } +) + +demo_attrs_check_solvable = AttrsTypedDict( + { + "conda-forge.yml": { + "provider": {"default_branch": "main"}, + "bot": {"check_solvable": True}, + } + } +) + + +def test_feedstock_context_default_branch_not_set(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.default_branch == "main" + + DEFAULT_BRANCHES["TEST-FEEDSTOCK-NAME"] = "develop" + assert context.default_branch == "develop" + + +def test_feedstock_context_default_branch_set(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs, "feature") + + DEFAULT_BRANCHES["TEST-FEEDSTOCK-NAME"] = "develop" + assert context.default_branch == "feature" + + # reset the default branches + DEFAULT_BRANCHES.pop("TEST-FEEDSTOCK-NAME") + + # test the default branch is still the same + assert context.default_branch == "feature" + + +def test_feedstock_context_git_repo_owner(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.git_repo_owner == "conda-forge" + + +def test_feedstock_context_git_repo_name(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.git_repo_name == "TEST-FEEDSTOCK-NAME-feedstock" + + +def test_feedstock_context_git_href(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert ( + context.git_href + == "https://github.com/conda-forge/TEST-FEEDSTOCK-NAME-feedstock" + ) + + +@pytest.mark.parametrize("automerge", [True, False]) +def test_feedstock_context_automerge(automerge: bool): + context = FeedstockContext( + "TEST-FEEDSTOCK-NAME", demo_attrs_automerge if automerge else demo_attrs + ) + + assert context.automerge == automerge + + +@pytest.mark.parametrize("check_solvable", [True, False]) +def test_feedstock_context_check_solvable(check_solvable: bool): + context = FeedstockContext( + "TEST-FEEDSTOCK-NAME", + demo_attrs_check_solvable if check_solvable else demo_attrs, + ) + + assert context.check_solvable == check_solvable + + +@pytest.mark.parametrize("default_branch", [None, "feature"]) +@pytest.mark.parametrize( + "attrs", [demo_attrs, demo_attrs_automerge, demo_attrs_check_solvable] +) +def test_feedstock_context_reserve_clone_directory( + attrs: AttrsTypedDict, default_branch: str +): + context = FeedstockContext("pytest", attrs, default_branch) + + with context.reserve_clone_directory() as cloned_context: + assert cloned_context.feedstock_name == "pytest" + assert cloned_context.attrs == attrs + assert ( + cloned_context.default_branch == default_branch + if default_branch + else "main" + ) + assert cloned_context.git_repo_owner == "conda-forge" + assert cloned_context.git_repo_name == "pytest-feedstock" + assert ( + cloned_context.git_href == "https://github.com/conda-forge/pytest-feedstock" + ) + assert cloned_context.automerge == context.automerge + assert cloned_context.check_solvable == context.check_solvable + + assert cloned_context.local_clone_dir.exists() + assert cloned_context.local_clone_dir.is_dir() + assert cloned_context.local_clone_dir.name == "pytest-feedstock" + + with open(cloned_context.local_clone_dir / "test.txt", "w") as f: + f.write("test") + + assert (cloned_context.local_clone_dir / "test.txt").exists() + + assert not cloned_context.local_clone_dir.exists() diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index ccaf05eb0..1198cad17 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1,3 +1,6 @@ +import datetime +import json +import logging import subprocess import tempfile from pathlib import Path @@ -6,18 +9,27 @@ import github3.exceptions import pytest +import requests +from pydantic_core import Url +from requests.structures import CaseInsensitiveDict from conda_forge_tick.git_utils import ( Bound, DryRunBackend, + DuplicatePullRequestError, GitCli, GitCliError, GitConnectionMode, GitHubBackend, GitPlatformBackend, + GitPlatformError, RepositoryNotFoundError, trim_pr_json_keys, ) +from conda_forge_tick.models.pr_json import ( + GithubPullRequestMergeableState, + PullRequestState, +) """ Note: You have to have git installed on your machine to run these tests. @@ -56,6 +68,187 @@ def test_git_cli_run_git_command_error(subprocess_run_mock: MagicMock): cli._run_git_command(["GIT_COMMAND"], working_directory) +@pytest.mark.parametrize("token_hidden", [True, False]) +@pytest.mark.parametrize("check_error", [True, False]) +@mock.patch("subprocess.run") +def test_git_cli_run_git_command_mock( + subprocess_run_mock: MagicMock, check_error: bool, token_hidden: bool +): + """ + This test checks if all parameters are passed correctly to the subprocess.run function. + """ + cli = GitCli() + + working_directory = Path("TEST_DIR") + + if token_hidden: + cli.add_hidden_token("TOKEN") + + cli._run_git_command(["COMMAND", "ARG1", "ARG2"], working_directory, check_error) + + stderr_args = {"stderr": subprocess.PIPE} if token_hidden else {} + + subprocess_run_mock.assert_called_once_with( + ["git", "COMMAND", "ARG1", "ARG2"], + check=check_error, + cwd=working_directory, + stdout=subprocess.PIPE, + **stderr_args, + text=True, + ) + + +@pytest.mark.parametrize("token_hidden", [True, False]) +@pytest.mark.parametrize("check_error", [True, False]) +def test_git_cli_run_git_command_stdout_captured( + capfd, check_error: bool, token_hidden: bool +): + """ + Verify that the stdout of the git command is captured and not printed to the console. + """ + cli = GitCli() + + if token_hidden: + cli.add_hidden_token("TOKEN") + p = cli._run_git_command(["version"], check_error=check_error) + + captured = capfd.readouterr() + + assert captured.out == "" + assert p.stdout.startswith("git version") + + +def test_git_cli_run_git_command_stderr_not_captured(capfd): + """ + Verify that the stderr of the git command is not captured if no token is hidden. + """ + cli = GitCli() + + p = cli._run_git_command(["non-existing-command"], check_error=False) + + captured = capfd.readouterr() + + assert captured.out == "" + assert "not a git command" in captured.err + assert p.stderr is None + + +def test_git_cli_hide_token_stdout_no_error(capfd): + cli = GitCli() + + cli.add_hidden_token("git") + p = cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert "git" not in captured.out + assert "git" not in captured.err + assert "git" not in p.stdout + assert "git" not in p.stderr + + assert p.stdout.count("***") > 5 + + +def test_git_cli_hide_token_stdout_error_check_error(caplog, capfd): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + cli.add_hidden_token("all") + with pytest.raises(GitCliError): + # git help --a prints to stdout (!) and then exits with an error + cli._run_git_command(["help", "--a"]) + + captured = capfd.readouterr() + + assert "all" not in captured.out + assert "all" not in captured.err + assert "all" not in caplog.text + + assert "***" in caplog.text + + +def test_git_cli_hide_token_stdout_error_no_check_error(caplog, capfd): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + cli.add_hidden_token("all") + p = cli._run_git_command(["help", "--a"], check_error=False) + + captured = capfd.readouterr() + + assert "all" not in captured.out + assert "all" not in captured.err + assert "all" not in p.stdout + assert "all" not in p.stderr + assert "all" not in caplog.text + + assert "***" in p.stdout + + +def test_git_cli_hide_token_stderr_no_check_error(capfd): + cli = GitCli() + + cli.add_hidden_token("command") + p = cli._run_git_command(["non-existing-command"], check_error=False) + + captured = capfd.readouterr() + + assert "command" not in captured.out + assert "command" not in captured.err + assert "command" not in p.stdout + assert "command" not in p.stderr + + assert p.stderr.count("*******") >= 2 + assert captured.err.count("*******") >= 2 + + +def test_git_cli_hide_token_run_git_command_check_error(capfd, caplog): + cli = GitCli() + + caplog.set_level(logging.INFO) + + cli.add_hidden_token("command") + with pytest.raises(GitCliError): + cli._run_git_command(["non-existing-command"]) + + print(caplog.text) + assert "Command 'git non-existing-command' failed." in caplog.text + assert ( + caplog.text.count("command") == 1 + ) # only the command itself is printed directly by us + + assert "'non-existing-*******' is not a git *******" in caplog.text + + +def test_git_cli_hide_token_multiple(capfd, caplog): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + cli.add_hidden_token("clone") + cli.add_hidden_token("commit") + p1 = cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert "clone" not in captured.out + assert "clone" not in captured.err + assert "clone" not in p1.stdout + assert "clone" not in p1.stderr + + assert "commit" not in captured.out + assert "commit" not in captured.err + assert "commit" not in p1.stdout + assert "commit" not in p1.stderr + + assert "clone" not in caplog.text + assert "commit" not in caplog.text + + assert p1.stdout.count("*****") >= 2 + + def test_git_cli_outside_repo(): with tempfile.TemporaryDirectory() as tmpdir: dir_path = Path(tmpdir) @@ -84,9 +277,10 @@ def test_git_cli_outside_repo(): # noinspection PyProtectedMember -def init_temp_git_repo(git_dir: Path): +def init_temp_git_repo(git_dir: Path, bare: bool = False): cli = GitCli() - cli._run_git_command(["init"], working_directory=git_dir) + bare_arg = ["--bare"] if bare else [] + cli._run_git_command(["init", *bare_arg, "-b", "main"], working_directory=git_dir) cli._run_git_command( ["config", "user.name", "CI Test User"], working_directory=git_dir ) @@ -96,6 +290,150 @@ def init_temp_git_repo(git_dir: Path): ) +@pytest.mark.parametrize( + "n_paths,all_", [(0, True), (1, False), (1, True), (2, False), (2, True)] +) +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_add_success_mock( + run_git_command_mock: MagicMock, n_paths: int, all_: bool +): + cli = GitCli() + + git_dir = Path("TEST_DIR") + paths = [Path(f"test{i}.txt") for i in range(n_paths)] + + cli.add(git_dir, *paths, all_=all_) + + expected_all_arg = ["--all"] if all_ else [] + + run_git_command_mock.assert_called_once_with( + ["add", *expected_all_arg, *paths], git_dir + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_add_no_arguments_error(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + with pytest.raises(ValueError, match="Either pathspec or all_ must be set"): + cli.add(git_dir) + + run_git_command_mock.assert_not_called() + + +@pytest.mark.parametrize( + "n_paths,all_", [(0, True), (1, False), (1, True), (2, False), (2, True)] +) +def test_git_cli_add_success(n_paths: int, all_: bool): + with tempfile.TemporaryDirectory() as tmp_dir: + git_dir = Path(tmp_dir) + init_temp_git_repo(git_dir) + + pathspec = [git_dir / f"test{i}.txt" for i in range(n_paths)] + + for path in pathspec + [git_dir / "all_tracker.txt"]: + path.touch() + + cli = GitCli() + cli.add(git_dir, *pathspec, all_=all_) + + tracked_files = cli._run_git_command(["ls-files", "-s"], git_dir).stdout + + for path in pathspec: + assert path.name in tracked_files + + if all_ and n_paths == 0: + # note that n_paths has to be zero to add unknown files to the working tree + assert "all_tracker.txt" in tracked_files + else: + assert "all_tracker.txt" not in tracked_files + + +@pytest.mark.parametrize("allow_empty", [True, False]) +@pytest.mark.parametrize("all_", [True, False]) +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_commit_success_mock( + run_git_command_mock: MagicMock, all_: bool, allow_empty: bool +): + git_dir = Path("GIT_DIR") + message = "COMMIT_MESSAGE" + + cli = GitCli() + cli.commit(git_dir, message, all_, allow_empty) + + expected_all_arg = ["-a"] if all_ else [] + expected_allow_empty_arg = ["--allow-empty"] if allow_empty else [] + + run_git_command_mock.assert_called_once_with( + ["commit", *expected_all_arg, *expected_allow_empty_arg, "-m", message], git_dir + ) + + +@pytest.mark.parametrize("allow_empty", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("all_", [True, False]) +def test_git_cli_commit(all_: bool, empty: bool, allow_empty: bool): + with tempfile.TemporaryDirectory() as tmp_dir: + git_dir = Path(tmp_dir) + init_temp_git_repo(git_dir) + + cli = GitCli() + + test_file = git_dir.joinpath("test.txt") + with test_file.open("w") as f: + f.write("Hello, World!") + cli.add(git_dir, git_dir / "test.txt") + cli.commit(git_dir, "Add Test") + + if not empty: + test_file.unlink() + if not all_: + cli.add(git_dir, git_dir / "test.txt") + + if empty and not allow_empty: + with pytest.raises(GitCliError): + cli.commit(git_dir, "Add Test", all_, allow_empty) + return + + cli.commit(git_dir, "Add Test", all_, allow_empty) + + git_log = cli._run_git_command(["log"], git_dir).stdout + + assert "Add Test" in git_log + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_rev_parse_head_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + run_git_command_mock.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeef\n" + ) + + head_rev = cli.rev_parse_head(git_dir) + run_git_command_mock.assert_called_once_with( + ["rev-parse", "HEAD"], git_dir, capture_text=True + ) + + assert head_rev == "deadbeef" + + +def test_git_cli_rev_parse_head(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + init_temp_git_repo(dir_path) + cli.commit(dir_path, "Initial commit", allow_empty=True) + head_rev = cli.rev_parse_head(dir_path) + assert len(head_rev) == 40 + assert all(c in "0123456789abcdef" for c in head_rev) + + def test_git_cli_reset_hard_already_reset(): cli = GitCli() with tempfile.TemporaryDirectory() as tmpdir: @@ -278,6 +616,68 @@ def test_git_cli_add_remote(): assert remote_url in output.stdout.decode() +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_push_to_url_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + remote_url = "https://git-repository.com/repo.git" + + cli.push_to_url(git_dir, remote_url, "BRANCH_NAME") + + run_git_command_mock.assert_called_once_with( + ["push", remote_url, "BRANCH_NAME"], git_dir + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_push_to_url_mock_error(run_git_command_mock: MagicMock): + cli = GitCli() + + run_git_command_mock.side_effect = GitCliError("Error") + + with pytest.raises(GitCliError): + cli.push_to_url( + Path("TEST_DIR"), "https://git-repository.com/repo.git", "BRANCH_NAME" + ) + + +def test_git_cli_push_to_url_local_repository(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + source_repo = dir_path / "source_repo" + source_repo.mkdir() + init_temp_git_repo(source_repo, bare=True) + + local_repo = dir_path / "local_repo" + local_repo.mkdir() + cli._run_git_command(["clone", source_repo.resolve(), local_repo]) + + # remove all references to the original repo + cli._run_git_command( + ["remote", "remove", "origin"], working_directory=local_repo + ) + + with local_repo.joinpath("test.txt").open("w") as f: + f.write("Hello, World!") + + cli._run_git_command(["add", "test.txt"], working_directory=local_repo) + cli._run_git_command( + ["commit", "-am", "Add test.txt"], working_directory=local_repo + ) + + cli.push_to_url(local_repo, str(source_repo.resolve()), "main") + + source_git_log = subprocess.run( + "git log", cwd=source_repo, shell=True, capture_output=True + ).stdout.decode() + + assert "test.txt" in source_git_log + + @mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") def test_git_cli_fetch_all_mock(run_git_command_mock: MagicMock): cli = GitCli() @@ -434,6 +834,62 @@ def test_git_cli_checkout_branch_no_track(): ) +def test_git_cli_diffed_files(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + cli.commit(dir_path, "Initial commit", allow_empty=True) + dir_path.joinpath("test.txt").touch() + cli.add(dir_path, dir_path / "test.txt") + cli.commit(dir_path, "Add test.txt") + + diffed_files = list(cli.diffed_files(dir_path, "HEAD~1")) + + assert (dir_path / "test.txt") in diffed_files + assert len(diffed_files) == 1 + + +def test_git_cli_diffed_files_no_diff(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + cli.commit(dir_path, "Initial commit", allow_empty=True) + + diffed_files = list(cli.diffed_files(dir_path, "HEAD")) + + assert len(diffed_files) == 0 + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_diffed_files_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + commit = "COMMIT" + + run_git_command_mock.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="test.txt\n" + ) + + diffed_files = list(cli.diffed_files(git_dir, commit)) + + run_git_command_mock.assert_called_once_with( + ["diff", "--name-only", "--relative", commit, "HEAD"], + git_dir, + capture_text=True, + ) + + assert diffed_files == [git_dir / "test.txt"] + + def test_git_cli_clone_fork_and_branch_minimal(): fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" @@ -488,7 +944,7 @@ def test_git_cli_clone_fork_and_branch_mock( fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) cli = GitCli() @@ -577,7 +1033,7 @@ def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(c new_branch = "NEW_BRANCH" cli = GitCli() - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) with tempfile.TemporaryDirectory() as tmpdir: dir_path = Path(tmpdir) / "duckdb-feedstock" @@ -589,13 +1045,39 @@ def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(c assert "trying to reset hard" in caplog.text -def test_git_platform_backend_get_remote_url_https(): - owner = "OWNER" - repo = "REPO" +def _github_api_json_fixture(name: str) -> dict: + with Path(__file__).parent.joinpath(f"github_api/{name}.json").open() as f: + return json.load(f) - url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS) - assert url == f"https://github.com/{owner}/{repo}.git" +@pytest.fixture() +def github_response_create_issue_comment() -> dict: + return _github_api_json_fixture("create_issue_comment_pytest") + + +@pytest.fixture() +def github_response_create_pull_duplicate() -> dict: + return _github_api_json_fixture("create_pull_duplicate") + + +@pytest.fixture() +def github_response_create_pull_validation_error() -> dict: + return _github_api_json_fixture("create_pull_validation_error") + + +@pytest.fixture() +def github_response_get_pull() -> dict: + return _github_api_json_fixture("get_pull_pytest") + + +@pytest.fixture() +def github_response_get_repo() -> dict: + return _github_api_json_fixture("get_repo_pytest") + + +@pytest.fixture() +def github_response_headers() -> dict: + return _github_api_json_fixture("github_response_headers") def test_github_backend_from_token(): @@ -607,11 +1089,33 @@ def test_github_backend_from_token(): # we cannot verify the pygithub token trivially +@pytest.mark.parametrize("from_token", [True, False]) +def test_github_backend_token_to_hide(caplog, capfd, from_token: bool): + caplog.set_level(logging.DEBUG) + token = "commit" + + if from_token: + backend = GitHubBackend.from_token(token) + else: + backend = GitHubBackend(MagicMock(), MagicMock(), token) + + # the token should be hidden by default, without any context manager + p = backend.cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert token not in caplog.text + assert token not in captured.out + assert token not in captured.err + assert token not in p.stdout + assert token not in p.stderr + + @pytest.mark.parametrize("does_exist", [True, False]) def test_github_backend_does_repository_exist(does_exist: bool): github3_client = MagicMock() - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") github3_client.repository.return_value = MagicMock() if does_exist else None @@ -619,6 +1123,27 @@ def test_github_backend_does_repository_exist(does_exist: bool): github3_client.repository.assert_called_once_with("OWNER", "REPO") +def test_github_backend_get_remote_url_https(): + owner = "OWNER" + repo = "REPO" + backend = GitHubBackend(MagicMock(), MagicMock(), "") + + url = backend.get_remote_url(owner, repo, GitConnectionMode.HTTPS) + + assert url == f"https://github.com/{owner}/{repo}.git" + + +def test_github_backend_get_remote_url_token(): + owner = "OWNER" + repo = "REPO" + token = "TOKEN" + backend = GitHubBackend(MagicMock(), MagicMock(), "") + + url = backend.get_remote_url(owner, repo, GitConnectionMode.HTTPS, token) + + assert url == f"https://{token}@github.com/{owner}/{repo}.git" + + @mock.patch("time.sleep", return_value=None) @mock.patch( "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock @@ -633,7 +1158,7 @@ def test_github_backend_fork_not_exists_repo_found( repository = MagicMock() github3_client.repository.return_value = repository - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") user_mock.return_value = "USER" backend.fork("UPSTREAM-OWNER", "REPO") @@ -643,6 +1168,21 @@ def test_github_backend_fork_not_exists_repo_found( sleep_mock.assert_called_once_with(5) +@mock.patch("conda_forge_tick.git_utils.GitCli.push_to_url") +def test_github_backend_push_to_repository(push_to_url_mock: MagicMock): + backend = GitHubBackend.from_token("THIS_IS_THE_TOKEN") + + git_dir = Path("GIT_DIR") + + backend.push_to_repository("OWNER", "REPO", git_dir, "BRANCH_NAME") + + push_to_url_mock.assert_called_once_with( + git_dir, + "https://THIS_IS_THE_TOKEN@github.com/OWNER/REPO.git", + "BRANCH_NAME", + ) + + @pytest.mark.parametrize("branch_already_synced", [True, False]) @mock.patch("time.sleep", return_value=None) @mock.patch( @@ -656,7 +1196,7 @@ def test_github_backend_fork_exists( branch_already_synced: bool, caplog, ): - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) exists_mock.return_value = True user_mock.return_value = "USER" @@ -681,7 +1221,7 @@ def get_repo(full_name: str): upstream_repo.default_branch = "UPSTREAM_BRANCH_NAME" fork_repo.default_branch = "FORK_BRANCH_NAME" - backend = GitHubBackend(MagicMock(), pygithub_client) + backend = GitHubBackend(MagicMock(), pygithub_client, "") backend.fork("UPSTREAM-OWNER", "REPO") if not branch_already_synced: @@ -704,7 +1244,7 @@ def test_github_backend_remote_does_not_exist( github3_client = MagicMock() github3_client.repository.return_value = None - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") user_mock.return_value = "USER" @@ -721,7 +1261,7 @@ def test_github_backend_user(): user.login = "USER" pygithub_client.get_user.return_value = user - backend = GitHubBackend(MagicMock(), pygithub_client) + backend = GitHubBackend(MagicMock(), pygithub_client, "") for _ in range(4): # cached property @@ -736,7 +1276,7 @@ def test_github_backend_get_api_requests_left_github_exception(caplog): "API Error" ) - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() is None assert "API error while fetching" in caplog.text @@ -748,7 +1288,7 @@ def test_github_backend_get_api_requests_left_unexpected_response_schema(caplog) github3_client = MagicMock() github3_client.rate_limit.return_value = {"some": "gibberish data"} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() is None assert "API Error while parsing" @@ -760,7 +1300,7 @@ def test_github_backend_get_api_requests_left_nonzero(): github3_client = MagicMock() github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 5}}} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 5 @@ -772,7 +1312,7 @@ def test_github_backend_get_api_requests_left_zero_invalid_reset_time(caplog): github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 0}}} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 0 @@ -792,16 +1332,329 @@ def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): "resources": {"core": {"remaining": 0, "reset": reset_timestamp}} } - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 0 github3_client.rate_limit.assert_called_once() - assert f"will reset at {reset_timestamp_str}" in caplog.text # + assert f"will reset at {reset_timestamp_str}" in caplog.text + + +@mock.patch("requests.Session.request") +def test_github_backend_create_pull_request_mock( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_headers: dict, + github_response_get_pull: dict, +): + def request_side_effect(method, _url, **_kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if method == "POST": + response.status_code = 201 + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_get_pull + response.headers = CaseInsensitiveDict(github_response_headers) + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") + + pr_data = backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) + + request_mock.assert_called_with( + "POST", + "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls", + data='{"title": "TITLE", "body": "BODY", "base": "BASE_BRANCH", "head": "CURRENT_USER:HEAD_BRANCH"}', + json=None, + timeout=mock.ANY, + ) + + assert pr_data.base is not None + assert pr_data.base.repo.name == "pytest-feedstock" + assert pr_data.closed_at is None + assert pr_data.created_at is not None + assert pr_data.created_at == datetime.datetime( + 2024, 5, 3, 17, 4, 20, tzinfo=datetime.timezone.utc + ) + assert pr_data.head is not None + assert pr_data.head.ref == "HEAD_BRANCH" + assert pr_data.html_url == Url( + "https://github.com/conda-forge/pytest-feedstock/pull/1337" + ) + assert pr_data.id == 1853804278 + assert pr_data.labels == [] + assert pr_data.mergeable is True + assert pr_data.mergeable_state == GithubPullRequestMergeableState.CLEAN + assert pr_data.merged is False + assert pr_data.merged_at is None + assert pr_data.number == 1337 + assert pr_data.state == PullRequestState.OPEN + assert pr_data.updated_at == datetime.datetime( + 2024, 5, 27, 13, 31, 50, tzinfo=datetime.timezone.utc + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_create_pull_request_duplicate( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_create_pull_duplicate: dict, +): + def request_side_effect(method, _url, **_kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if method == "POST": + response.status_code = 422 + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_create_pull_duplicate + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") + + with pytest.raises( + DuplicatePullRequestError, + match="Pull request from CURRENT_USER:HEAD_BRANCH to conda-forge:BASE_BRANCH already exists", + ): + backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_create_pull_request_validation_error( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_create_pull_validation_error: dict, +): + """ + Test that other GitHub API 422 validation errors are not caught as DuplicatePullRequestError. + """ + + def request_side_effect(method, _url, **_kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if method == "POST": + response.status_code = 422 + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_create_pull_validation_error + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") + + with pytest.raises(github3.exceptions.UnprocessableEntity): + backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_success( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_get_pull: dict, + github_response_create_issue_comment: dict, +): + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 200 + response.json = lambda: github_response_get_pull + return response + if ( + method == "POST" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" + ): + response.status_code = 201 + response.json = lambda: github_response_create_issue_comment + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") + + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + request_mock.assert_called_with( + "POST", + "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments", + data='{"body": "COMMENT"}', + json=None, + timeout=mock.ANY, + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_repo_not_found(request_mock: MagicMock): + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 404 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") + + with pytest.raises(RepositoryNotFoundError): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_pull_request_not_found( + request_mock: MagicMock, + github_response_get_repo: dict, +): + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 404 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") + + with pytest.raises( + GitPlatformError, + match="Pull request conda-forge/pytest-feedstock#1337 not found", + ): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_unexpected_response( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_get_pull: dict, +): + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 200 + response.json = lambda: github_response_get_pull + return response + if ( + method == "POST" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" + ): + response.status_code = 500 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") + + with pytest.raises(GitPlatformError, match="Could not comment on pull request"): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) @pytest.mark.parametrize( - "backend", [GitHubBackend(MagicMock(), MagicMock()), DryRunBackend()] + "backend", [GitHubBackend(MagicMock(), MagicMock(), ""), DryRunBackend()] ) @mock.patch( "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock @@ -820,7 +1673,7 @@ def test_git_platform_backend_clone_fork_and_branch( user_mock.return_value = "USER" - backend = GitHubBackend(MagicMock(), MagicMock()) + backend = GitHubBackend(MagicMock(), MagicMock(), "") backend.clone_fork_and_branch( upstream_owner, repo_name, target_dir, new_branch, base_branch ) @@ -857,26 +1710,81 @@ def test_dry_run_backend_does_repository_exist_other_repo(): ) -def test_dry_run_backend_fork(caplog): - caplog.set_level("DEBUG") +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_non_fork(token: str | None): + backend = DryRunBackend() + + url = backend.get_remote_url("OWNER", "REPO", GitConnectionMode.HTTPS, token) + + if token is None: + assert url == "https://github.com/OWNER/REPO.git" + else: + assert url == "https://TOKEN@github.com/OWNER/REPO.git" + +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_non_existing_fork(token: str | None): backend = DryRunBackend() - backend.fork("UPSTREAM_OWNER", "REPO") + with pytest.raises(RepositoryNotFoundError, match="does not exist"): + backend.get_remote_url(backend.user, "REPO", GitConnectionMode.HTTPS, token) + + backend.fork("UPSTREAM_OWNER", "REPO2") + + with pytest.raises(RepositoryNotFoundError, match="does not exist"): + backend.get_remote_url(backend.user, "REPO", GitConnectionMode.HTTPS, token) + + +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_existing_fork(token: str | None): + backend = DryRunBackend() + + backend.fork("UPSTREAM_OWNER", "pytest-feedstock") + + url = backend.get_remote_url( + backend.user, "pytest-feedstock", GitConnectionMode.HTTPS, token + ) + + # note that the URL does not indicate anymore that it is a fork assert ( - "Dry Run: Creating fork of UPSTREAM_OWNER/REPO for user auto-tick-bot-dry-run" + url + == f"https://{f'{token}@' if token else ''}github.com/UPSTREAM_OWNER/pytest-feedstock.git" + ) + + +def test_dry_run_backend_push_to_repository(caplog): + caplog.set_level(logging.DEBUG) + + backend = DryRunBackend() + + git_dir = Path("GIT_DIR") + + backend.push_to_repository("OWNER", "REPO", git_dir, "BRANCH_NAME") + + assert ( + "Dry Run: Pushing changes from GIT_DIR to OWNER/REPO on branch BRANCH_NAME" in caplog.text ) - with pytest.raises(ValueError, match="Fork of REPO already exists"): - backend.fork("UPSTREAM_OWNER", "REPO") - with pytest.raises(ValueError, match="Fork of REPO already exists"): - backend.fork("OTHER_OWNER", "REPO") +def test_dry_run_backend_fork(caplog): + caplog.set_level(logging.DEBUG) + + backend = DryRunBackend() + + backend.fork("conda-forge", "pytest-feedstock") + assert ( + "Dry Run: Creating fork of conda-forge/pytest-feedstock for user auto-tick-bot-dry-run" + in caplog.text + ) + + # this should not raise an error + with pytest.raises(RepositoryNotFoundError): + backend.fork("conda-forge", "this-repository-does-not-exist") def test_dry_run_backend_sync_default_branch(caplog): - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) backend = DryRunBackend() @@ -891,6 +1799,59 @@ def test_dry_run_backend_user(): assert backend.user == "auto-tick-bot-dry-run" +def test_dry_run_backend_create_pull_request(caplog): + backend = DryRunBackend() + caplog.set_level(logging.DEBUG) + + pr_data = backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY_TEXT", + ) + + # caplog validation + assert "Create Pull Request" in caplog.text + assert 'Title: "TITLE"' in caplog.text + assert "Target Repository: conda-forge/pytest-feedstock" in caplog.text + assert ( + f"Branches: {backend.user}:HEAD_BRANCH -> conda-forge:BASE_BRANCH" + in caplog.text + ) + assert "Body:\nBODY_TEXT" in caplog.text + + # pr_data validation + assert pr_data.e_tag == "GITHUB_PR_ETAG" + assert pr_data.last_modified is not None + assert pr_data.id == 13371337 + assert pr_data.html_url == Url( + "https://github.com/conda-forge/pytest-feedstock/pulls/1337" + ) + assert pr_data.created_at is not None + assert pr_data.number == 1337 + assert pr_data.state == PullRequestState.OPEN + assert pr_data.head.ref == "HEAD_BRANCH" + assert pr_data.base.repo.name == "pytest-feedstock" + + +def test_dry_run_backend_comment_on_pull_request(caplog): + backend = DryRunBackend() + caplog.set_level(logging.DEBUG) + + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + assert "Comment on Pull Request" in caplog.text + assert "Comment:\nCOMMENT" in caplog.text + assert "Pull Request: conda-forge/pytest-feedstock#1337" in caplog.text + + def test_trim_pr_json_keys(): pr_json = { "ETag": "blah", diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 05bd82355..eea57195c 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -518,7 +518,6 @@ def run_test_migration( feedstock_name=name, attrs=pmy, ) - fctx.feedstock_dir = os.path.dirname(tmpdir) m.effective_graph.add_node(name) m.effective_graph.nodes[name]["payload"] = MockLazyJson({}) m.pr_body(fctx)