From 1ebe094a9619cb276f9463e56f590b934ea7e223 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 13 Jun 2024 19:23:32 +0200 Subject: [PATCH 01/38] Reapply "Readd git backend" This reverts commit 311a442c6caf7b70891eb78969eb33f9d2402895. --- conda_forge_tick/auto_tick.py | 18 +- conda_forge_tick/executors.py | 48 +- conda_forge_tick/git_utils.py | 822 +++++++++++++++++------ conda_forge_tick/lazy_json_backends.py | 5 +- conda_forge_tick/update_prs.py | 3 +- tests/test_executors.py | 27 +- tests/test_git_utils.py | 892 ++++++++++++++++++++++++- 7 files changed, 1563 insertions(+), 252 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 6ea8a2ead..e1da77301 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -30,8 +30,8 @@ from conda_forge_tick.git_utils import ( GIT_CLONE_DIR, comment_on_pr, - get_github_api_requests_left, get_repo, + github_backend, is_github_api_limit_reached, push_repo, ) @@ -195,13 +195,7 @@ def run( # TODO: run this in parallel feedstock_dir, repo = get_repo( - fctx=feedstock_ctx, - branch=branch_name, - feedstock=feedstock_ctx.feedstock_name, - protocol=protocol, - pull_request=pull_request, - fork=fork, - base_branch=base_branch, + fctx=feedstock_ctx, branch=branch_name, base_branch=base_branch ) if not feedstock_dir or not repo: logger.critical( @@ -618,7 +612,8 @@ def _run_migrator_on_feedstock_branch( fctx.feedstock_name, ) - if is_github_api_limit_reached(e): + if is_github_api_limit_reached(): + logger.warning("GitHub API error", exc_info=e) break_loop = True except VersionMigrationError as e: @@ -699,7 +694,8 @@ def _run_migrator_on_feedstock_branch( def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): curr_time = time.time() - api_req = get_github_api_requests_left() + backend = github_backend() + api_req = backend.get_api_requests_left() if curr_time - START_TIME > TIMEOUT: logger.info( @@ -1131,5 +1127,5 @@ def main(ctx: CliContext) -> None: # ], # ) - logger.info("API Calls Remaining: %d", get_github_api_requests_left()) + logger.info("API Calls Remaining: %d", github_backend().get_api_requests_left()) logger.info("Done") diff --git a/conda_forge_tick/executors.py b/conda_forge_tick/executors.py index e11e08d03..5623e808d 100644 --- a/conda_forge_tick/executors.py +++ b/conda_forge_tick/executors.py @@ -16,9 +16,21 @@ def __exit__(self, *args, **kwargs): pass -TRLOCK = TRLock() -PRLOCK = DummyLock() -DRLOCK = DummyLock() +GIT_LOCK_THREAD = TRLock() +GIT_LOCK_PROCESS = DummyLock() +GIT_LOCK_DASK = DummyLock() + + +@contextlib.contextmanager +def lock_git_operation(): + """ + A context manager to lock git operations - it can be acquired once per thread, once per process, + and once per dask worker. + Note that this is a reentrant lock, so it can be acquired multiple times by the same thread/process/worker. + """ + + with GIT_LOCK_THREAD, GIT_LOCK_PROCESS, GIT_LOCK_DASK: + yield logger = logging.getLogger(__name__) @@ -27,10 +39,12 @@ def __exit__(self, *args, **kwargs): class DaskRLock(DaskLock): """A reentrant lock for dask that is always blocking and never times out.""" - def acquire(self): - if not hasattr(self, "_rcount"): - self._rcount = 0 + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._rcount = 0 + self._rdata = None + def acquire(self, *args): self._rcount += 1 if self._rcount == 1: @@ -39,29 +53,29 @@ def acquire(self): return self._rdata def release(self): - if not hasattr(self, "_rcount") or self._rcount == 0: + if self._rcount == 0: raise RuntimeError("Lock not acquired so cannot be released!") self._rcount -= 1 if self._rcount == 0: - delattr(self, "_rdata") + self._rdata = None return super().release() else: return None def _init_process(lock): - global PRLOCK - PRLOCK = lock + global GIT_LOCK_PROCESS + GIT_LOCK_PROCESS = lock def _init_dask(lock): - global DRLOCK - # it appears we have to construct the locak by name instead + global GIT_LOCK_DASK + # it appears we have to construct the lock by name instead # of passing the object itself # otherwise dask uses a regular lock - DRLOCK = DaskRLock(name=lock) + GIT_LOCK_DASK = DaskRLock(name=lock) @contextlib.contextmanager @@ -70,8 +84,8 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut This allows us to easily use other executors as needed. """ - global DRLOCK - global PRLOCK + global GIT_LOCK_DASK + global GIT_LOCK_PROCESS if kind == "thread": with ThreadPoolExecutor(max_workers=max_workers) as pool_t: @@ -85,7 +99,7 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut initargs=(lock,), ) as pool_p: yield pool_p - PRLOCK = DummyLock() + GIT_LOCK_PROCESS = DummyLock() elif kind in ["dask", "dask-process", "dask-thread"]: import dask import distributed @@ -101,6 +115,6 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut with distributed.Client(cluster) as client: client.run(_init_dask, "cftick") yield ClientExecutor(client) - DRLOCK = DummyLock() + GIT_LOCK_DASK = DummyLock() else: raise NotImplementedError("That kind is not implemented") diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index bded19127..87395c0f0 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1,14 +1,17 @@ """Utilities for managing github repos""" import copy -import datetime +import enum import logging -import os +import math import subprocess -import sys import threading import time -from typing import Dict, Optional, Tuple, Union +from abc import ABC, abstractmethod +from datetime import datetime +from functools import cached_property +from pathlib import Path +from typing import Dict, Literal, Optional, Tuple, Union import backoff import github @@ -26,6 +29,7 @@ from conda_forge_tick.lazy_json_backends import LazyJson from .contexts import FeedstockContext +from .executors import lock_git_operation from .os_utils import pushd from .utils import get_bot_run_url, run_command_hiding_token @@ -71,6 +75,9 @@ def github3_client() -> github3.GitHub: + """ + This will be removed in the future, use the GitHubBackend class instead. + """ if not hasattr(GITHUB3_CLIENT, "client"): with sensitive_env() as env: GITHUB3_CLIENT.client = github3.login(token=env["BOT_TOKEN"]) @@ -78,6 +85,9 @@ def github3_client() -> github3.GitHub: def github_client() -> github.Github: + """ + This will be removed in the future, use the GitHubBackend class instead. + """ if not hasattr(GITHUB_CLIENT, "client"): with sensitive_env() as env: GITHUB_CLIENT.client = github.Github( @@ -87,77 +97,586 @@ def github_client() -> github.Github: return GITHUB_CLIENT.client -def get_default_branch(feedstock_name): - """Get the default branch for a feedstock +class Bound(float, enum.Enum): + def __str__(self): + return str(self.value) - Parameters - ---------- - feedstock_name : str - The feedstock without '-feedstock'. + INFINITY = math.inf + """ + Python does not have support for a literal infinity type, so we use this enum for it. + """ - Returns - ------- - branch : str - The default branch (e.g., 'main'). + +class GitConnectionMode(enum.StrEnum): + """ + We don't need anything else than HTTPS for now, but this would be the place to + add more connection modes (e.g. SSH). """ - return ( - github_client() - .get_repo(f"conda-forge/{feedstock_name}-feedstock") - .default_branch - ) + HTTPS = "https" -def get_github_api_requests_left() -> Union[int, None]: - """Get the number of remaining GitHub API requests. - Returns - ------- - left : int or None - The number of remaining requests, or None if there is an exception. +class GitCliError(Exception): + pass + + +class RepositoryNotFoundError(Exception): + """ + Raised when a repository is not found. """ - gh = github3_client() - try: - left = gh.rate_limit()["resources"]["core"]["remaining"] - except Exception: - left = None - return left + pass -def is_github_api_limit_reached( - e: Union[github3.GitHubError, github.GithubException], -) -> bool: - """Prints diagnostic information about a github exception. +class GitCli: + """ + A simple wrapper around the git command line interface. - Parameters - ---------- - e - The exception to check. + Git operations are locked (globally) to prevent operations from interfering with each other. + If this does impact performance too much, we can consider a per-repository locking strategy. + """ - Returns - ------- - out_of_api_calls - A flag to indicate that the api call limit has been reached. + @staticmethod + @lock_git_operation() + def _run_git_command( + cmd: list[str | Path], + working_directory: Path | None = None, + check_error: bool = True, + capture_text: bool = False, + ) -> subprocess.CompletedProcess: + """ + Run a git command. + :param cmd: The command to run, as a list of strings. + :param working_directory: The directory to run the command in. If None, the command will be run in the current + working directory. + :param check_error: If True, raise a GitCliError if the git command fails. + :param capture_text: If True, capture the output of the git command as text. The output will be in the + returned result object. + :return: The result of the git command. + :raises GitCliError: If the git command fails and check_error is True. + :raises FileNotFoundError: If the working directory does not exist. + """ + git_command = ["git"] + cmd + + logger.debug(f"Running git command: {git_command}") + capture_args = {"capture_output": True, "text": True} if capture_text else {} + + try: + return subprocess.run( + git_command, check=check_error, cwd=working_directory, **capture_args + ) + except subprocess.CalledProcessError as e: + raise GitCliError("Error running git command.") from e + + @lock_git_operation() + def reset_hard(self, git_dir: Path, to_treeish: str = "HEAD"): + """ + Reset the git index of a directory to the state of the last commit with `git reset --hard HEAD`. + :param git_dir: The directory to reset. + :param to_treeish: The treeish to reset to. Defaults to "HEAD". + :raises GitCliError: If the git command fails. + :raises FileNotFoundError: If the git_dir does not exist. + """ + self._run_git_command(["reset", "--quiet", "--hard", to_treeish], git_dir) + + @lock_git_operation() + def clone_repo(self, origin_url: str, target_dir: Path): + """ + Clone a Git repository. + If target_dir exists and is non-empty, this method will fail with GitCliError. + If target_dir exists and is empty, it will work. + If target_dir does not exist, it will work. + :param target_dir: The directory to clone the repository into. + :param origin_url: The URL of the repository to clone. + :raises GitCliError: If the git command fails (e.g. because origin_url does not point to valid remote or + target_dir is not empty). + """ + try: + self._run_git_command(["clone", "--quiet", origin_url, target_dir]) + except GitCliError as e: + raise GitCliError( + f"Error cloning repository from {origin_url}. Does the repository exist? Is target_dir empty?" + ) from e + + @lock_git_operation() + def add_remote(self, git_dir: Path, remote_name: str, remote_url: str): + """ + Add a remote to a git repository. + :param remote_name: The name of the remote. + :param remote_url: The URL of the remote. + :param git_dir: The directory of the git repository. + :raises GitCliError: If the git command fails (e.g., the remote already exists). + :raises FileNotFoundError: If git_dir does not exist + """ + self._run_git_command(["remote", "add", remote_name, remote_url], git_dir) + + @lock_git_operation() + def fetch_all(self, git_dir: Path): + """ + Fetch all changes from all remotes. + :param git_dir: The directory of the git repository. + :raises GitCliError: If the git command fails. + :raises FileNotFoundError: If git_dir does not exist + """ + self._run_git_command(["fetch", "--all", "--quiet"], git_dir) + + def does_branch_exist(self, git_dir: Path, branch_name: str): + """ + Check if a branch exists in a git repository. + Note: If git_dir is not a git repository, this method will return False. + Note: This method is intentionally not locked with lock_git_operation, as it only reads the git repository and + does not modify it. + :param branch_name: The name of the branch. + :param git_dir: The directory of the git repository. + :return: True if the branch exists, False otherwise. + :raises GitCliError: If the git command fails. + :raises FileNotFoundError: If git_dir does not exist + """ + ret = self._run_git_command( + ["show-ref", "--verify", "--quiet", f"refs/heads/{branch_name}"], + git_dir, + check_error=False, + ) + + return ret.returncode == 0 + + def does_remote_exist(self, remote_url: str) -> bool: + """ + Check if a remote exists. + Note: This method is intentionally not locked with lock_git_operation, as it only reads a remote and does not + modify a git repository. + :param remote_url: The URL of the remote. + :return: True if the remote exists, False otherwise. + """ + ret = self._run_git_command(["ls-remote", remote_url], check_error=False) + + return ret.returncode == 0 + + @lock_git_operation() + def checkout_branch( + self, + git_dir: Path, + branch: str, + track: bool = False, + ): + """ + Checkout a branch in a git repository. + :param git_dir: The directory of the git repository. + :param branch: The branch to check out. + :param track: If True, set the branch to track the remote branch with the same name (sets the --track flag). + A new local branch will be created with the name inferred from branch. + For example, if branch is "upstream/main", the new branch will be "main". + :raises GitCliError: If the git command fails. + :raises FileNotFoundError: If git_dir does not exist + """ + track_flag = ["--track"] if track else [] + self._run_git_command( + ["checkout", "--quiet"] + track_flag + [branch], + git_dir, + ) + + @lock_git_operation() + def checkout_new_branch( + self, git_dir: Path, branch: str, start_point: str | None = None + ): + """ + Checkout a new branch in a git repository. + :param git_dir: The directory of the git repository. + :param branch: The name of the new branch. + :param start_point: The name of the branch to branch from, or None to branch from the current branch. + :raises FileNotFoundError: If git_dir does not exist + """ + start_point_option = [start_point] if start_point else [] + + self._run_git_command( + ["checkout", "--quiet", "-b", branch] + start_point_option, git_dir + ) + + @lock_git_operation() + def clone_fork_and_branch( + self, + origin_url: str, + target_dir: Path, + upstream_url: str, + new_branch: str, + base_branch: str = "main", + ): + """ + Convenience method to do the following: + 1. Clone the repository at origin_url into target_dir (resetting the directory if it already exists). + 2. Add a remote named "upstream" with the URL upstream_url (ignoring if it already exists). + 3. Fetch all changes from all remotes. + 4. Checkout the base branch. + 5. Create a new branch from the base branch with the name new_branch. + + This is usually used to create a new branch for a pull request. In this case, origin_url is the URL of the + user's fork, and upstream_url is the URL of the upstream repository. + + :param origin_url: The URL of the repository (fork) to clone. + :param target_dir: The directory to clone the repository into. + :param upstream_url: The URL of the upstream repository. + :param new_branch: The name of the branch to create. + :param base_branch: The name of the base branch to branch from. + + :raises GitCliError: If a git command fails. + """ + try: + self.clone_repo(origin_url, target_dir) + except GitCliError: + if not target_dir.exists(): + raise GitCliError( + f"Could not clone {origin_url} - does the remote exist?" + ) + logger.debug( + f"Cloning {origin_url} into {target_dir} was not successful - " + f"trying to reset hard since the directory already exists. This will fail if the target directory is " + f"not a git repository." + ) + self.reset_hard(target_dir) + + try: + self.add_remote(target_dir, "upstream", upstream_url) + except GitCliError as e: + logger.debug( + "It looks like remote 'upstream' already exists. Ignoring.", exc_info=e + ) + pass + + self.fetch_all(target_dir) + + if self.does_branch_exist(target_dir, base_branch): + self.checkout_branch(target_dir, base_branch) + else: + try: + self.checkout_branch(target_dir, f"upstream/{base_branch}", track=True) + except GitCliError as e: + logger.debug( + "Could not check out with git checkout --track. Trying git checkout -b.", + exc_info=e, + ) + + # not sure why this is needed, but it was in the original code + self.checkout_new_branch( + target_dir, + base_branch, + start_point=f"upstream/{base_branch}", + ) + + # not sure why this is needed, but it was in the original code + self.reset_hard(target_dir, f"upstream/{base_branch}") + + try: + logger.debug( + f"Trying to checkout branch {new_branch} without creating a new branch" + ) + self.checkout_branch(target_dir, new_branch) + except GitCliError: + logger.debug( + f"It seems branch {new_branch} does not exist. Creating it.", + ) + self.checkout_new_branch(target_dir, new_branch, start_point=base_branch) + + +class GitPlatformBackend(ABC): """ - gh = github3_client() + A backend for interacting with a git platform (e.g. GitHub). - logger.warning("GitHub API error:", exc_info=e) + Implementation Note: If you wonder what should be in this class vs. the GitCli class, the GitPlatformBackend class + should contain the logic for interacting with the platform (e.g. GitHub), while the GitCli class should contain the + logic for interacting with the git repository itself. If you need to know anything specific about the platform, + it should be in the GitPlatformBackend class. - try: - c = gh.rate_limit()["resources"]["core"] - except Exception: - # if we can't connect to the rate limit API, let's assume it has been reached - return True - - if c["remaining"] == 0: - ts = c["reset"] - logger.warning( - "GitHub API timeout, API returns at %s", - datetime.datetime.utcfromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%SZ"), + Git operations are locked (globally) to prevent operations from interfering with each other. + If this does impact performance too much, we can consider a per-repository locking strategy. + """ + + def __init__(self, git_cli: GitCli): + """ + Create a new GitPlatformBackend. + :param git_cli: The GitCli instance to use for interacting with git repositories. + """ + self.cli = git_cli + + @abstractmethod + def does_repository_exist(self, owner: str, repo_name: str) -> bool: + """ + Check if a repository exists. + :param owner: The owner of the repository. + :param repo_name: The name of the repository. + """ + pass + + @staticmethod + def get_remote_url( + owner: str, + repo_name: str, + connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, + ) -> str: + """ + Get the URL of a remote repository. + :param owner: The owner of the repository. + :param repo_name: The name of the repository. + :param connection_mode: The connection mode to use. + :raises ValueError: If the connection mode is not supported. + """ + # Currently we don't need any abstraction for other platforms than GitHub, so we don't build such abstractions. + match connection_mode: + case GitConnectionMode.HTTPS: + return f"https://github.com/{owner}/{repo_name}.git" + case _: + raise ValueError(f"Unsupported connection mode: {connection_mode}") + + @abstractmethod + def fork(self, owner: str, repo_name: str): + """ + Fork a repository. If the fork already exists, do nothing except syncing the default branch name. + Forks are created under the current user's account (see `self.user`). + The name of the forked repository is the same as the original repository. + :param owner: The owner of the repository. + :param repo_name: The name of the repository. + :raises RepositoryNotFoundError: If the repository does not exist. + """ + pass + + @lock_git_operation() + def clone_fork_and_branch( + self, + upstream_owner: str, + repo_name: str, + target_dir: Path, + new_branch: str, + base_branch: str = "main", + ): + """ + Identical to `GitCli::clone_fork_and_branch`, but generates the URLs from the repository name. + + :param upstream_owner: The owner of the upstream repository. + :param repo_name: The name of the repository. + :param target_dir: The directory to clone the repository into. + :param new_branch: The name of the branch to create. + :param base_branch: The name of the base branch to branch from. + + :raises GitCliError: If a git command fails. + """ + self.cli.clone_fork_and_branch( + origin_url=self.get_remote_url(self.user, repo_name), + target_dir=target_dir, + upstream_url=self.get_remote_url(upstream_owner, repo_name), + new_branch=new_branch, + base_branch=base_branch, + ) + + @property + @abstractmethod + def user(self) -> str: + """ + The username of the logged-in user, i.e. the owner of forked repositories. + """ + pass + + @abstractmethod + def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): + """ + Sync the default branch of the forked repository with the upstream repository. + :param upstream_owner: The owner of the upstream repository. + :param upstream_repo: The name of the upstream repository. + """ + pass + + @abstractmethod + def get_api_requests_left(self) -> int | Bound | None: + """ + Get the number of remaining API requests for the backend. + Returns `Bound.INFINITY` if the backend does not have a rate limit. + Returns None if an exception occurred while getting the rate limit. + + Implementations may print diagnostic information about the API limit. + """ + pass + + def is_api_limit_reached(self) -> bool: + """ + Returns True if the API limit has been reached, False otherwise. + + If an exception occurred while getting the rate limit, this method returns True, assuming the limit has + been reached. + + Additionally, implementations may print diagnostic information about the API limit. + """ + return self.get_api_requests_left() in (0, None) + + +class GitHubBackend(GitPlatformBackend): + """ + A git backend for GitHub, using both PyGithub and github3.py as clients. + Both clients are used for historical reasons. In the future, this should be refactored to use only one client. + + Git operations are locked (globally) to prevent operations from interfering with each other. + If this does impact performance too much, we can consider a per-repository locking strategy. + """ + + _GITHUB_PER_PAGE = 100 + """ + The number of items to fetch per page from the GitHub API. + """ + + def __init__(self, github3_client: github3.GitHub, pygithub_client: github.Github): + super().__init__(GitCli()) + self.github3_client = github3_client + self.pygithub_client = pygithub_client + + @classmethod + def from_token(cls, token: str): + return cls( + github3.login(token=token), + github.Github(auth=github.Auth.Token(token), per_page=cls._GITHUB_PER_PAGE), + ) + + def does_repository_exist(self, owner: str, repo_name: str) -> bool: + repo = self.github3_client.repository(owner, repo_name) + return repo is not None + + @lock_git_operation() + def fork(self, owner: str, repo_name: str): + if self.does_repository_exist(self.user, repo_name): + # The fork already exists, so we only sync the default branch. + self._sync_default_branch(owner, repo_name) + return + + repo = self.github3_client.repository(owner, repo_name) + if repo is None: + raise RepositoryNotFoundError( + f"Repository {owner}/{repo_name} does not exist." + ) + + logger.debug(f"Forking {owner}/{repo_name}.") + repo.create_fork() + + # Sleep to make sure the fork is created before we go after it + time.sleep(5) + + @lock_git_operation() + def _sync_default_branch(self, upstream_owner: str, repo_name: str): + fork_owner = self.user + + upstream_repo = self.pygithub_client.get_repo(f"{upstream_owner}/{repo_name}") + fork = self.pygithub_client.get_repo(f"{fork_owner}/{repo_name}") + + if upstream_repo.default_branch == fork.default_branch: + return + + logger.info( + f"Syncing default branch of {fork_owner}/{repo_name} with {upstream_owner}/{repo_name}..." + ) + + fork.rename_branch(fork.default_branch, upstream_repo.default_branch) + + # Sleep to wait for branch name change + time.sleep(5) + + @cached_property + def user(self) -> str: + return self.pygithub_client.get_user().login + + def get_api_requests_left(self) -> int | None: + try: + limit_info = self.github3_client.rate_limit() + except github3.exceptions.GitHubException as e: + logger.warning("GitHub API error while fetching rate limit.", exc_info=e) + return None + + try: + core_resource = limit_info["resources"]["core"] + remaining_limit = core_resource["remaining"] + except KeyError as e: + logger.warning("GitHub API error while parsing rate limit.", exc_info=e) + return None + + if remaining_limit != 0: + return remaining_limit + + # try to log when the limit will be reset + try: + reset_timestamp = core_resource["reset"] + except KeyError as e: + logger.warning( + "GitHub API error while fetching rate limit reset time.", + exc_info=e, + ) + return remaining_limit + + logger.info( + "GitHub API limit reached, will reset at " + f"{datetime.utcfromtimestamp(reset_timestamp).strftime('%Y-%m-%dT%H:%M:%SZ')}" ) - return True - return False + return remaining_limit + + +class DryRunBackend(GitPlatformBackend): + """ + A git backend that doesn't modify anything and only relies on public APIs that do not require authentication. + Useful for local testing with dry-run. + + By default, the dry run backend assumes that the current user has not created any forks yet. + If forks are created, their names are stored in memory and can be checked with `does_repository_exist`. + """ + + _USER = "auto-tick-bot-dry-run" + + def __init__(self): + super().__init__(GitCli()) + self._repos: set[str] = set() + + def get_api_requests_left(self) -> Bound: + return Bound.INFINITY + + def does_repository_exist(self, owner: str, repo_name: str) -> bool: + if owner == self._USER: + return repo_name in self._repos + + # We do not use the GitHub API because unauthenticated requests are quite strictly rate-limited. + return self.cli.does_remote_exist( + self.get_remote_url(owner, repo_name, GitConnectionMode.HTTPS) + ) + + @lock_git_operation() + def fork(self, owner: str, repo_name: str): + if repo_name in self._repos: + raise ValueError(f"Fork of {repo_name} already exists.") + + logger.debug( + f"Dry Run: Creating fork of {owner}/{repo_name} for user {self._USER}." + ) + self._repos.add(repo_name) + + def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): + logger.debug( + f"Dry Run: Syncing default branch of {upstream_owner}/{upstream_repo}." + ) + + @property + def user(self) -> str: + return self._USER + + +def github_backend() -> GitHubBackend: + """ + This helper method will be removed in the future, use the GitHubBackend class directly. + """ + with sensitive_env() as env: + return GitHubBackend.from_token(env["BOT_TOKEN"]) + + +def is_github_api_limit_reached() -> bool: + """ + Return True if the GitHub API limit has been reached, False otherwise. + + This method will be removed in the future, use the GitHubBackend class directly. + """ + backend = github_backend() + + return backend.is_api_limit_reached() def feedstock_url(fctx: FeedstockContext, protocol: str = "ssh") -> str: @@ -177,126 +696,30 @@ def feedstock_url(fctx: FeedstockContext, protocol: str = "ssh") -> str: elif protocol == "ssh": url = "git@github.com:conda-forge/" + feedstock + ".git" else: - msg = "Unrecognized github protocol {0!r}, must be ssh, http, or https." - raise ValueError(msg.format(protocol)) + msg = f"Unrecognized github protocol {protocol}, must be ssh, http, or https." + raise ValueError(msg) return url def feedstock_repo(fctx: FeedstockContext) -> str: """Gets the name of the feedstock repository.""" - repo = fctx.feedstock_name + "-feedstock" - if repo.endswith(".git"): - repo = repo[:-4] - return repo - - -def fork_url(feedstock_url: str, username: str) -> str: - """Creates the URL of the user's fork.""" - beg, end = feedstock_url.rsplit("/", 1) - beg = beg[:-11] # chop off 'conda-forge' - url = beg + username + "/" + end - return url - - -def fetch_repo(*, feedstock_dir, origin, upstream, branch, base_branch="main"): - """fetch a repo and make a PR branch - - Parameters - ---------- - feedstock_dir : str - The directory where you want to clone the feedstock. - origin : str - The origin to clone from. - upstream : str - The upstream repo to add as a remote named `upstream`. - branch : str - The branch to make and checkout. - base_branch : str, optional - The branch from which to branch from to make `branch`. Defaults to "main". - - Returns - ------- - success : bool - True if the fetch worked, False otherwise. - """ - if not os.path.isdir(feedstock_dir): - p = subprocess.run( - ["git", "clone", "-q", origin, feedstock_dir], - ) - if p.returncode != 0: - msg = "Could not clone " + origin - msg += ". Do you have a personal fork of the feedstock?" - print(msg, file=sys.stderr) - return False - reset_hard = False - else: - reset_hard = True - - def _run_git_cmd(cmd, **kwargs): - return subprocess.run(["git"] + cmd, check=True, **kwargs) - - quiet = "--quiet" - with pushd(feedstock_dir): - if reset_hard: - _run_git_cmd(["reset", "--hard", "HEAD"]) - - # doesn't work if the upstream already exists - try: - # always run upstream - _run_git_cmd(["remote", "add", "upstream", upstream]) - except subprocess.CalledProcessError: - pass - - # fetch remote changes - _run_git_cmd(["fetch", "--all", quiet]) - if _run_git_cmd( - ["branch", "--list", base_branch], - capture_output=True, - ).stdout: - _run_git_cmd(["checkout", base_branch, quiet]) - else: - try: - _run_git_cmd(["checkout", "--track", f"upstream/{base_branch}", quiet]) - except subprocess.CalledProcessError: - _run_git_cmd( - ["checkout", "-b", base_branch, f"upstream/{base_branch}", quiet], - ) - _run_git_cmd(["reset", "--hard", f"upstream/{base_branch}", quiet]) - - # make and modify version branch - try: - _run_git_cmd(["checkout", branch, quiet]) - except subprocess.CalledProcessError: - _run_git_cmd(["checkout", "-b", branch, base_branch, quiet]) - - return True + return fctx.feedstock_name + "-feedstock" +@lock_git_operation() def get_repo( fctx: FeedstockContext, branch: str, - feedstock: Optional[str] = None, - protocol: str = "ssh", - pull_request: bool = True, - fork: bool = True, base_branch: str = "main", -) -> Tuple[str, github3.repos.Repository]: +) -> Tuple[str, github3.repos.Repository] | Tuple[Literal[False], Literal[False]]: """Get the feedstock repo Parameters ---------- - fcts : FeedstockContext + fctx : FeedstockContext Feedstock context used for constructing feedstock urls, etc. branch : str The branch to be made. - feedstock : str, optional - The feedstock to clone if None use $FEEDSTOCK - protocol : str, optional - The git protocol to use, defaults to ``ssh`` - pull_request : bool, optional - If true issue pull request, defaults to true - fork : bool - If true create a fork, defaults to true base_branch : str, optional The base branch from which to make the new branch. @@ -307,65 +730,38 @@ def get_repo( repo : github3 repository The github3 repository object. """ - gh = github3_client() - gh_username = gh.me().login + backend = github_backend() + feedstock_repo_name = feedstock_repo(fctx) - # first, let's grab the feedstock locally - upstream = feedstock_url(fctx=fctx, protocol=protocol) - origin = fork_url(upstream, gh_username) - feedstock_reponame = feedstock_repo(fctx=fctx) + try: + backend.fork("conda-forge", feedstock_repo_name) + except RepositoryNotFoundError: + logger.warning(f"Could not fork conda-forge/{feedstock_repo_name}") + with fctx.attrs["pr_info"] as pri: + pri["bad"] = f"{fctx.feedstock_name}: Git repository not found.\n" + return False, False - if pull_request or fork: - repo = gh.repository("conda-forge", feedstock_reponame) - if repo is None: - print("could not fork conda-forge/%s!" % feedstock_reponame, flush=True) - with fctx.attrs["pr_info"] as pri: - pri["bad"] = f"{fctx.feedstock_name}: could not find feedstock\n" - return False, False + feedstock_dir = Path(GIT_CLONE_DIR) / (fctx.feedstock_name + "-feedstock") - # Check if fork exists - if fork: - try: - fork_repo = gh.repository(gh_username, feedstock_reponame) - except github3.GitHubError: - fork_repo = None - if fork_repo is None or (hasattr(fork_repo, "is_null") and fork_repo.is_null()): - print("Fork doesn't exist creating feedstock fork...") - repo.create_fork() - # Sleep to make sure the fork is created before we go after it - time.sleep(5) - - # sync the default branches if needed - _sync_default_branches(feedstock_reponame) - - feedstock_dir = os.path.join(GIT_CLONE_DIR, fctx.feedstock_name + "-feedstock") - - if fetch_repo( - feedstock_dir=feedstock_dir, - origin=origin, - upstream=upstream, - branch=branch, + backend.clone_fork_and_branch( + upstream_owner="conda-forge", + repo_name=feedstock_repo_name, + target_dir=feedstock_dir, + new_branch=branch, base_branch=base_branch, - ): - return feedstock_dir, repo - else: - return False, False + ) + # This is needed because we want to migrate to the new backend step-by-step + repo: github3.repos.Repository | None = github3_client().repository( + "conda-forge", feedstock_repo_name + ) -def _sync_default_branches(reponame): - gh = github_client() - forked_user = gh.get_user().login - default_branch = gh.get_repo(f"conda-forge/{reponame}").default_branch - forked_default_branch = gh.get_repo(f"{forked_user}/{reponame}").default_branch - if default_branch != forked_default_branch: - print("Fork's default branch doesn't match upstream, syncing...") - forked_repo = gh.get_repo(f"{forked_user}/{reponame}") - forked_repo.rename_branch(forked_default_branch, default_branch) + assert repo is not None - # sleep to wait for branch name change - time.sleep(5) + return feedstock_repo_name, repo +@lock_git_operation() def delete_branch(pr_json: LazyJson, dry_run: bool = False) -> None: ref = pr_json["head"]["ref"] if dry_run: @@ -447,9 +843,6 @@ def lazy_update_pr_json( force : bool, optional If True, forcibly update the PR json even if it is not out of date according to the ETag. Default is False. - trim : bool, optional - If True, trim the PR json keys to ones in the global PR_KEYS_TO_KEEP. - Default is True. Returns ------- @@ -589,6 +982,7 @@ def close_out_labels( return None +@lock_git_operation() def push_repo( fctx: FeedstockContext, feedstock_dir: str, @@ -604,7 +998,7 @@ def push_repo( Parameters ---------- - fcts : FeedstockContext + fctx : FeedstockContext Feedstock context used for constructing feedstock urls, etc. feedstock_dir : str The feedstock directory diff --git a/conda_forge_tick/lazy_json_backends.py b/conda_forge_tick/lazy_json_backends.py index 6e4751c45..4619a83ff 100644 --- a/conda_forge_tick/lazy_json_backends.py +++ b/conda_forge_tick/lazy_json_backends.py @@ -26,6 +26,7 @@ import requests from .cli_context import CliContext +from .executors import lock_git_operation logger = logging.getLogger(__name__) @@ -159,10 +160,8 @@ def hgetall(self, name: str, hashval: bool = False) -> Dict[str, str]: } def hdel(self, name: str, keys: Iterable[str]) -> None: - from .executors import DRLOCK, PRLOCK, TRLOCK - lzj_names = [get_sharded_path(f"{name}/{key}.json") for key in keys] - with PRLOCK, DRLOCK, TRLOCK: + with lock_git_operation(): subprocess.run( ["git", "rm", "--ignore-unmatch", "-f"] + lzj_names, capture_output=True, diff --git a/conda_forge_tick/update_prs.py b/conda_forge_tick/update_prs.py index 86b80be21..5c295c10f 100644 --- a/conda_forge_tick/update_prs.py +++ b/conda_forge_tick/update_prs.py @@ -89,7 +89,8 @@ def _update_pr(update_function, dry_run, gx, job, n_jobs): except (github3.GitHubError, github.GithubException) as e: logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}") failed_refresh += 1 - if is_github_api_limit_reached(e): + if is_github_api_limit_reached(): + logger.warning("GitHub API error", exc_info=e) break except (github3.exceptions.ConnectionError, github.GithubException): logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}") diff --git a/tests/test_executors.py b/tests/test_executors.py index 15aeb1483..a71f9b9d5 100644 --- a/tests/test_executors.py +++ b/tests/test_executors.py @@ -8,10 +8,23 @@ def _square_with_lock(x): - from conda_forge_tick.executors import DRLOCK, PRLOCK, TRLOCK + from conda_forge_tick.executors import ( + GIT_LOCK_DASK, + GIT_LOCK_PROCESS, + GIT_LOCK_THREAD, + ) + + with GIT_LOCK_THREAD, GIT_LOCK_PROCESS, GIT_LOCK_DASK: + with GIT_LOCK_THREAD, GIT_LOCK_PROCESS, GIT_LOCK_DASK: + time.sleep(0.01) + return x * x + - with TRLOCK, PRLOCK, DRLOCK: - with TRLOCK, PRLOCK, DRLOCK: +def _square_with_lock_git_operation(x): + from conda_forge_tick.executors import lock_git_operation + + with lock_git_operation(): + with lock_git_operation(): time.sleep(0.01) return x * x @@ -46,6 +59,10 @@ def test_executor(kind): assert np.allclose(tot, par_tot) +@pytest.mark.parametrize( + "locked_square_function", + [_square_with_lock, _square_with_lock_git_operation], +) @pytest.mark.parametrize( "kind", [ @@ -56,7 +73,7 @@ def test_executor(kind): "dask-thread", ], ) -def test_executor_locking(kind): +def test_executor_locking(kind, locked_square_function): seed = 10 rng = np.random.RandomState(seed=seed) nums = rng.uniform(size=100) @@ -74,7 +91,7 @@ def test_executor_locking(kind): par_tot = 0 t0lock = time.time() with executor(kind, max_workers=4) as exe: - futs = [exe.submit(_square_with_lock, num) for num in nums] + futs = [exe.submit(locked_square_function, num) for num in nums] for fut in as_completed(futs): par_tot += fut.result() t0lock = time.time() - t0lock diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 47686fe43..ccaf05eb0 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1,4 +1,894 @@ -from conda_forge_tick.git_utils import trim_pr_json_keys +import subprocess +import tempfile +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +import github3.exceptions +import pytest + +from conda_forge_tick.git_utils import ( + Bound, + DryRunBackend, + GitCli, + GitCliError, + GitConnectionMode, + GitHubBackend, + GitPlatformBackend, + RepositoryNotFoundError, + trim_pr_json_keys, +) + +""" +Note: You have to have git installed on your machine to run these tests. +""" + + +@mock.patch("subprocess.run") +@pytest.mark.parametrize("check_error", [True, False]) +def test_git_cli_run_git_command_no_error( + subprocess_run_mock: MagicMock, check_error: bool +): + cli = GitCli() + + working_directory = Path("TEST_DIR") + + cli._run_git_command( + ["GIT_COMMAND", "ARG1", "ARG2"], working_directory, check_error + ) + + subprocess_run_mock.assert_called_once_with( + ["git", "GIT_COMMAND", "ARG1", "ARG2"], check=check_error, cwd=working_directory + ) + + +@mock.patch("subprocess.run") +def test_git_cli_run_git_command_error(subprocess_run_mock: MagicMock): + cli = GitCli() + + working_directory = Path("TEST_DIR") + + subprocess_run_mock.side_effect = subprocess.CalledProcessError( + returncode=1, cmd="" + ) + + with pytest.raises(GitCliError): + cli._run_git_command(["GIT_COMMAND"], working_directory) + + +def test_git_cli_outside_repo(): + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + with dir_path.joinpath("test.txt").open("w") as f: + f.write("Hello, World!") + + cli = GitCli() + + with pytest.raises(GitCliError): + cli._run_git_command(["status"], working_directory=dir_path) + + with pytest.raises(GitCliError): + cli.reset_hard(dir_path) + + with pytest.raises(GitCliError): + cli.add_remote(dir_path, "origin", "https://github.com/torvalds/linux.git") + + with pytest.raises(GitCliError): + cli.fetch_all(dir_path) + + assert not cli.does_branch_exist(dir_path, "main") + + with pytest.raises(GitCliError): + cli.checkout_branch(dir_path, "main") + + +# noinspection PyProtectedMember +def init_temp_git_repo(git_dir: Path): + cli = GitCli() + cli._run_git_command(["init"], working_directory=git_dir) + cli._run_git_command( + ["config", "user.name", "CI Test User"], working_directory=git_dir + ) + cli._run_git_command( + ["config", "user.email", "ci-test-user-invalid@example.com"], + working_directory=git_dir, + ) + + +def test_git_cli_reset_hard_already_reset(): + cli = GitCli() + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + cli._run_git_command( + ["commit", "--allow-empty", "-m", "First commit"], + working_directory=dir_path, + ) + + cli._run_git_command( + ["commit", "--allow-empty", "-m", "Second commit"], + working_directory=dir_path, + ) + + cli.reset_hard(dir_path) + + git_log = subprocess.run( + "git log", cwd=dir_path, shell=True, capture_output=True + ).stdout.decode() + + assert "First commit" in git_log + assert "Second commit" in git_log + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_reset_hard_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + cli.reset_hard(git_dir) + + run_git_command_mock.assert_called_once_with( + ["reset", "--quiet", "--hard", "HEAD"], git_dir + ) + + +def test_git_cli_reset_hard(): + cli = GitCli() + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + cli._run_git_command( + ["commit", "--allow-empty", "-m", "Initial commit"], + working_directory=dir_path, + ) + + with dir_path.joinpath("test.txt").open("w") as f: + f.write("Hello, World!") + + cli._run_git_command(["add", "test.txt"], working_directory=dir_path) + cli._run_git_command( + ["commit", "-am", "Add test.txt"], working_directory=dir_path + ) + + with dir_path.joinpath("test.txt").open("w") as f: + f.write("Hello, World! Again!") + + cli.reset_hard(dir_path) + + with dir_path.joinpath("test.txt").open("r") as f: + assert f.read() == "Hello, World!" + + +def test_git_cli_clone_repo_not_exists(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + with pytest.raises(GitCliError): + cli.clone_repo( + "https://github.com/conda-forge/this-repo-does-not-exist.git", dir_path + ) + + +def test_git_cli_clone_repo_success(): + cli = GitCli() + + git_url = "https://github.com/conda-forge/duckdb-feedstock.git" + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "duckdb-feedstock" + + # this is an archived feedstock that should not change + cli.clone_repo(git_url, dir_path) + + readme_file = dir_path.joinpath("README.md") + + assert readme_file.exists() + + +def test_git_cli_clone_repo_existing_empty_dir(): + cli = GitCli() + + git_url = "https://github.com/conda-forge/duckdb-feedstock.git" + + with tempfile.TemporaryDirectory() as tmpdir: + target = Path(tmpdir) / "duckdb-feedstock" + + target.mkdir() + + cli.clone_repo(git_url, target) + + readme_file = target.joinpath("README.md") + + assert readme_file.exists() + + +@mock.patch("conda_forge_tick.git_utils.GitCli.reset_hard") +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_clone_repo_mock_success( + run_git_command_mock: MagicMock, reset_hard_mock: MagicMock +): + cli = GitCli() + + git_url = "https://git-repository.com/repo.git" + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "repo" + + cli.clone_repo(git_url, dir_path) + + run_git_command_mock.assert_called_once_with( + ["clone", "--quiet", git_url, dir_path] + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_clone_repo_mock_error(run_git_command_mock: MagicMock): + cli = GitCli() + + git_url = "https://git-repository.com/repo.git" + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "repo" + + run_git_command_mock.side_effect = GitCliError("Error") + + with pytest.raises(GitCliError, match="Error cloning repository"): + cli.clone_repo(git_url, dir_path) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_add_remote_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + remote_name = "origin" + remote_url = "https://git-repository.com/repo.git" + + cli.add_remote(git_dir, remote_name, remote_url) + + run_git_command_mock.assert_called_once_with( + ["remote", "add", remote_name, remote_url], git_dir + ) + + +def test_git_cli_add_remote(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + remote_name = "remote24" + remote_url = "https://git-repository.com/repo.git" + + cli.add_remote(dir_path, remote_name, remote_url) + + output = subprocess.run( + "git remote -v", cwd=dir_path, shell=True, capture_output=True + ) + + assert remote_name in output.stdout.decode() + assert remote_url in output.stdout.decode() + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_fetch_all_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + cli.fetch_all(git_dir) + + run_git_command_mock.assert_called_once_with(["fetch", "--all", "--quiet"], git_dir) + + +def test_git_cli_fetch_all(): + cli = GitCli() + + git_url = "https://github.com/conda-forge/duckdb-feedstock.git" + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "duckdb-feedstock" + + cli.clone_repo(git_url, dir_path) + cli.fetch_all(dir_path) + + +def test_git_cli_does_branch_exist(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + assert not cli.does_branch_exist(dir_path, "main") + + cli._run_git_command(["checkout", "-b", "main"], working_directory=dir_path) + cli._run_git_command( + ["commit", "--allow-empty", "-m", "Initial commit"], + working_directory=dir_path, + ) + + assert cli.does_branch_exist(dir_path, "main") + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +@pytest.mark.parametrize("does_exist", [True, False]) +def test_git_cli_does_branch_exist_mock( + run_git_command_mock: MagicMock, does_exist: bool +): + cli = GitCli() + + git_dir = Path("TEST_DIR") + branch_name = "main" + + run_git_command_mock.return_value = ( + subprocess.CompletedProcess(args=[], returncode=0) + if does_exist + else subprocess.CompletedProcess(args=[], returncode=1) + ) + + assert cli.does_branch_exist(git_dir, branch_name) is does_exist + + run_git_command_mock.assert_called_once_with( + ["show-ref", "--verify", "--quiet", f"refs/heads/{branch_name}"], + git_dir, + check_error=False, + ) + + +def test_git_cli_does_remote_exist_false(): + cli = GitCli() + + remote_url = "https://github.com/conda-forge/this-repo-does-not-exist.git" + + assert not cli.does_remote_exist(remote_url) + + +def test_git_cli_does_remote_exist_true(): + cli = GitCli() + + remote_url = "https://github.com/conda-forge/pytest-feedstock.git" + + assert cli.does_remote_exist(remote_url) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +@pytest.mark.parametrize("does_exist", [True, False]) +def test_git_cli_does_remote_exist_mock( + run_git_command_mock: MagicMock, does_exist: bool +): + cli = GitCli() + + remote_url = "https://git-repository.com/repo.git" + + run_git_command_mock.return_value = ( + subprocess.CompletedProcess(args=[], returncode=0) + if does_exist + else subprocess.CompletedProcess(args=[], returncode=1) + ) + + assert cli.does_remote_exist(remote_url) is does_exist + + run_git_command_mock.assert_called_once_with( + ["ls-remote", remote_url], check_error=False + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +@pytest.mark.parametrize("track", [True, False]) +def test_git_cli_checkout_branch_mock(run_git_command_mock: MagicMock, track: bool): + branch_name = "BRANCH_NAME" + + cli = GitCli() + git_dir = Path("TEST_DIR") + + cli.checkout_branch(git_dir, branch_name, track=track) + + track_flag = ["--track"] if track else [] + + run_git_command_mock.assert_called_once_with( + ["checkout", "--quiet", *track_flag, branch_name], git_dir + ) + + +def test_git_cli_checkout_branch_no_track(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + cli._run_git_command(["checkout", "-b", "main"], working_directory=dir_path) + cli._run_git_command( + ["commit", "--allow-empty", "-m", "Initial commit"], + working_directory=dir_path, + ) + + assert ( + "main" + in subprocess.run( + "git status", cwd=dir_path, shell=True, capture_output=True + ).stdout.decode() + ) + + branch_name = "new-branch-name" + + cli._run_git_command(["branch", branch_name], working_directory=dir_path) + + cli.checkout_branch(dir_path, branch_name) + + assert ( + branch_name + in subprocess.run( + "git status", cwd=dir_path, shell=True, capture_output=True + ).stdout.decode() + ) + + +def test_git_cli_clone_fork_and_branch_minimal(): + fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" + upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" + + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "pytest-feedstock" + + new_branch_name = "new_branch_name" + + cli.clone_fork_and_branch(fork_url, dir_path, upstream_url, new_branch_name) + + assert cli.does_branch_exist(dir_path, "main") + assert ( + new_branch_name + in subprocess.run( + "git status", cwd=dir_path, shell=True, capture_output=True + ).stdout.decode() + ) + + +@pytest.mark.parametrize("remote_already_exists", [True, False]) +@pytest.mark.parametrize( + "base_branch_exists,git_checkout_track_error", + [(True, False), (False, False), (False, True)], +) +@pytest.mark.parametrize("new_branch_already_exists", [True, False]) +@pytest.mark.parametrize("target_repo_already_exists", [True, False]) +@mock.patch("conda_forge_tick.git_utils.GitCli.reset_hard") +@mock.patch("conda_forge_tick.git_utils.GitCli.checkout_new_branch") +@mock.patch("conda_forge_tick.git_utils.GitCli.checkout_branch") +@mock.patch("conda_forge_tick.git_utils.GitCli.does_branch_exist") +@mock.patch("conda_forge_tick.git_utils.GitCli.fetch_all") +@mock.patch("conda_forge_tick.git_utils.GitCli.add_remote") +@mock.patch("conda_forge_tick.git_utils.GitCli.clone_repo") +def test_git_cli_clone_fork_and_branch_mock( + clone_repo_mock: MagicMock, + add_remote_mock: MagicMock, + fetch_all_mock: MagicMock, + does_branch_exist_mock: MagicMock, + checkout_branch_mock: MagicMock, + checkout_new_branch_mock: MagicMock, + reset_hard_mock: MagicMock, + remote_already_exists: bool, + base_branch_exists: bool, + git_checkout_track_error: bool, + new_branch_already_exists: bool, + target_repo_already_exists: bool, + caplog, +): + fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" + upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" + + caplog.set_level("DEBUG") + + cli = GitCli() + + if target_repo_already_exists: + clone_repo_mock.side_effect = GitCliError( + "target_dir is not an empty directory" + ) + + if remote_already_exists: + add_remote_mock.side_effect = GitCliError("Remote already exists") + + does_branch_exist_mock.return_value = base_branch_exists + + def checkout_branch_side_effect(_git_dir: Path, branch: str, track: bool = False): + if track and git_checkout_track_error: + raise GitCliError("Error checking out branch with --track") + + if new_branch_already_exists and branch == "new_branch_name": + raise GitCliError("Branch new_branch_name already exists") + + checkout_branch_mock.side_effect = checkout_branch_side_effect + + with tempfile.TemporaryDirectory() as tmpdir: + git_dir = Path(tmpdir) / "pytest-feedstock" + + if target_repo_already_exists: + git_dir.mkdir() + + cli.clone_fork_and_branch( + fork_url, git_dir, upstream_url, "new_branch_name", "base_branch" + ) + + clone_repo_mock.assert_called_once_with(fork_url, git_dir) + if target_repo_already_exists: + reset_hard_mock.assert_any_call(git_dir) + + add_remote_mock.assert_called_once_with(git_dir, "upstream", upstream_url) + if remote_already_exists: + assert "remote 'upstream' already exists" in caplog.text + + fetch_all_mock.assert_called_once_with(git_dir) + + if base_branch_exists: + checkout_branch_mock.assert_any_call(git_dir, "base_branch") + else: + checkout_branch_mock.assert_any_call( + git_dir, "upstream/base_branch", track=True + ) + + if git_checkout_track_error: + assert "Could not check out with git checkout --track" in caplog.text + + checkout_new_branch_mock.assert_any_call( + git_dir, "base_branch", start_point="upstream/base_branch" + ) + + reset_hard_mock.assert_called_with(git_dir, "upstream/base_branch") + checkout_branch_mock.assert_any_call(git_dir, "new_branch_name") + + if not new_branch_already_exists: + return + + assert "branch new_branch_name does not exist" in caplog.text + checkout_new_branch_mock.assert_called_with( + git_dir, "new_branch_name", start_point="base_branch" + ) + + +def test_git_cli_clone_fork_and_branch_non_existing_remote(): + origin_url = "https://github.com/conda-forge/this-repo-does-not-exist.git" + upstream_url = "https://github.com/conda-forge/duckdb-feedstock.git" + new_branch = "NEW_BRANCH" + + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "duckdb-feedstock" + + with pytest.raises(GitCliError, match="does the remote exist?"): + cli.clone_fork_and_branch(origin_url, dir_path, upstream_url, new_branch) + + +def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(caplog): + origin_url = "https://github.com/conda-forge/this-repo-does-not-exist.git" + upstream_url = "https://github.com/conda-forge/duckdb-feedstock.git" + new_branch = "NEW_BRANCH" + + cli = GitCli() + caplog.set_level("DEBUG") + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) / "duckdb-feedstock" + dir_path.mkdir() + + with pytest.raises(GitCliError): + cli.clone_fork_and_branch(origin_url, dir_path, upstream_url, new_branch) + + assert "trying to reset hard" in caplog.text + + +def test_git_platform_backend_get_remote_url_https(): + owner = "OWNER" + repo = "REPO" + + url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS) + + assert url == f"https://github.com/{owner}/{repo}.git" + + +def test_github_backend_from_token(): + token = "TOKEN" + + backend = GitHubBackend.from_token(token) + + assert backend.github3_client.session.auth.token == token + # we cannot verify the pygithub token trivially + + +@pytest.mark.parametrize("does_exist", [True, False]) +def test_github_backend_does_repository_exist(does_exist: bool): + github3_client = MagicMock() + + backend = GitHubBackend(github3_client, MagicMock()) + + github3_client.repository.return_value = MagicMock() if does_exist else None + + assert backend.does_repository_exist("OWNER", "REPO") is does_exist + github3_client.repository.assert_called_once_with("OWNER", "REPO") + + +@mock.patch("time.sleep", return_value=None) +@mock.patch( + "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock +) +@mock.patch("conda_forge_tick.git_utils.GitHubBackend.does_repository_exist") +def test_github_backend_fork_not_exists_repo_found( + exists_mock: MagicMock, user_mock: MagicMock, sleep_mock: MagicMock +): + exists_mock.return_value = False + + github3_client = MagicMock() + repository = MagicMock() + github3_client.repository.return_value = repository + + backend = GitHubBackend(github3_client, MagicMock()) + user_mock.return_value = "USER" + backend.fork("UPSTREAM-OWNER", "REPO") + + exists_mock.assert_called_once_with("USER", "REPO") + github3_client.repository.assert_called_once_with("UPSTREAM-OWNER", "REPO") + repository.create_fork.assert_called_once() + sleep_mock.assert_called_once_with(5) + + +@pytest.mark.parametrize("branch_already_synced", [True, False]) +@mock.patch("time.sleep", return_value=None) +@mock.patch( + "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock +) +@mock.patch("conda_forge_tick.git_utils.GitHubBackend.does_repository_exist") +def test_github_backend_fork_exists( + exists_mock: MagicMock, + user_mock: MagicMock, + sleep_mock: MagicMock, + branch_already_synced: bool, + caplog, +): + caplog.set_level("DEBUG") + + exists_mock.return_value = True + user_mock.return_value = "USER" + + pygithub_client = MagicMock() + upstream_repo = MagicMock() + fork_repo = MagicMock() + + def get_repo(full_name: str): + if full_name == "UPSTREAM-OWNER/REPO": + return upstream_repo + if full_name == "USER/REPO": + return fork_repo + assert False, f"Unexpected repo full name: {full_name}" + + pygithub_client.get_repo.side_effect = get_repo + + if branch_already_synced: + upstream_repo.default_branch = "BRANCH_NAME" + fork_repo.default_branch = "BRANCH_NAME" + else: + upstream_repo.default_branch = "UPSTREAM_BRANCH_NAME" + fork_repo.default_branch = "FORK_BRANCH_NAME" + + backend = GitHubBackend(MagicMock(), pygithub_client) + backend.fork("UPSTREAM-OWNER", "REPO") + + if not branch_already_synced: + pygithub_client.get_repo.assert_any_call("UPSTREAM-OWNER/REPO") + pygithub_client.get_repo.assert_any_call("USER/REPO") + + assert "Syncing default branch" in caplog.text + sleep_mock.assert_called_once_with(5) + + +@mock.patch( + "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock +) +@mock.patch("conda_forge_tick.git_utils.GitHubBackend.does_repository_exist") +def test_github_backend_remote_does_not_exist( + exists_mock: MagicMock, user_mock: MagicMock +): + exists_mock.return_value = False + + github3_client = MagicMock() + github3_client.repository.return_value = None + + backend = GitHubBackend(github3_client, MagicMock()) + + user_mock.return_value = "USER" + + with pytest.raises(RepositoryNotFoundError): + backend.fork("UPSTREAM-OWNER", "REPO") + + exists_mock.assert_called_once_with("USER", "REPO") + github3_client.repository.assert_called_once_with("UPSTREAM-OWNER", "REPO") + + +def test_github_backend_user(): + pygithub_client = MagicMock() + user = MagicMock() + user.login = "USER" + pygithub_client.get_user.return_value = user + + backend = GitHubBackend(MagicMock(), pygithub_client) + + for _ in range(4): + # cached property + assert backend.user == "USER" + + pygithub_client.get_user.assert_called_once() + + +def test_github_backend_get_api_requests_left_github_exception(caplog): + github3_client = MagicMock() + github3_client.rate_limit.side_effect = github3.exceptions.GitHubException( + "API Error" + ) + + backend = GitHubBackend(github3_client, MagicMock()) + + assert backend.get_api_requests_left() is None + assert "API error while fetching" in caplog.text + + github3_client.rate_limit.assert_called_once() + + +def test_github_backend_get_api_requests_left_unexpected_response_schema(caplog): + github3_client = MagicMock() + github3_client.rate_limit.return_value = {"some": "gibberish data"} + + backend = GitHubBackend(github3_client, MagicMock()) + + assert backend.get_api_requests_left() is None + assert "API Error while parsing" + + github3_client.rate_limit.assert_called_once() + + +def test_github_backend_get_api_requests_left_nonzero(): + github3_client = MagicMock() + github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 5}}} + + backend = GitHubBackend(github3_client, MagicMock()) + + assert backend.get_api_requests_left() == 5 + + github3_client.rate_limit.assert_called_once() + + +def test_github_backend_get_api_requests_left_zero_invalid_reset_time(caplog): + github3_client = MagicMock() + + github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 0}}} + + backend = GitHubBackend(github3_client, MagicMock()) + + assert backend.get_api_requests_left() == 0 + + github3_client.rate_limit.assert_called_once() + assert "GitHub API error while fetching rate limit reset time" in caplog.text + + +def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): + caplog.set_level("INFO") + + github3_client = MagicMock() + + reset_timestamp = 1716303697 + reset_timestamp_str = "2024-05-21T15:01:37Z" + + github3_client.rate_limit.return_value = { + "resources": {"core": {"remaining": 0, "reset": reset_timestamp}} + } + + backend = GitHubBackend(github3_client, MagicMock()) + + assert backend.get_api_requests_left() == 0 + + github3_client.rate_limit.assert_called_once() + assert f"will reset at {reset_timestamp_str}" in caplog.text # + + +@pytest.mark.parametrize( + "backend", [GitHubBackend(MagicMock(), MagicMock()), DryRunBackend()] +) +@mock.patch( + "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock +) +@mock.patch("conda_forge_tick.git_utils.GitCli.clone_fork_and_branch") +def test_git_platform_backend_clone_fork_and_branch( + convenience_method_mock: MagicMock, + user_mock: MagicMock, + backend: GitPlatformBackend, +): + upstream_owner = "UPSTREAM-OWNER" + repo_name = "REPO" + target_dir = Path("TARGET_DIR") + new_branch = "NEW_BRANCH" + base_branch = "BASE_BRANCH" + + user_mock.return_value = "USER" + + backend = GitHubBackend(MagicMock(), MagicMock()) + backend.clone_fork_and_branch( + upstream_owner, repo_name, target_dir, new_branch, base_branch + ) + + convenience_method_mock.assert_called_once_with( + origin_url=f"https://github.com/USER/{repo_name}.git", + target_dir=target_dir, + upstream_url=f"https://github.com/{upstream_owner}/{repo_name}.git", + new_branch=new_branch, + base_branch=base_branch, + ) + + +def test_dry_run_backend_get_api_requests_left(): + backend = DryRunBackend() + + assert backend.get_api_requests_left() is Bound.INFINITY + + +def test_dry_run_backend_does_repository_exist_own_repo(): + backend = DryRunBackend() + + assert not backend.does_repository_exist("auto-tick-bot-dry-run", "REPO") + backend.fork("UPSTREAM_OWNER", "REPO") + assert backend.does_repository_exist("auto-tick-bot-dry-run", "REPO") + + +def test_dry_run_backend_does_repository_exist_other_repo(): + backend = DryRunBackend() + + assert backend.does_repository_exist("conda-forge", "pytest-feedstock") + assert not backend.does_repository_exist( + "conda-forge", "this-repository-does-not-exist" + ) + + +def test_dry_run_backend_fork(caplog): + caplog.set_level("DEBUG") + + backend = DryRunBackend() + + backend.fork("UPSTREAM_OWNER", "REPO") + assert ( + "Dry Run: Creating fork of UPSTREAM_OWNER/REPO for user auto-tick-bot-dry-run" + in caplog.text + ) + + with pytest.raises(ValueError, match="Fork of REPO already exists"): + backend.fork("UPSTREAM_OWNER", "REPO") + + with pytest.raises(ValueError, match="Fork of REPO already exists"): + backend.fork("OTHER_OWNER", "REPO") + + +def test_dry_run_backend_sync_default_branch(caplog): + caplog.set_level("DEBUG") + + backend = DryRunBackend() + + backend._sync_default_branch("UPSTREAM_OWNER", "REPO") + + assert "Dry Run: Syncing default branch of UPSTREAM_OWNER/REPO" in caplog.text + + +def test_dry_run_backend_user(): + backend = DryRunBackend() + + assert backend.user == "auto-tick-bot-dry-run" def test_trim_pr_json_keys(): From 411af0d04df57714ea194c364693e6e1aded592c Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 13 Jun 2024 19:25:58 +0200 Subject: [PATCH 02/38] return feedstock_dir, not dirname --- conda_forge_tick/git_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 87395c0f0..c3d40151c 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -758,7 +758,7 @@ def get_repo( assert repo is not None - return feedstock_repo_name, repo + return str(feedstock_dir), repo @lock_git_operation() From 7c238522ba08fdb700f09c018685dbf60ccb7946 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 28 May 2024 11:38:48 +0200 Subject: [PATCH 03/38] add push_to_url --- conda_forge_tick/git_utils.py | 16 ++++++-- tests/test_git_utils.py | 69 ++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index c3d40151c..5ad602b39 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -210,6 +210,18 @@ def add_remote(self, git_dir: Path, remote_name: str, remote_url: str): """ self._run_git_command(["remote", "add", remote_name, remote_url], git_dir) + def push_to_url(self, git_dir: Path, remote_url: str, branch: str): + """ + Push changes to a remote URL. + + :param git_dir: The directory of the git repository. + :param remote_url: The URL of the remote. + :param branch: The branch to push to. + :raises GitCliError: If the git command fails. + """ + + self._run_git_command(["push", remote_url, branch], git_dir) + @lock_git_operation() def fetch_all(self, git_dir: Path): """ @@ -991,7 +1003,6 @@ def push_repo( title: str, branch: str, base_branch: str = "main", - head: Optional[str] = None, dry_run: bool = False, ) -> Union[dict, bool, None]: """Push a repo up to github @@ -1026,8 +1037,7 @@ def push_repo( token = env["BOT_TOKEN"] gh_username = github3_client().me().login - if head is None: - head = gh_username + ":" + branch + head = gh_username + ":" + branch deploy_repo = gh_username + "/" + fctx.feedstock_name + "-feedstock" if dry_run: diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index ccaf05eb0..1884d42fc 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -84,9 +84,10 @@ def test_git_cli_outside_repo(): # noinspection PyProtectedMember -def init_temp_git_repo(git_dir: Path): +def init_temp_git_repo(git_dir: Path, bare: bool = False): cli = GitCli() - cli._run_git_command(["init"], working_directory=git_dir) + bare_arg = ["--bare"] if bare else [] + cli._run_git_command(["init", *bare_arg, "-b", "main"], working_directory=git_dir) cli._run_git_command( ["config", "user.name", "CI Test User"], working_directory=git_dir ) @@ -278,6 +279,70 @@ def test_git_cli_add_remote(): assert remote_url in output.stdout.decode() +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_push_to_url_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + remote_url = "https://git-repository.com/repo.git" + + cli.push_to_url(git_dir, remote_url, "BRANCH_NAME") + + run_git_command_mock.assert_called_once_with( + ["push", remote_url, "BRANCH_NAME"], git_dir + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_push_to_url_mock_error(run_git_command_mock: MagicMock): + cli = GitCli() + + run_git_command_mock.side_effect = GitCliError("Error") + + with pytest.raises(GitCliError): + cli.push_to_url( + Path("TEST_DIR"), "https://git-repository.com/repo.git", "BRANCH_NAME" + ) + + +def test_git_cli_push_to_url_local_repository(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + source_repo = dir_path / "source_repo" + source_repo.mkdir() + init_temp_git_repo(source_repo, bare=True) + + local_repo = dir_path / "local_repo" + local_repo.mkdir() + cli._run_git_command( + ["clone", source_repo.resolve(), local_repo], + ) + + # remove all references to the original repo + cli._run_git_command( + ["remote", "remove", "origin"], working_directory=local_repo + ) + + with local_repo.joinpath("test.txt").open("w") as f: + f.write("Hello, World!") + + cli._run_git_command(["add", "test.txt"], working_directory=local_repo) + cli._run_git_command( + ["commit", "-am", "Add test.txt"], working_directory=local_repo + ) + + cli.push_to_url(local_repo, str(source_repo.resolve()), "main") + + source_git_log = subprocess.run( + "git log", cwd=source_repo, shell=True, capture_output=True + ).stdout.decode() + + assert "test.txt" in source_git_log + + @mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") def test_git_cli_fetch_all_mock(run_git_command_mock: MagicMock): cli = GitCli() From 78e00bf5b5d2e990f7bbd0e8280cefe447add654 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 28 May 2024 11:41:13 +0200 Subject: [PATCH 04/38] clean up auto_tick:run signature --- conda_forge_tick/auto_tick.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index e1da77301..b7a42f49a 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -142,10 +142,7 @@ def _get_pre_pr_migrator_attempts(attrs, migrator_name, *, is_version): def run( feedstock_ctx: FeedstockContext, migrator: Migrator, - protocol: str = "ssh", - pull_request: bool = True, rerender: bool = True, - fork: bool = True, base_branch: str = "main", dry_run: bool = False, **kwargs: typing.Any, @@ -158,16 +155,12 @@ def run( The node attributes migrator: Migrator instance The migrator to run on the feedstock - protocol : str, optional - The git protocol to use, defaults to ``ssh`` - pull_request : bool, optional - If true issue pull request, defaults to true rerender : bool Whether to rerender - fork : bool - If true create a fork, defaults to true base_branch : str, optional - The base branch to which the PR will be targeted. Defaults to "main". + The base branch to which the PR will be targeted. + dry_run : bool, optional + Whether to run in dry run mode. kwargs: dict The keyword arguments to pass to the migrator. @@ -568,10 +561,9 @@ def _run_migrator_on_feedstock_branch( feedstock_ctx=fctx, migrator=migrator, rerender=migrator.rerender, - protocol="https", - hash_type=attrs.get("hash_type", "sha256"), base_branch=base_branch, dry_run=dry_run, + hash_type=attrs.get("hash_type", "sha256"), ) finally: fctx.attrs.pop("new_version", None) From 304c0877aff9f1e7f2d3b12789b66c73d418b4bd Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Wed, 29 May 2024 18:08:12 +0200 Subject: [PATCH 05/38] add feedstock attributes to FeedstockContext --- conda_forge_tick/contexts.py | 15 +++++++ conda_forge_tick/git_utils.py | 11 +++-- ...migrators_types.pyi => migrators_types.py} | 23 +++++++++- conda_forge_tick/status_report.py | 6 +-- tests/test_contexts.py | 42 +++++++++++++++++++ 5 files changed, 86 insertions(+), 11 deletions(-) rename conda_forge_tick/{migrators_types.pyi => migrators_types.py} (98%) create mode 100644 tests/test_contexts.py diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index 5cc545831..451d0cd98 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -43,3 +43,18 @@ def default_branch(self): @default_branch.setter def default_branch(self, v): self._default_branch = v + + @property + def git_repo_owner(self) -> str: + return "conda-forge" + + @property + def git_repo_name(self) -> str: + return f"{self.feedstock_name}-feedstock" + + @property + def git_href(self) -> str: + """ + A link to the feedstocks GitHub repository. + """ + return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}" diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 5ad602b39..5a917dcd4 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -743,12 +743,11 @@ def get_repo( The github3 repository object. """ backend = github_backend() - feedstock_repo_name = feedstock_repo(fctx) try: - backend.fork("conda-forge", feedstock_repo_name) + backend.fork(fctx.git_repo_owner, fctx.git_repo_name) except RepositoryNotFoundError: - logger.warning(f"Could not fork conda-forge/{feedstock_repo_name}") + logger.warning(f"Could not fork {fctx.git_repo_owner}/{fctx.git_repo_name}") with fctx.attrs["pr_info"] as pri: pri["bad"] = f"{fctx.feedstock_name}: Git repository not found.\n" return False, False @@ -756,8 +755,8 @@ def get_repo( feedstock_dir = Path(GIT_CLONE_DIR) / (fctx.feedstock_name + "-feedstock") backend.clone_fork_and_branch( - upstream_owner="conda-forge", - repo_name=feedstock_repo_name, + upstream_owner=fctx.git_repo_owner, + repo_name=fctx.git_repo_name, target_dir=feedstock_dir, new_branch=branch, base_branch=base_branch, @@ -765,7 +764,7 @@ def get_repo( # This is needed because we want to migrate to the new backend step-by-step repo: github3.repos.Repository | None = github3_client().repository( - "conda-forge", feedstock_repo_name + fctx.git_repo_owner, fctx.git_repo_name ) assert repo is not None diff --git a/conda_forge_tick/migrators_types.pyi b/conda_forge_tick/migrators_types.py similarity index 98% rename from conda_forge_tick/migrators_types.pyi rename to conda_forge_tick/migrators_types.py index a0005c23d..1296f5200 100644 --- a/conda_forge_tick/migrators_types.pyi +++ b/conda_forge_tick/migrators_types.py @@ -5,6 +5,7 @@ PackageName = typing.NewType("PackageName", str) + class AboutTypedDict(TypedDict, total=False): description: str dev_url: str @@ -15,6 +16,7 @@ class AboutTypedDict(TypedDict, total=False): license_file: str summary: str + # PRStateOpen: Literal["open"] # PRStateClosed: Literal["closed"] # PRStateMerged: Literal["merged"] @@ -22,35 +24,43 @@ class AboutTypedDict(TypedDict, total=False): # PRState = Literal[PRStateClosed, PRStateMerged, PRStateOpen] PRState = typing.NewType("PRState", str) -class PRHead_TD(TypedDict, tota=False): + +class PRHead_TD(TypedDict, total=False): ref: str + class PR_TD(TypedDict, total=False): state: PRState head: PRHead_TD + class BlasRebuildMigrateTypedDict(TypedDict): bot_rerun: bool migrator_name: str migrator_version: int name: str + class BuildRunExportsDict(TypedDict, total=False): strong: List[PackageName] weak: List[PackageName] + class BuildTypedDict(TypedDict, total=False): noarch: str number: str script: str run_exports: Union[List[PackageName], BuildRunExportsDict] + ExtraTypedDict = TypedDict("ExtraTypedDict", {"recipe-maintainers": List[str]}) + # class HTypedDict(TypedDict): # data: 'DataTypedDict' # keys: List[str] + class MetaYamlOutputs(TypedDict, total=False): name: str requirements: "RequirementsTypedDict" @@ -58,6 +68,7 @@ class MetaYamlOutputs(TypedDict, total=False): # TODO: Not entirely sure this is right build: BuildRunExportsDict + class MetaYamlTypedDict(TypedDict, total=False): about: "AboutTypedDict" build: "BuildTypedDict" @@ -68,6 +79,7 @@ class MetaYamlTypedDict(TypedDict, total=False): test: "TestTypedDict" outputs: List[MetaYamlOutputs] + class MigrationUidTypedDict(TypedDict, total=False): bot_rerun: bool migrator_name: str @@ -77,31 +89,37 @@ class MigrationUidTypedDict(TypedDict, total=False): # Used by version migrators version: str + class PackageTypedDict(TypedDict): name: str version: str + class RequirementsTypedDict(TypedDict, total=False): build: List[str] host: List[str] run: List[str] + class SourceTypedDict(TypedDict, total=False): fn: str patches: List[str] sha256: str url: str + class TestTypedDict(TypedDict, total=False): commands: List[str] imports: List[str] requires: List[str] requirements: List[str] + class PRedElementTypedDict(TypedDict, total=False): data: MigrationUidTypedDict PR: PR_TD + class AttrsTypedDict_(TypedDict, total=False): about: AboutTypedDict build: BuildTypedDict @@ -124,12 +142,15 @@ class AttrsTypedDict_(TypedDict, total=False): # TODO: ADD in # "conda-forge.yml": + class CondaForgeYamlContents(TypedDict, total=False): provider: Dict[str, str] + CondaForgeYaml = TypedDict( "CondaForgeYaml", {"conda-forge.yml": CondaForgeYamlContents} ) + class AttrsTypedDict(AttrsTypedDict_, CondaForgeYaml): pass diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 77aebb673..54c2dae46 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -39,8 +39,6 @@ load_existing_graph, ) -from .git_utils import feedstock_url - GH_MERGE_STATE_STATUS = [ "behind", "blocked", @@ -273,7 +271,7 @@ def graph_migrator_status( .get("PR", {}) .get( "html_url", - feedstock_url(fctx=feedstock_ctx, protocol="https").strip(".git"), + feedstock_ctx.git_href, ), ) @@ -300,7 +298,7 @@ def graph_migrator_status( # I needed to fake some PRs they don't have html_urls though node_metadata["pr_url"] = pr_json["PR"].get( "html_url", - feedstock_url(fctx=feedstock_ctx, protocol="https").strip(".git"), + feedstock_ctx.git_href, ) node_metadata["pr_status"] = pr_json["PR"].get("mergeable_state", "") diff --git a/tests/test_contexts.py b/tests/test_contexts.py new file mode 100644 index 000000000..0a72dff86 --- /dev/null +++ b/tests/test_contexts.py @@ -0,0 +1,42 @@ +from conda_forge_tick.contexts import DEFAULT_BRANCHES, FeedstockContext +from conda_forge_tick.migrators_types import AttrsTypedDict + +# to make the typechecker happy, this satisfies the AttrsTypedDict type +demo_attrs = AttrsTypedDict( + {"conda-forge.yml": {"provider": {"default_branch": "main"}}} +) + + +def test_feedstock_context_default_branch(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.default_branch == "main" + + DEFAULT_BRANCHES["TEST-FEEDSTOCK-NAME"] = "develop" + assert context.default_branch == "develop" + + context.default_branch = "feature" + assert context.default_branch == "feature" + + # reset the default branches + DEFAULT_BRANCHES.pop("TEST-FEEDSTOCK-NAME") + + # test the default branch is still the same + assert context.default_branch == "feature" + + +def test_feedstock_context_git_repo_owner(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.git_repo_owner == "conda-forge" + + +def test_feedstock_context_git_repo_name(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert context.git_repo_name == "TEST-FEEDSTOCK-NAME-feedstock" + + +def test_feedstock_context_git_href(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) + assert ( + context.git_href + == "https://github.com/conda-forge/TEST-FEEDSTOCK-NAME-feedstock" + ) From 6ba48ceea9da476f3991ed9845ea20add1f48947 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 30 May 2024 16:08:36 +0200 Subject: [PATCH 06/38] add create_pull_request to git backend --- conda_forge_tick/git_utils.py | 157 +++++++- conda_forge_tick/models/common.py | 2 +- conda_forge_tick/models/pr_json.py | 24 +- tests/github_api/get_pull_pytest.json | 360 ++++++++++++++++++ tests/github_api/get_repo_pytest.json | 130 +++++++ tests/github_api/github_response_headers.json | 28 ++ tests/test_git_utils.py | 125 +++++- 7 files changed, 812 insertions(+), 14 deletions(-) create mode 100644 tests/github_api/get_pull_pytest.json create mode 100644 tests/github_api/get_repo_pytest.json create mode 100644 tests/github_api/github_response_headers.json diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 5a917dcd4..56f29bf2e 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -9,6 +9,7 @@ import time from abc import ABC, abstractmethod from datetime import datetime +from email import utils from functools import cached_property from pathlib import Path from typing import Dict, Literal, Optional, Tuple, Union @@ -20,7 +21,9 @@ import github3.pulls import github3.repos import requests +from github3.session import GitHubSession from requests.exceptions import RequestException, Timeout +from requests.structures import CaseInsensitiveDict from conda_forge_tick import sensitive_env @@ -30,6 +33,14 @@ from .contexts import FeedstockContext from .executors import lock_git_operation +from .models.pr_json import ( + GithubPullRequestBase, + GithubPullRequestMergeableState, + GithubRepository, + PullRequestData, + PullRequestInfoHead, + PullRequestState, +) from .os_utils import pushd from .utils import get_bot_run_url, run_command_hiding_token @@ -52,7 +63,7 @@ # these keys are kept from github PR json blobs # to add more keys to keep, put them in the right spot in the dict and -# set them to None. Also add them to the PullRequestInfo Pydantic model! +# set them to None. Also add them to the PullRequestData Pydantic model! PR_KEYS_TO_KEEP = { "ETag": None, "Last-Modified": None, @@ -117,6 +128,18 @@ class GitConnectionMode(enum.StrEnum): class GitCliError(Exception): + """ + A generic error that occurred while running a git CLI command. + """ + + pass + + +class GitPlatformError(Exception): + """ + A generic error that occurred while interacting with a git platform. + """ + pass @@ -517,6 +540,58 @@ def is_api_limit_reached(self) -> bool: """ return self.get_api_requests_left() in (0, None) + @abstractmethod + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + """ + Create a pull request from a forked repository. It is assumed that the forked repository is owned by the + current user and has the same name as the target repository. + + :param target_owner: The owner of the target repository. + :param target_repo: The name of the target repository. + :param base_branch: The base branch of the pull request, located in the target repository. + :param head_branch: The head branch of the pull request, located in the forked repository. + :param title: The title of the pull request. + :param body: The body of the pull request. + + :returns: The data of the created pull request. + + :raises GitPlatformError: If the pull request could not be created. + """ + pass + + +class _Github3SessionWrapper: + """ + This is a wrapper around the github3.session.GitHubSession that allows us to intercept the response headers. + """ + + def __init__(self, session: GitHubSession): + super().__init__() + self._session = session + self.last_response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict() + + def __getattr__(self, item): + return getattr(self._session, item) + + def _forward_request(self, method, *args, **kwargs): + response = method(*args, **kwargs) + self.last_response_headers = copy.deepcopy(response.headers) + return response + + def post(self, *args, **kwargs): + return self._forward_request(self._session.post, *args, **kwargs) + + def get(self, *args, **kwargs): + return self._forward_request(self._session.get, *args, **kwargs) + class GitHubBackend(GitPlatformBackend): """ @@ -533,8 +608,17 @@ class GitHubBackend(GitPlatformBackend): """ def __init__(self, github3_client: github3.GitHub, pygithub_client: github.Github): + """ + Create a new GitHubBackend. + + Note: Because we need additional response headers, we wrap the github3 session of the github3 client + with our own session wrapper and replace the github3 client's session with it. + """ super().__init__(GitCli()) self.github3_client = github3_client + self._github3_session = _Github3SessionWrapper(self.github3_client.session) + self.github3_client.session = self._github3_session + self.pygithub_client = pygithub_client @classmethod @@ -624,6 +708,38 @@ def get_api_requests_left(self) -> int | None: return remaining_limit + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + repo: github3.repos.Repository = self.github3_client.repository( + target_owner, target_repo + ) + + response: github3.pulls.ShortPullRequest | None = repo.create_pull( + title=title, + base=base_branch, + head=f"{self.user}:{head_branch}", + body=body, + ) + + if response is None: + raise GitPlatformError("Could not create pull request.") + + # fields like ETag and Last-Modified are stored in the response headers, we need to extract them + header_fields = { + k: self._github3_session.last_response_headers[k] + for k in PullRequestData.HEADER_FIELDS + } + + # note: this ignores extra fields in the response + return PullRequestData.model_validate(response.as_dict() | header_fields) + class DryRunBackend(GitPlatformBackend): """ @@ -671,6 +787,45 @@ def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): def user(self) -> str: return self._USER + def create_pull_request( + self, + target_owner: str, + target_repo: str, + base_branch: str, + head_branch: str, + title: str, + body: str, + ) -> PullRequestData: + logger.debug( + f"==============================================================" + f"Dry Run: Create Pull Request" + f'Title: "{title}"' + f"Target Repository: {target_owner}/{target_repo}" + f"Branches: {self.user}:{head_branch} -> {target_owner}:{base_branch}" + f"Body:" + f"{body}" + f"==============================================================" + ) + + now = datetime.now() + return PullRequestData.model_validate( + { + "ETag": "GITHUB_PR_ETAG", + "Last-Modified": utils.format_datetime(now), + "id": 13371337, + "html_url": f"https://github.com/{target_owner}/{target_repo}/pulls/1337", + "created_at": now, + "mergeable_state": GithubPullRequestMergeableState.CLEAN, + "mergeable": True, + "merged": False, + "draft": False, + "number": 1337, + "state": PullRequestState.OPEN, + "head": PullRequestInfoHead(ref=head_branch), + "base": GithubPullRequestBase(repo=GithubRepository(name=target_repo)), + } + ) + def github_backend() -> GitHubBackend: """ diff --git a/conda_forge_tick/models/common.py b/conda_forge_tick/models/common.py index beb445213..b07e33f8f 100644 --- a/conda_forge_tick/models/common.py +++ b/conda_forge_tick/models/common.py @@ -25,7 +25,7 @@ class StrictBaseModel(BaseModel): class ValidatedBaseModel(BaseModel): - model_config = ConfigDict(validate_assignment=True, extra="allow") + model_config = ConfigDict(validate_assignment=True, extra="ignore") def before_validator_ensure_dict(value: Any) -> dict: diff --git a/conda_forge_tick/models/pr_json.py b/conda_forge_tick/models/pr_json.py index 261b3e8d7..db7b4789f 100644 --- a/conda_forge_tick/models/pr_json.py +++ b/conda_forge_tick/models/pr_json.py @@ -1,11 +1,15 @@ from datetime import datetime from enum import StrEnum -from typing import Literal +from typing import ClassVar, Literal from pydantic import UUID4, AnyHttpUrl, Field, TypeAdapter from pydantic_extra_types.color import Color -from conda_forge_tick.models.common import RFC2822Date, StrictBaseModel +from conda_forge_tick.models.common import ( + RFC2822Date, + StrictBaseModel, + ValidatedBaseModel, +) class PullRequestLabelShort(StrictBaseModel): @@ -48,7 +52,7 @@ class PullRequestState(StrEnum): """ -class PullRequestInfoHead(StrictBaseModel): +class PullRequestInfoHead(ValidatedBaseModel): ref: str """ The head branch of the pull request. @@ -73,15 +77,15 @@ class GithubPullRequestMergeableState(StrEnum): CLEAN = "clean" -class GithubRepository(StrictBaseModel): +class GithubRepository(ValidatedBaseModel): name: str -class GithubPullRequestBase(StrictBaseModel): +class GithubPullRequestBase(ValidatedBaseModel): repo: GithubRepository -class PullRequestDataValid(StrictBaseModel): +class PullRequestDataValid(ValidatedBaseModel): """ Information about a pull request, as retrieved from the GitHub API. Refer to git_utils.PR_KEYS_TO_KEEP for the keys that are kept in the PR object. @@ -89,6 +93,14 @@ class PullRequestDataValid(StrictBaseModel): GitHub documentation: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#get-a-pull-request """ + HEADER_FIELDS: ClassVar[set[str]] = { + "ETag", + "Last-Modified", + } + """ + A set of all header fields that are stored in the PR object. + """ + e_tag: str | None = Field(None, alias="ETag") """ HTTP ETag header field, allowing us to quickly check if the PR has changed. diff --git a/tests/github_api/get_pull_pytest.json b/tests/github_api/get_pull_pytest.json new file mode 100644 index 000000000..ba8b9902e --- /dev/null +++ b/tests/github_api/get_pull_pytest.json @@ -0,0 +1,360 @@ +{ + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919", + "id": 1853804278, + "node_id": "PR_kwDOAgM_Js5ufs72", + "html_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919", + "diff_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919.diff", + "patch_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919.patch", + "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919", + "number": 1919, + "state": "open", + "locked": false, + "title": "PR_TITLE", + "user": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "body": "PR_BODY", + "created_at": "2024-05-03T17:04:20Z", + "updated_at": "2024-05-27T13:31:50Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "351d0b862d129b53b8c7db2260d208d3a27fb204", + "assignee": null, + "assignees": [], + "requested_reviewers": [ + ], + "requested_teams": [], + "labels": [], + "milestone": null, + "draft": false, + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/commits", + "review_comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/comments", + "review_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919/comments", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095", + "head": { + "label": "regro-cf-autotick-bot:HEAD_BRANCH", + "ref": "HEAD_BRANCH", + "sha": "0eaa1de035b8720c8b85a7f435a0ae7037fe9095", + "user": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "repo": { + "id": 772632103, + "node_id": "R_kgDOLg1uJw", + "name": "pytest-feedstock", + "full_name": "regro-cf-autotick-bot/pytest-feedstock", + "private": false, + "owner": { + "login": "regro-cf-autotick-bot", + "id": 12345678, + "node_id": "MDQ6VXNlcjI1OTA2Mjcw", + "avatar_url": "https://avatars.githubusercontent.com/u/12345678?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock", + "description": "The tool for managing conda-forge feedstocks.", + "fork": true, + "url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock", + "forks_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/regro-cf-autotick-bot/pytest-feedstock/deployments", + "created_at": "2024-03-15T15:21:35Z", + "updated_at": "2024-05-16T15:11:20Z", + "pushed_at": "2024-05-16T15:20:39Z", + "git_url": "git://github.com/regro-cf-autotick-bot/pytest-feedstock.git", + "ssh_url": "git@github.com:regro-cf-autotick-bot/pytest-feedstock.git", + "clone_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git", + "svn_url": "https://github.com/regro-cf-autotick-bot/pytest-feedstock", + "homepage": "https://conda-forge.org/", + "size": 3959, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Python", + "has_issues": false, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "main" + } + }, + "base": { + "label": "conda-forge:main", + "ref": "main", + "sha": "59aa8df51b362904f0a8eb72274ad9458a5e4d8e", + "user": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 33767206, + "node_id": "MDEwOlJlcG9zaXRvcnkzMzc2NzIwNg==", + "name": "pytest-feedstock", + "full_name": "conda-forge/pytest-feedstock", + "private": false, + "owner": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/conda-forge/pytest-feedstock", + "description": "The tool for managing conda-forge feedstocks.", + "fork": false, + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock", + "forks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/deployments", + "created_at": "2015-04-11T07:38:36Z", + "updated_at": "2024-05-28T09:55:10Z", + "pushed_at": "2024-05-28T09:55:05Z", + "git_url": "git://github.com/conda-forge/pytest-feedstock.git", + "ssh_url": "git@github.com:conda-forge/pytest-feedstock.git", + "clone_url": "https://github.com/conda-forge/pytest-feedstock.git", + "svn_url": "https://github.com/conda-forge/pytest-feedstock", + "homepage": "https://conda-forge.org/", + "size": 3808, + "stargazers_count": 147, + "watchers_count": 147, + "language": "Python", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 166, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 334, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [ + "continuous-integration", + "hacktoberfest" + ], + "visibility": "public", + "forks": 166, + "open_issues": 334, + "watchers": 147, + "default_branch": "main" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919" + }, + "html": { + "href": "https://github.com/conda-forge/pytest-feedstock/pull/1919" + }, + "issue": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919" + }, + "comments": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095" + } + }, + "author_association": "MEMBER", + "auto_merge": null, + "body_html": "BODY_HTML", + "body_text": "BODY_TEXT", + "active_lock_reason": null, + "merged": false, + "mergeable": true, + "rebaseable": true, + "mergeable_state": "clean", + "merged_by": null, + "comments": 7, + "review_comments": 23, + "maintainer_can_modify": true, + "commits": 8, + "additions": 601, + "deletions": 752, + "changed_files": 32 +} diff --git a/tests/github_api/get_repo_pytest.json b/tests/github_api/get_repo_pytest.json new file mode 100644 index 000000000..4d4c25dc4 --- /dev/null +++ b/tests/github_api/get_repo_pytest.json @@ -0,0 +1,130 @@ +{ + "id": 62477336, + "node_id": "MDEwOlJlcG9zaXRvcnk2MjQ3NzMzNg==", + "name": "pytest-feedstock", + "full_name": "conda-forge/pytest-feedstock", + "private": false, + "owner": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/conda-forge/pytest-feedstock", + "description": "A conda-smithy repository for pytest.", + "fork": false, + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock", + "forks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/forks", + "keys_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/teams", + "hooks_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/hooks", + "issue_events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/events{/number}", + "events_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/events", + "assignees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/assignees{/user}", + "branches_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/branches{/branch}", + "tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/tags", + "blobs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/{sha}", + "languages_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/languages", + "stargazers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/stargazers", + "contributors_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contributors", + "subscribers_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscribers", + "subscription_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/subscription", + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/contents/{+path}", + "compare_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/merges", + "archive_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/downloads", + "issues_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues{/number}", + "pulls_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls{/number}", + "milestones_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/milestones{/number}", + "notifications_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/labels{/name}", + "releases_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/releases{/id}", + "deployments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/deployments", + "created_at": "2016-07-03T02:02:04Z", + "updated_at": "2024-05-20T16:07:25Z", + "pushed_at": "2024-05-20T16:07:21Z", + "git_url": "git://github.com/conda-forge/pytest-feedstock.git", + "ssh_url": "git@github.com:conda-forge/pytest-feedstock.git", + "clone_url": "https://github.com/conda-forge/pytest-feedstock.git", + "svn_url": "https://github.com/conda-forge/pytest-feedstock", + "homepage": null, + "size": 310, + "stargazers_count": 2, + "watchers_count": 2, + "language": null, + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": false, + "has_pages": false, + "has_discussions": false, + "forks_count": 27, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 1, + "license": { + "key": "bsd-3-clause", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "spdx_id": "BSD-3-Clause", + "url": "https://api.github.com/licenses/bsd-3-clause", + "node_id": "MDc6TGljZW5zZTU=" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 27, + "open_issues": 1, + "watchers": 2, + "default_branch": "main", + "temp_clone_token": null, + "custom_properties": {}, + "organization": { + "login": "conda-forge", + "id": 11897326, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODk3MzI2", + "avatar_url": "https://avatars.githubusercontent.com/u/11897326?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/conda-forge", + "html_url": "https://github.com/conda-forge", + "followers_url": "https://api.github.com/users/conda-forge/followers", + "following_url": "https://api.github.com/users/conda-forge/following{/other_user}", + "gists_url": "https://api.github.com/users/conda-forge/gists{/gist_id}", + "starred_url": "https://api.github.com/users/conda-forge/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/conda-forge/subscriptions", + "organizations_url": "https://api.github.com/users/conda-forge/orgs", + "repos_url": "https://api.github.com/users/conda-forge/repos", + "events_url": "https://api.github.com/users/conda-forge/events{/privacy}", + "received_events_url": "https://api.github.com/users/conda-forge/received_events", + "type": "Organization", + "site_admin": false + }, + "network_count": 27, + "subscribers_count": 7 +} diff --git a/tests/github_api/github_response_headers.json b/tests/github_api/github_response_headers.json new file mode 100644 index 000000000..4ba208057 --- /dev/null +++ b/tests/github_api/github_response_headers.json @@ -0,0 +1,28 @@ +{ + "Server": "GitHub.com", + "Date": "Wed, 29 May 2024 12:07:38 GMT", + "Content-Type": "application/json; charset=utf-8", + "Cache-Control": "public, max-age=60, s-maxage=60", + "Vary": "Accept, Accept-Encoding, Accept, X-Requested-With", + "ETag": "W/\"7ba8c0b529b1303243a8c4636a95ce2e337591d152d69f8e90608c202a166483\"", + "Last-Modified": "Wed, 10 Apr 2024 13:15:22 GMT", + "X-GitHub-Media-Type": "github.v3; format=json", + "x-github-api-version-selected": "2022-11-28", + "Access-Control-Expose-Headers": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset", + "Access-Control-Allow-Origin": "*", + "Strict-Transport-Security": "max-age=31536000; includeSubdomains; preload", + "X-Frame-Options": "deny", + "X-Content-Type-Options": "nosniff", + "X-XSS-Protection": "0", + "Referrer-Policy": "origin-when-cross-origin, strict-origin-when-cross-origin", + "Content-Security-Policy": "default-src 'none'", + "Content-Encoding": "gzip", + "X-RateLimit-Limit": "60", + "X-RateLimit-Remaining": "51", + "X-RateLimit-Reset": "1716987025", + "X-RateLimit-Resource": "core", + "X-RateLimit-Used": "9", + "Accept-Ranges": "bytes", + "Content-Length": "4456", + "X-GitHub-Request-Id": "F7B3:2D05B1:3948EFC:3998343:74382A8F" +} diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 1884d42fc..1ec296b09 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1,3 +1,6 @@ +import datetime +import json +import logging import subprocess import tempfile from pathlib import Path @@ -6,6 +9,9 @@ import github3.exceptions import pytest +import requests +from pydantic_core import Url +from requests.structures import CaseInsensitiveDict from conda_forge_tick.git_utils import ( Bound, @@ -18,6 +24,10 @@ RepositoryNotFoundError, trim_pr_json_keys, ) +from conda_forge_tick.models.pr_json import ( + GithubPullRequestMergeableState, + PullRequestState, +) """ Note: You have to have git installed on your machine to run these tests. @@ -553,7 +563,7 @@ def test_git_cli_clone_fork_and_branch_mock( fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) cli = GitCli() @@ -642,7 +652,7 @@ def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(c new_branch = "NEW_BRANCH" cli = GitCli() - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) with tempfile.TemporaryDirectory() as tmpdir: dir_path = Path(tmpdir) / "duckdb-feedstock" @@ -721,7 +731,7 @@ def test_github_backend_fork_exists( branch_already_synced: bool, caplog, ): - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) exists_mock.return_value = True user_mock.return_value = "USER" @@ -862,7 +872,73 @@ def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): assert backend.get_api_requests_left() == 0 github3_client.rate_limit.assert_called_once() - assert f"will reset at {reset_timestamp_str}" in caplog.text # + assert f"will reset at {reset_timestamp_str}" in caplog.text + + +@mock.patch("requests.Session.request") +def test_github_backend_create_pull_request_mock(request_mock: MagicMock): + github3_client = github3.login(token="TOKEN") + + with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: + get_repo_response = json.load(f) + + with open( + Path(__file__).parent / "github_api" / "github_response_headers.json" + ) as f: + response_headers = json.load(f) + + with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: + create_pull_response = json.load(f) + + def request_side_effect(method, url, **kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: get_repo_response + return response + if method == "POST": + response.status_code = 201 + response.json = lambda: create_pull_response + response.headers = CaseInsensitiveDict(response_headers) + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3_client, MagicMock()) + + pr_data = backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) + + assert pr_data.base is not None + assert pr_data.base.repo.name == "pytest-feedstock" + assert pr_data.closed_at is None + assert pr_data.created_at is not None + assert pr_data.created_at == datetime.datetime( + 2024, 5, 3, 17, 4, 20, tzinfo=datetime.timezone.utc + ) + assert pr_data.head is not None + assert pr_data.head.ref == "HEAD_BRANCH" + assert pr_data.html_url == Url( + "https://github.com/conda-forge/pytest-feedstock/pull/1919" + ) + assert pr_data.id == 1853804278 + assert pr_data.labels == [] + assert pr_data.mergeable is True + assert pr_data.mergeable_state == GithubPullRequestMergeableState.CLEAN + assert pr_data.merged is False + assert pr_data.merged_at is None + assert pr_data.number == 1919 + assert pr_data.state == PullRequestState.OPEN + assert pr_data.updated_at == datetime.datetime( + 2024, 5, 27, 13, 31, 50, tzinfo=datetime.timezone.utc + ) @pytest.mark.parametrize( @@ -923,7 +999,7 @@ def test_dry_run_backend_does_repository_exist_other_repo(): def test_dry_run_backend_fork(caplog): - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) backend = DryRunBackend() @@ -941,7 +1017,7 @@ def test_dry_run_backend_fork(caplog): def test_dry_run_backend_sync_default_branch(caplog): - caplog.set_level("DEBUG") + caplog.set_level(logging.DEBUG) backend = DryRunBackend() @@ -956,6 +1032,43 @@ def test_dry_run_backend_user(): assert backend.user == "auto-tick-bot-dry-run" +def test_dry_run_backend_create_pull_request(caplog): + backend = DryRunBackend() + caplog.set_level(logging.DEBUG) + + pr_data = backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY_TEXT", + ) + + # caplog validation + assert "Create Pull Request" in caplog.text + assert 'Title: "TITLE"' in caplog.text + assert "Target Repository: conda-forge/pytest-feedstock" in caplog.text + assert ( + f"Branches: {backend.user}:HEAD_BRANCH -> conda-forge:BASE_BRANCH" + in caplog.text + ) + assert "BODY_TEXT" in caplog.text + + # pr_data validation + assert pr_data.e_tag == "GITHUB_PR_ETAG" + assert pr_data.last_modified is not None + assert pr_data.id == 13371337 + assert pr_data.html_url == Url( + "https://github.com/conda-forge/pytest-feedstock/pulls/1337" + ) + assert pr_data.created_at is not None + assert pr_data.number == 1337 + assert pr_data.state == PullRequestState.OPEN + assert pr_data.head.ref == "HEAD_BRANCH" + assert pr_data.base.repo.name == "pytest-feedstock" + + def test_trim_pr_json_keys(): pr_json = { "ETag": "blah", From 289d404ed94b623e1881829da53994bc3fa2e9a9 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 30 May 2024 17:04:42 +0200 Subject: [PATCH 07/38] git add and git commit --- conda_forge_tick/git_utils.py | 58 +++++++++++---------- tests/test_git_utils.py | 98 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 27 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 56f29bf2e..e5c527533 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -191,6 +191,37 @@ def _run_git_command( except subprocess.CalledProcessError as e: raise GitCliError("Error running git command.") from e + @lock_git_operation() + def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): + """ + Add files to the git index with `git add`. + + :param git_dir: The directory of the git repository. + :param pathspec: The files to add. + :param all_: If True, not only add the files in pathspec, but also where the index already has an entry. + If _all is set with empty pathspec, all files in the entire working tree are updated. + :raises ValueError: If pathspec is empty and all_ is False. + :raises GitCliError: If the git command fails. + """ + if not pathspec and not all_: + raise ValueError("Either pathspec or all_ must be set.") + + all_arg = ["--all"] if all_ else [] + + self._run_git_command(["add", *all_arg, *pathspec], git_dir) + + @lock_git_operation() + def commit(self, git_dir: Path, message: str, all_: bool = False): + """ + Commit changes to the git repository with `git commit`. + :param git_dir: The directory of the git repository. + :param message: The commit message. + :param all_: Automatically stage files that have been modified and deleted, but new files are not affected. + :raises GitCliError: If the git command fails. + """ + all_arg = ["-a"] if all_ else [] + self._run_git_command(["commit", *all_arg, "-m", message], git_dir) + @lock_git_operation() def reset_hard(self, git_dir: Path, to_treeish: str = "HEAD"): """ @@ -846,33 +877,6 @@ def is_github_api_limit_reached() -> bool: return backend.is_api_limit_reached() -def feedstock_url(fctx: FeedstockContext, protocol: str = "ssh") -> str: - """Returns the URL for a conda-forge feedstock.""" - feedstock = fctx.feedstock_name + "-feedstock" - if feedstock.startswith("http://github.com/"): - return feedstock - elif feedstock.startswith("https://github.com/"): - return feedstock - elif feedstock.startswith("git@github.com:"): - return feedstock - protocol = protocol.lower() - if protocol == "http": - url = "http://github.com/conda-forge/" + feedstock + ".git" - elif protocol == "https": - url = "https://github.com/conda-forge/" + feedstock + ".git" - elif protocol == "ssh": - url = "git@github.com:conda-forge/" + feedstock + ".git" - else: - msg = f"Unrecognized github protocol {protocol}, must be ssh, http, or https." - raise ValueError(msg) - return url - - -def feedstock_repo(fctx: FeedstockContext) -> str: - """Gets the name of the feedstock repository.""" - return fctx.feedstock_name + "-feedstock" - - @lock_git_operation() def get_repo( fctx: FeedstockContext, diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 1ec296b09..b721b44d1 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -107,6 +107,104 @@ def init_temp_git_repo(git_dir: Path, bare: bool = False): ) +@pytest.mark.parametrize( + "n_paths,all_", [(0, True), (1, False), (1, True), (2, False), (2, True)] +) +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_add_success_mock( + run_git_command_mock: MagicMock, n_paths: int, all_: bool +): + cli = GitCli() + + git_dir = Path("TEST_DIR") + paths = [Path(f"test{i}.txt") for i in range(n_paths)] + + cli.add(git_dir, *paths, all_=all_) + + expected_all_arg = ["--all"] if all_ else [] + + run_git_command_mock.assert_called_once_with( + ["add", *expected_all_arg, *paths], git_dir + ) + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_add_no_arguments_error(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + with pytest.raises(ValueError, match="Either pathspec or all_ must be set"): + cli.add(git_dir) + + run_git_command_mock.assert_not_called() + + +@pytest.mark.parametrize( + "n_paths,all_", [(0, True), (1, False), (1, True), (2, False), (2, True)] +) +def test_git_cli_add_success(n_paths: int, all_: bool): + with tempfile.TemporaryDirectory() as tmp_dir: + git_dir = Path(tmp_dir) + init_temp_git_repo(git_dir) + + pathspec = [git_dir / f"test{i}.txt" for i in range(n_paths)] + + for path in pathspec + [git_dir / "all_tracker.txt"]: + path.touch() + + cli = GitCli() + cli.add(git_dir, *pathspec, all_=all_) + + tracked_files = cli._run_git_command( + ["ls-files", "-s"], git_dir, capture_text=True + ).stdout + + for path in pathspec: + assert path.name in tracked_files + + if all_ and n_paths == 0: + # note that n_paths has to be zero to add unknown files to the working tree + assert "all_tracker.txt" in tracked_files + else: + assert "all_tracker.txt" not in tracked_files + + +@pytest.mark.parametrize("all_", [True, False]) +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_commit_success_mock(run_git_command_mock: MagicMock, all_: bool): + git_dir = Path("GIT_DIR") + message = "COMMIT_MESSAGE" + + cli = GitCli() + cli.commit(git_dir, message, all_) + + expected_all_arg = ["-a"] if all_ else [] + + run_git_command_mock.assert_called_once_with( + ["commit", *expected_all_arg, "-m", message], git_dir + ) + + +@pytest.mark.parametrize("all_", [True, False]) +def test_git_cli_commit_success(all_: bool): + with tempfile.TemporaryDirectory() as tmp_dir: + git_dir = Path(tmp_dir) + init_temp_git_repo(git_dir) + + cli = GitCli() + + with git_dir.joinpath("test.txt").open("w") as f: + f.write("Hello, World!") + + cli.add(git_dir, git_dir / "test.txt") + cli.commit(git_dir, "Add Test") + + git_log = cli._run_git_command(["log"], git_dir, capture_text=True).stdout + + assert "Add Test" in git_log + + def test_git_cli_reset_hard_already_reset(): cli = GitCli() with tempfile.TemporaryDirectory() as tmpdir: From 44420f859218137bcf7eabf77a7f0ddfbb783584 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 30 May 2024 17:22:52 +0200 Subject: [PATCH 08/38] add --allow-empty to git --- conda_forge_tick/git_utils.py | 11 +++++++++-- tests/test_git_utils.py | 30 ++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index e5c527533..a538bf4b5 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -211,16 +211,23 @@ def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): self._run_git_command(["add", *all_arg, *pathspec], git_dir) @lock_git_operation() - def commit(self, git_dir: Path, message: str, all_: bool = False): + def commit( + self, git_dir: Path, message: str, all_: bool = False, allow_empty: bool = False + ): """ Commit changes to the git repository with `git commit`. :param git_dir: The directory of the git repository. :param message: The commit message. + :param allow_empty: If True, allow an empty commit. :param all_: Automatically stage files that have been modified and deleted, but new files are not affected. :raises GitCliError: If the git command fails. """ all_arg = ["-a"] if all_ else [] - self._run_git_command(["commit", *all_arg, "-m", message], git_dir) + allow_empty_arg = ["--allow-empty"] if allow_empty else [] + + self._run_git_command( + ["commit", *all_arg, *allow_empty_arg, "-m", message], git_dir + ) @lock_git_operation() def reset_hard(self, git_dir: Path, to_treeish: str = "HEAD"): diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index b721b44d1..2b2808608 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -170,36 +170,54 @@ def test_git_cli_add_success(n_paths: int, all_: bool): assert "all_tracker.txt" not in tracked_files +@pytest.mark.parametrize("allow_empty", [True, False]) @pytest.mark.parametrize("all_", [True, False]) @mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") -def test_git_cli_commit_success_mock(run_git_command_mock: MagicMock, all_: bool): +def test_git_cli_commit_success_mock( + run_git_command_mock: MagicMock, all_: bool, allow_empty: bool +): git_dir = Path("GIT_DIR") message = "COMMIT_MESSAGE" cli = GitCli() - cli.commit(git_dir, message, all_) + cli.commit(git_dir, message, all_, allow_empty) expected_all_arg = ["-a"] if all_ else [] + expected_allow_empty_arg = ["--allow-empty"] if allow_empty else [] run_git_command_mock.assert_called_once_with( - ["commit", *expected_all_arg, "-m", message], git_dir + ["commit", *expected_all_arg, *expected_allow_empty_arg, "-m", message], git_dir ) +@pytest.mark.parametrize("allow_empty", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) @pytest.mark.parametrize("all_", [True, False]) -def test_git_cli_commit_success(all_: bool): +def test_git_cli_commit(all_: bool, empty: bool, allow_empty: bool): with tempfile.TemporaryDirectory() as tmp_dir: git_dir = Path(tmp_dir) init_temp_git_repo(git_dir) cli = GitCli() - with git_dir.joinpath("test.txt").open("w") as f: + test_file = git_dir.joinpath("test.txt") + with test_file.open("w") as f: f.write("Hello, World!") - cli.add(git_dir, git_dir / "test.txt") cli.commit(git_dir, "Add Test") + if not empty: + test_file.unlink() + if not all_: + cli.add(git_dir, git_dir / "test.txt") + + if empty and not allow_empty: + with pytest.raises(GitCliError): + cli.commit(git_dir, "Add Test", all_, allow_empty) + return + + cli.commit(git_dir, "Add Test", all_, allow_empty) + git_log = cli._run_git_command(["log"], git_dir, capture_text=True).stdout assert "Add Test" in git_log From 79a9f38c79e2464b0e5195129898c8a4ed044850 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Fri, 31 May 2024 15:00:58 +0200 Subject: [PATCH 09/38] add comment_on_pull_request --- conda_forge_tick/git_utils.py | 82 ++++++- .../create_issue_comment_pytest.json | 33 +++ tests/github_api/get_pull_pytest.json | 30 +-- tests/test_git_utils.py | 215 +++++++++++++++++- 4 files changed, 330 insertions(+), 30 deletions(-) create mode 100644 tests/github_api/create_issue_comment_pytest.json diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index a538bf4b5..90406e077 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -5,6 +5,7 @@ import logging import math import subprocess +import textwrap import threading import time from abc import ABC, abstractmethod @@ -605,6 +606,22 @@ def create_pull_request( """ pass + @abstractmethod + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ) -> None: + """ + Comment on an existing pull request. + :param repo_owner: The owner of the repository. + :param repo_name: The name of the repository. + :param pr_number: The number of the pull request. + :param comment: The comment to post. + + :raises RepositoryNotFoundError: If the repository does not exist. + :raises GitPlatformError: If the comment could not be posted, including if the pull request does not exist. + """ + pass + class _Github3SessionWrapper: """ @@ -778,6 +795,30 @@ def create_pull_request( # note: this ignores extra fields in the response return PullRequestData.model_validate(response.as_dict() | header_fields) + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ) -> None: + try: + repo = self.github3_client.repository(repo_owner, repo_name) + except github3.exceptions.NotFoundError as e: + raise RepositoryNotFoundError( + f"Repository {repo_owner}/{repo_name} not found." + ) from e + + try: + pr = repo.pull_request(pr_number) + except github3.exceptions.NotFoundError as e: + raise GitPlatformError( + f"Pull request {repo_owner}/{repo_name}#{pr_number} not found." + ) from e + + try: + pr.create_comment(comment) + except github3.GitHubError as e: + raise GitPlatformError( + f"Could not comment on pull request {repo_owner}/{repo_name}#{pr_number}." + ) from e + class DryRunBackend(GitPlatformBackend): """ @@ -835,14 +876,18 @@ def create_pull_request( body: str, ) -> PullRequestData: logger.debug( - f"==============================================================" - f"Dry Run: Create Pull Request" - f'Title: "{title}"' - f"Target Repository: {target_owner}/{target_repo}" - f"Branches: {self.user}:{head_branch} -> {target_owner}:{base_branch}" - f"Body:" - f"{body}" - f"==============================================================" + textwrap.dedent( + f""" + ============================================================== + Dry Run: Create Pull Request + Title: "{title}" + Target Repository: {target_owner}/{target_repo} + Branches: {self.user}:{head_branch} -> {target_owner}:{base_branch} + Body: + {body} + ============================================================== + """ + ) ) now = datetime.now() @@ -864,6 +909,27 @@ def create_pull_request( } ) + def comment_on_pull_request( + self, repo_owner: str, repo_name: str, pr_number: int, comment: str + ): + if not self.does_repository_exist(repo_owner, repo_name): + raise RepositoryNotFoundError( + f"Repository {repo_owner}/{repo_name} not found." + ) + + logger.debug( + textwrap.dedent( + f""" + ============================================================== + Dry Run: Comment on Pull Request + Pull Request: {repo_owner}/{repo_name}#{pr_number} + Comment: + {comment} + ============================================================== + """ + ) + ) + def github_backend() -> GitHubBackend: """ diff --git a/tests/github_api/create_issue_comment_pytest.json b/tests/github_api/create_issue_comment_pytest.json new file mode 100644 index 000000000..ba97b19f9 --- /dev/null +++ b/tests/github_api/create_issue_comment_pytest.json @@ -0,0 +1,33 @@ +{ + "id": 1, + "node_id": "MDEyOklzc3VlQ29tbWVudDE=", + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/comments/1", + "html_url": "https://github.com/conda-forge/pytest-feedstock/issues/1337#issuecomment-1", + "body": "ISSUE_COMMENT_BODY", + "body_html": "ISSUE_COMMENT_BODY_HTML", + "body_text": "ISSUE_COMMENT_BODY_TEXT", + "user": { + "login": "regro-cf-autotick-bot", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/regro-cf-autotick-bot", + "html_url": "https://github.com/regro-cf-autotick-bot", + "followers_url": "https://api.github.com/users/regro-cf-autotick-bot/followers", + "following_url": "https://api.github.com/users/regro-cf-autotick-bot/following{/other_user}", + "gists_url": "https://api.github.com/users/regro-cf-autotick-bot/gists{/gist_id}", + "starred_url": "https://api.github.com/users/regro-cf-autotick-bot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/regro-cf-autotick-bot/subscriptions", + "organizations_url": "https://api.github.com/users/regro-cf-autotick-bot/orgs", + "repos_url": "https://api.github.com/users/regro-cf-autotick-bot/repos", + "events_url": "https://api.github.com/users/regro-cf-autotick-bot/events{/privacy}", + "received_events_url": "https://api.github.com/users/regro-cf-autotick-bot/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2011-04-14T16:00:49Z", + "updated_at": "2011-04-14T16:00:49Z", + "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337", + "author_association": "CONTRIBUTOR" +} diff --git a/tests/github_api/get_pull_pytest.json b/tests/github_api/get_pull_pytest.json index ba8b9902e..9bb5754db 100644 --- a/tests/github_api/get_pull_pytest.json +++ b/tests/github_api/get_pull_pytest.json @@ -1,12 +1,12 @@ { - "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919", + "url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337", "id": 1853804278, "node_id": "PR_kwDOAgM_Js5ufs72", - "html_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919", - "diff_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919.diff", - "patch_url": "https://github.com/conda-forge/pytest-feedstock/pull/1919.patch", - "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919", - "number": 1919, + "html_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337", + "diff_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337.diff", + "patch_url": "https://github.com/conda-forge/pytest-feedstock/pull/1337.patch", + "issue_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337", + "number": 1337, "state": "open", "locked": false, "title": "PR_TITLE", @@ -44,10 +44,10 @@ "labels": [], "milestone": null, "draft": false, - "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/commits", - "review_comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/comments", + "commits_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/commits", + "review_comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/comments", "review_comment_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}", - "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919/comments", + "comments_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments", "statuses_url": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095", "head": { "label": "regro-cf-autotick-bot:HEAD_BRANCH", @@ -316,25 +316,25 @@ }, "_links": { "self": { - "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919" + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" }, "html": { - "href": "https://github.com/conda-forge/pytest-feedstock/pull/1919" + "href": "https://github.com/conda-forge/pytest-feedstock/pull/1337" }, "issue": { - "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919" + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337" }, "comments": { - "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1919/comments" + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" }, "review_comments": { - "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/comments" + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/comments" }, "review_comment": { "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/comments{/number}" }, "commits": { - "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1919/commits" + "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337/commits" }, "statuses": { "href": "https://api.github.com/repos/conda-forge/pytest-feedstock/statuses/0eaa1de035b8720c8b85a7f435a0ae7037fe9095" diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 2b2808608..2b4c737a2 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -21,6 +21,7 @@ GitConnectionMode, GitHubBackend, GitPlatformBackend, + GitPlatformError, RepositoryNotFoundError, trim_pr_json_keys, ) @@ -993,8 +994,6 @@ def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): @mock.patch("requests.Session.request") def test_github_backend_create_pull_request_mock(request_mock: MagicMock): - github3_client = github3.login(token="TOKEN") - with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: get_repo_response = json.load(f) @@ -1006,7 +1005,7 @@ def test_github_backend_create_pull_request_mock(request_mock: MagicMock): with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: create_pull_response = json.load(f) - def request_side_effect(method, url, **kwargs): + def request_side_effect(method, _url, **_kwargs): response = requests.Response() if method == "GET": response.status_code = 200 @@ -1021,7 +1020,10 @@ def request_side_effect(method, url, **kwargs): request_mock.side_effect = request_side_effect - backend = GitHubBackend(github3_client, MagicMock()) + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock) pr_data = backend.create_pull_request( "conda-forge", @@ -1032,6 +1034,14 @@ def request_side_effect(method, url, **kwargs): "BODY", ) + request_mock.assert_called_with( + "POST", + "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls", + data='{"title": "TITLE", "body": "BODY", "base": "BASE_BRANCH", "head": "CURRENT_USER:HEAD_BRANCH"}', + json=None, + timeout=mock.ANY, + ) + assert pr_data.base is not None assert pr_data.base.repo.name == "pytest-feedstock" assert pr_data.closed_at is None @@ -1042,7 +1052,7 @@ def request_side_effect(method, url, **kwargs): assert pr_data.head is not None assert pr_data.head.ref == "HEAD_BRANCH" assert pr_data.html_url == Url( - "https://github.com/conda-forge/pytest-feedstock/pull/1919" + "https://github.com/conda-forge/pytest-feedstock/pull/1337" ) assert pr_data.id == 1853804278 assert pr_data.labels == [] @@ -1050,13 +1060,188 @@ def request_side_effect(method, url, **kwargs): assert pr_data.mergeable_state == GithubPullRequestMergeableState.CLEAN assert pr_data.merged is False assert pr_data.merged_at is None - assert pr_data.number == 1919 + assert pr_data.number == 1337 assert pr_data.state == PullRequestState.OPEN assert pr_data.updated_at == datetime.datetime( 2024, 5, 27, 13, 31, 50, tzinfo=datetime.timezone.utc ) +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_success(request_mock: MagicMock): + with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: + get_repo_response = json.load(f) + + with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: + get_pull_response = json.load(f) + + with open( + Path(__file__).parent / "github_api" / "create_issue_comment_pytest.json" + ) as f: + create_comment_response = json.load(f) + + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: get_repo_response + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 200 + response.json = lambda: get_pull_response + return response + if ( + method == "POST" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" + ): + response.status_code = 201 + response.json = lambda: create_comment_response + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + request_mock.assert_called_with( + "POST", + "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments", + data='{"body": "COMMENT"}', + json=None, + timeout=mock.ANY, + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_repo_not_found(request_mock: MagicMock): + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 404 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + + with pytest.raises(RepositoryNotFoundError): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_pull_request_not_found( + request_mock: MagicMock, +): + with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: + get_repo_response = json.load(f) + + def request_side_effect(method, url, **_kwargs): + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: get_repo_response + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 404 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + + with pytest.raises( + GitPlatformError, + match="Pull request conda-forge/pytest-feedstock#1337 not found", + ): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_unexpected_response( + request_mock: MagicMock, +): + with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: + get_repo_response = json.load(f) + + with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: + get_pull_response = json.load(f) + + def request_side_effect(method, url, **_kwargs): + # noinspection DuplicatedCode + response = requests.Response() + if ( + method == "GET" + and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" + ): + response.status_code = 200 + response.json = lambda: get_repo_response + return response + if ( + method == "GET" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" + ): + response.status_code = 200 + response.json = lambda: get_pull_response + return response + if ( + method == "POST" + and url + == "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" + ): + response.status_code = 500 + return response + assert False, f"Unexpected endpoint: {method} {url}" + + request_mock.side_effect = request_side_effect + + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + + with pytest.raises(GitPlatformError, match="Could not comment on pull request"): + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + @pytest.mark.parametrize( "backend", [GitHubBackend(MagicMock(), MagicMock()), DryRunBackend()] ) @@ -1169,7 +1354,7 @@ def test_dry_run_backend_create_pull_request(caplog): f"Branches: {backend.user}:HEAD_BRANCH -> conda-forge:BASE_BRANCH" in caplog.text ) - assert "BODY_TEXT" in caplog.text + assert "Body:\nBODY_TEXT" in caplog.text # pr_data validation assert pr_data.e_tag == "GITHUB_PR_ETAG" @@ -1185,6 +1370,22 @@ def test_dry_run_backend_create_pull_request(caplog): assert pr_data.base.repo.name == "pytest-feedstock" +def test_dry_run_backend_comment_on_pull_request(caplog): + backend = DryRunBackend() + caplog.set_level(logging.DEBUG) + + backend.comment_on_pull_request( + "conda-forge", + "pytest-feedstock", + 1337, + "COMMENT", + ) + + assert "Comment on Pull Request" in caplog.text + assert "Comment:\nCOMMENT" in caplog.text + assert "Pull Request: conda-forge/pytest-feedstock#1337" in caplog.text + + def test_trim_pr_json_keys(): pr_json = { "ETag": "blah", From 4a06d346cd2d2e017ab2a0cc07e50bde4e7eff05 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Fri, 31 May 2024 20:37:14 +0200 Subject: [PATCH 10/38] add rev-parse HEAD command --- conda_forge_tick/git_utils.py | 49 ++++++++++++++++++----------------- tests/test_git_utils.py | 30 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 90406e077..746aacc62 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -54,8 +54,6 @@ MAX_GITHUB_TIMEOUT = 60 -GIT_CLONE_DIR = "./feedstocks/" - BOT_RERUN_LABEL = { "name": "bot-rerun", } @@ -169,7 +167,9 @@ def _run_git_command( capture_text: bool = False, ) -> subprocess.CompletedProcess: """ - Run a git command. + Run a git command. stdout is only printed if the command fails. stderr is always printed. + If outputs are captured, they are never printed. + :param cmd: The command to run, as a list of strings. :param working_directory: The directory to run the command in. If None, the command will be run in the current working directory. @@ -183,14 +183,23 @@ def _run_git_command( git_command = ["git"] + cmd logger.debug(f"Running git command: {git_command}") + + stdout_args = {"stdout": subprocess.PIPE} if not capture_text else {} capture_args = {"capture_output": True, "text": True} if capture_text else {} try: return subprocess.run( - git_command, check=check_error, cwd=working_directory, **capture_args + git_command, + check=check_error, + cwd=working_directory, + **stdout_args, + **capture_args, ) except subprocess.CalledProcessError as e: - raise GitCliError("Error running git command.") from e + logger.info( + f"Command {git_command} failed. stdout:\n{e.stdout}\nend of stdout" + ) + raise GitCliError(f"Error running git command: {repr(e)}") from e @lock_git_operation() def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): @@ -230,6 +239,17 @@ def commit( ["commit", *all_arg, *allow_empty_arg, "-m", message], git_dir ) + def rev_parse_head(self, git_dir: Path) -> str: + """ + Get the commit hash of HEAD with `git rev-parse HEAD`. + :param git_dir: The directory of the git repository. + :return: The commit hash of HEAD. + :raises GitCliError: If the git command fails. + """ + ret = self._run_git_command(["rev-parse", "HEAD"], git_dir, capture_text=True) + + return ret.stdout.strip() + @lock_git_operation() def reset_hard(self, git_dir: Path, to_treeish: str = "HEAD"): """ @@ -974,25 +994,6 @@ def get_repo( repo : github3 repository The github3 repository object. """ - backend = github_backend() - - try: - backend.fork(fctx.git_repo_owner, fctx.git_repo_name) - except RepositoryNotFoundError: - logger.warning(f"Could not fork {fctx.git_repo_owner}/{fctx.git_repo_name}") - with fctx.attrs["pr_info"] as pri: - pri["bad"] = f"{fctx.feedstock_name}: Git repository not found.\n" - return False, False - - feedstock_dir = Path(GIT_CLONE_DIR) / (fctx.feedstock_name + "-feedstock") - - backend.clone_fork_and_branch( - upstream_owner=fctx.git_repo_owner, - repo_name=fctx.git_repo_name, - target_dir=feedstock_dir, - new_branch=branch, - base_branch=base_branch, - ) # This is needed because we want to migrate to the new backend step-by-step repo: github3.repos.Repository | None = github3_client().repository( diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 2b4c737a2..5f0a6c11d 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -224,6 +224,36 @@ def test_git_cli_commit(all_: bool, empty: bool, allow_empty: bool): assert "Add Test" in git_log +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_rev_parse_head_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + + run_git_command_mock.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="deadbeef\n" + ) + + head_rev = cli.rev_parse_head(git_dir) + run_git_command_mock.assert_called_once_with( + ["rev-parse", "HEAD"], git_dir, capture_text=True + ) + + assert head_rev == "deadbeef" + + +def test_git_cli_rev_parse_head(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + init_temp_git_repo(dir_path) + cli.commit(dir_path, "Initial commit", allow_empty=True) + head_rev = cli.rev_parse_head(dir_path) + assert len(head_rev) == 40 + assert all(c in "0123456789abcdef" for c in head_rev) + + def test_git_cli_reset_hard_already_reset(): cli = GitCli() with tempfile.TemporaryDirectory() as tmpdir: From 44c7257b3d186f307f52edbfb85cc4933ce9c762 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Fri, 31 May 2024 22:01:58 +0200 Subject: [PATCH 11/38] add diffed_files to git CLI --- conda_forge_tick/git_utils.py | 23 +++++++++++++- tests/test_git_utils.py | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 746aacc62..f2c495582 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -13,7 +13,7 @@ from email import utils from functools import cached_property from pathlib import Path -from typing import Dict, Literal, Optional, Tuple, Union +from typing import Dict, Iterator, Literal, Optional, Tuple, Union import backoff import github @@ -386,6 +386,27 @@ def checkout_new_branch( ["checkout", "--quiet", "-b", branch] + start_point_option, git_dir ) + def diffed_files( + self, git_dir: Path, commit_a: str, commit_b: str = "HEAD" + ) -> Iterator[Path]: + """ + Get the files that are different between two commits. + :param git_dir: The directory of the git repository. This should be the root of the repository. + If it is a subdirectory, only the files in that subdirectory will be returned. + :param commit_a: The first commit. + :param commit_b: The second commit. + :return: An iterator over the files that are different between the two commits. + """ + + # --relative ensures that we do not assemble invalid paths below if git_dir is a subdirectory + ret = self._run_git_command( + ["diff", "--name-only", "--relative", commit_a, commit_b], + git_dir, + capture_text=True, + ) + + return (git_dir / line for line in ret.stdout.splitlines()) + @lock_git_operation() def clone_fork_and_branch( self, diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 5f0a6c11d..96d93dd6f 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -656,6 +656,62 @@ def test_git_cli_checkout_branch_no_track(): ) +def test_git_cli_diffed_files(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + cli.commit(dir_path, "Initial commit", allow_empty=True) + dir_path.joinpath("test.txt").touch() + cli.add(dir_path, dir_path / "test.txt") + cli.commit(dir_path, "Add test.txt") + + diffed_files = list(cli.diffed_files(dir_path, "HEAD~1")) + + assert (dir_path / "test.txt") in diffed_files + assert len(diffed_files) == 1 + + +def test_git_cli_diffed_files_no_diff(): + cli = GitCli() + + with tempfile.TemporaryDirectory() as tmpdir: + dir_path = Path(tmpdir) + + init_temp_git_repo(dir_path) + + cli.commit(dir_path, "Initial commit", allow_empty=True) + + diffed_files = list(cli.diffed_files(dir_path, "HEAD")) + + assert len(diffed_files) == 0 + + +@mock.patch("conda_forge_tick.git_utils.GitCli._run_git_command") +def test_git_cli_diffed_files_mock(run_git_command_mock: MagicMock): + cli = GitCli() + + git_dir = Path("TEST_DIR") + commit = "COMMIT" + + run_git_command_mock.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="test.txt\n" + ) + + diffed_files = list(cli.diffed_files(git_dir, commit)) + + run_git_command_mock.assert_called_once_with( + ["diff", "--name-only", "--relative", commit, "HEAD"], + git_dir, + capture_text=True, + ) + + assert diffed_files == [git_dir / "test.txt"] + + def test_git_cli_clone_fork_and_branch_minimal(): fork_url = "https://github.com/regro-cf-autotick-bot/pytest-feedstock.git" upstream_url = "https://github.com/conda-forge/pytest-feedstock.git" From cf2fbfdbfd023e937c2dd8f02c2c02024722e714 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 11 Jun 2024 12:49:32 +0200 Subject: [PATCH 12/38] feat: GitCLi supports hiding tokens --- conda_forge_tick/git_utils.py | 59 ++++++++---- conda_forge_tick/utils.py | 36 ++++++-- tests/test_git_utils.py | 164 ++++++++++++++++++++++++++++++++-- 3 files changed, 228 insertions(+), 31 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index f2c495582..1fcffefee 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1,10 +1,12 @@ """Utilities for managing github repos""" +import contextlib import copy import enum import logging import math import subprocess +import sys import textwrap import threading import time @@ -43,7 +45,7 @@ PullRequestState, ) from .os_utils import pushd -from .utils import get_bot_run_url, run_command_hiding_token +from .utils import get_bot_run_url, replace_tokens, run_command_hiding_token logger = logging.getLogger(__name__) @@ -158,24 +160,25 @@ class GitCli: If this does impact performance too much, we can consider a per-repository locking strategy. """ - @staticmethod + def __init__(self): + self.__hidden_tokens: list[str] = [] + @lock_git_operation() def _run_git_command( + self, cmd: list[str | Path], working_directory: Path | None = None, check_error: bool = True, - capture_text: bool = False, ) -> subprocess.CompletedProcess: """ Run a git command. stdout is only printed if the command fails. stderr is always printed. If outputs are captured, they are never printed. + stdout is always captured, we capture stderr only if tokens are hidden. :param cmd: The command to run, as a list of strings. :param working_directory: The directory to run the command in. If None, the command will be run in the current working directory. :param check_error: If True, raise a GitCliError if the git command fails. - :param capture_text: If True, capture the output of the git command as text. The output will be in the - returned result object. :return: The result of the git command. :raises GitCliError: If the git command fails and check_error is True. :raises FileNotFoundError: If the working directory does not exist. @@ -184,23 +187,46 @@ def _run_git_command( logger.debug(f"Running git command: {git_command}") - stdout_args = {"stdout": subprocess.PIPE} if not capture_text else {} - capture_args = {"capture_output": True, "text": True} if capture_text else {} + # we only need to capture stderr if we want to hide tokens + stderr_args = {"stderr": subprocess.PIPE} if self.__hidden_tokens else {} try: - return subprocess.run( + p = subprocess.run( git_command, check=check_error, cwd=working_directory, - **stdout_args, - **capture_args, + stdout=subprocess.PIPE, + **stderr_args, + text=True, ) except subprocess.CalledProcessError as e: + e.stdout = replace_tokens(e.stdout, self.__hidden_tokens) + e.stderr = replace_tokens(e.stderr, self.__hidden_tokens) logger.info( - f"Command {git_command} failed. stdout:\n{e.stdout}\nend of stdout" + f"Command '{' '.join(git_command)}' failed.\nstdout:\n{e.stdout}\nend of stdout" ) + if self.__hidden_tokens: + logger.info(f"stderr:\n{e.stderr}\nend of stderr") raise GitCliError(f"Error running git command: {repr(e)}") from e + p.stdout = replace_tokens(p.stdout, self.__hidden_tokens) + p.stderr = replace_tokens(p.stderr, self.__hidden_tokens) + + if self.__hidden_tokens: + # we suppressed stderr, so we need to print it here + print(p.stderr, file=sys.stderr, end="") + + return p + + @contextlib.contextmanager + def hide_token(self, token: str): + """ + Within this context manager, the given token will be hidden in the logs. + """ + self.__hidden_tokens.append(token) + yield + self.__hidden_tokens.pop() + @lock_git_operation() def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): """ @@ -246,7 +272,7 @@ def rev_parse_head(self, git_dir: Path) -> str: :return: The commit hash of HEAD. :raises GitCliError: If the git command fails. """ - ret = self._run_git_command(["rev-parse", "HEAD"], git_dir, capture_text=True) + ret = self._run_git_command(["rev-parse", "HEAD"], git_dir) return ret.stdout.strip() @@ -364,10 +390,7 @@ def checkout_branch( :raises FileNotFoundError: If git_dir does not exist """ track_flag = ["--track"] if track else [] - self._run_git_command( - ["checkout", "--quiet"] + track_flag + [branch], - git_dir, - ) + self._run_git_command(["checkout", "--quiet"] + track_flag + [branch], git_dir) @lock_git_operation() def checkout_new_branch( @@ -400,9 +423,7 @@ def diffed_files( # --relative ensures that we do not assemble invalid paths below if git_dir is a subdirectory ret = self._run_git_command( - ["diff", "--name-only", "--relative", commit_a, commit_b], - git_dir, - capture_text=True, + ["diff", "--name-only", "--relative", commit_a, commit_b], git_dir ) return (git_dir / line for line in ret.stdout.splitlines()) diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index c911a0693..f7ad3aecb 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -14,7 +14,7 @@ import typing import warnings from collections import defaultdict -from typing import Any, Callable, Dict, Iterable, Optional, Set, Tuple, cast +from typing import Any, Callable, Dict, Iterable, Optional, Set, Tuple, cast, overload import jinja2 import jinja2.sandbox @@ -1004,12 +1004,38 @@ def change_log_level(logger, new_level): logger.setLevel(saved_logger_level) +@overload +def replace_tokens(s: str, tokens: Iterable[str]) -> str: ... + + +@overload +def replace_tokens(s: None, tokens: Iterable[str]) -> None: ... + + +def replace_tokens(s: str | None, tokens: Iterable[str]) -> str | None: + """ + Replace tokens in a string with asterisks of the same length. + + None values are passed through. + + :param s: The string to replace tokens in. + :param tokens: The tokens to replace. + + :return: The string with the tokens replaced. + """ + if not s: + return s + for token in tokens: + s = s.replace(token, "*" * len(token)) + return s + + def print_subprocess_output_strip_token( - completed_process: subprocess.CompletedProcess, token: str + completed_process: subprocess.CompletedProcess, *tokens: str ) -> None: """ Use this function to print the outputs (stdout and stderr) of a subprocess.CompletedProcess object - that may contain sensitive information. The token will be replaced with a string + that may contain sensitive information. The token or tokens will be replaced with a string of asterisks of the same length. This function assumes that you have called subprocess.run() with the arguments text=True, stdout=subprocess.PIPE, @@ -1021,7 +1047,7 @@ def print_subprocess_output_strip_token( :param completed_process: The subprocess.CompletedProcess object to print the outputs of. You have probably obtained this object by calling subprocess.run(). - :param token: The token to replace with asterisks. + :param tokens: The token or tokens to replace with asterisks. :raises ValueError: If the completed_process object does not contain str in stdout or stderr. """ @@ -1037,7 +1063,7 @@ def print_subprocess_output_strip_token( "text=True." ) - captured = captured.replace(token, "*" * len(token)) + replace_tokens(captured, tokens) print(captured, file=out_dev, end="") out_dev.flush() diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 96d93dd6f..626c96e9a 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -67,6 +67,160 @@ def test_git_cli_run_git_command_error(subprocess_run_mock: MagicMock): cli._run_git_command(["GIT_COMMAND"], working_directory) +@pytest.mark.parametrize("token_hidden", [True, False]) +@pytest.mark.parametrize("check_error", [True, False]) +@mock.patch("subprocess.run") +def test_git_cli_run_git_command_mock( + subprocess_run_mock: MagicMock, check_error: bool, token_hidden: bool +): + """ + This test checks if all parameters are passed correctly to the subprocess.run function. + """ + cli = GitCli() + + working_directory = Path("TEST_DIR") + + def run_command(): + cli._run_git_command( + ["COMMAND", "ARG1", "ARG2"], working_directory, check_error + ) + + if token_hidden: + with cli.hide_token("TOKEN"): + run_command() + else: + run_command() + + stderr_args = {"stderr": subprocess.PIPE} if token_hidden else {} + + subprocess_run_mock.assert_called_once_with( + ["git", "COMMAND", "ARG1", "ARG2"], + check=check_error, + cwd=working_directory, + stdout=subprocess.PIPE, + **stderr_args, + text=True, + ) + + +@pytest.mark.parametrize("token_hidden", [True, False]) +@pytest.mark.parametrize("check_error", [True, False]) +def test_git_cli_run_git_command_stdout_captured( + capfd, check_error: bool, token_hidden: bool +): + """ + Verify that the stdout of the git command is captured and not printed to the console. + """ + cli = GitCli() + + def run_command() -> subprocess.CompletedProcess: + return cli._run_git_command(["version"], check_error=check_error) + + if token_hidden: + with cli.hide_token("TOKEN"): + p = run_command() + else: + p = run_command() + + captured = capfd.readouterr() + + assert captured.out == "" + assert p.stdout.startswith("git version") + + +def test_git_cli_run_git_command_stderr_not_captured(capfd): + """ + Verify that the stderr of the git command is not captured if no token is hidden. + """ + cli = GitCli() + + p = cli._run_git_command(["non-existing-command"], check_error=False) + + captured = capfd.readouterr() + + assert captured.out == "" + assert "not a git command" in captured.err + assert p.stderr is None + + +def test_git_cli_hide_token_stdout_no_error(capfd): + cli = GitCli() + + with cli.hide_token("git"): + p = cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert "git" not in captured.out + assert "git" not in captured.err + assert "git" not in p.stdout + assert "git" not in p.stderr + + assert p.stdout.count("***") > 5 + + +def test_git_cli_hide_token_stderr_no_check_error(capfd): + cli = GitCli() + + with cli.hide_token("command"): + p = cli._run_git_command(["non-existing-command"], check_error=False) + + captured = capfd.readouterr() + + assert "command" not in captured.out + assert "command" not in captured.err + assert "command" not in p.stdout + assert "command" not in p.stderr + + assert p.stderr.count("*******") >= 2 + assert captured.err.count("*******") >= 2 + + +def test_git_cli_hide_token_run_git_command_check_error(capfd, caplog): + cli = GitCli() + + caplog.set_level(logging.INFO) + + with cli.hide_token("command"): + with pytest.raises(GitCliError): + cli._run_git_command(["non-existing-command"]) + + print(caplog.text) + assert "Command 'git non-existing-command' failed." in caplog.text + assert ( + caplog.text.count("command") == 1 + ) # only the command itself is printed directly by us + + assert "'non-existing-*******' is not a git *******" in caplog.text + + +def test_hide_token_multiple(capfd, caplog): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + with cli.hide_token("clone"): + with cli.hide_token("commit"): + p = cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert "clone" not in captured.out + assert "clone" not in captured.err + assert "clone" not in p.stdout + assert "clone" not in p.stderr + + assert "commit" not in captured.out + assert "commit" not in captured.err + assert "commit" not in p.stdout + assert "commit" not in p.stderr + + assert "clone" not in caplog.text + assert "commit" not in caplog.text + + assert p.stdout.count("*****") >= 2 + + def test_git_cli_outside_repo(): with tempfile.TemporaryDirectory() as tmpdir: dir_path = Path(tmpdir) @@ -157,9 +311,7 @@ def test_git_cli_add_success(n_paths: int, all_: bool): cli = GitCli() cli.add(git_dir, *pathspec, all_=all_) - tracked_files = cli._run_git_command( - ["ls-files", "-s"], git_dir, capture_text=True - ).stdout + tracked_files = cli._run_git_command(["ls-files", "-s"], git_dir).stdout for path in pathspec: assert path.name in tracked_files @@ -219,7 +371,7 @@ def test_git_cli_commit(all_: bool, empty: bool, allow_empty: bool): cli.commit(git_dir, "Add Test", all_, allow_empty) - git_log = cli._run_git_command(["log"], git_dir, capture_text=True).stdout + git_log = cli._run_git_command(["log"], git_dir).stdout assert "Add Test" in git_log @@ -474,9 +626,7 @@ def test_git_cli_push_to_url_local_repository(): local_repo = dir_path / "local_repo" local_repo.mkdir() - cli._run_git_command( - ["clone", source_repo.resolve(), local_repo], - ) + cli._run_git_command(["clone", source_repo.resolve(), local_repo]) # remove all references to the original repo cli._run_git_command( From 5af5db307ee576d4777b141e4354d319804a288b Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 11 Jun 2024 14:43:18 +0200 Subject: [PATCH 13/38] add more tests for token hiding --- tests/test_git_utils.py | 78 +++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 626c96e9a..e767e537b 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -159,6 +159,44 @@ def test_git_cli_hide_token_stdout_no_error(capfd): assert p.stdout.count("***") > 5 +def test_git_cli_hide_token_stdout_error_check_error(caplog, capfd): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + with cli.hide_token("all"): + with pytest.raises(GitCliError): + # git help --a prints to stdout (!) and then exits with an error + cli._run_git_command(["help", "--a"]) + + captured = capfd.readouterr() + + assert "all" not in captured.out + assert "all" not in captured.err + assert "all" not in caplog.text + + assert "***" in caplog.text + + +def test_git_cli_hide_token_stdout_error_no_check_error(caplog, capfd): + cli = GitCli() + + caplog.set_level(logging.DEBUG) + + with cli.hide_token("all"): + p = cli._run_git_command(["help", "--a"], check_error=False) + + captured = capfd.readouterr() + + assert "all" not in captured.out + assert "all" not in captured.err + assert "all" not in p.stdout + assert "all" not in p.stderr + assert "all" not in caplog.text + + assert "***" in p.stdout + + def test_git_cli_hide_token_stderr_no_check_error(capfd): cli = GitCli() @@ -194,31 +232,43 @@ def test_git_cli_hide_token_run_git_command_check_error(capfd, caplog): assert "'non-existing-*******' is not a git *******" in caplog.text -def test_hide_token_multiple(capfd, caplog): +def test_git_cli_hide_token_multiple(capfd, caplog): cli = GitCli() caplog.set_level(logging.DEBUG) with cli.hide_token("clone"): with cli.hide_token("commit"): - p = cli._run_git_command(["help"]) + p1 = cli._run_git_command(["help"]) - captured = capfd.readouterr() + captured = capfd.readouterr() + + assert "clone" not in captured.out + assert "clone" not in captured.err + assert "clone" not in p1.stdout + assert "clone" not in p1.stderr + + assert "commit" not in captured.out + assert "commit" not in captured.err + assert "commit" not in p1.stdout + assert "commit" not in p1.stderr + + assert "clone" not in caplog.text + assert "commit" not in caplog.text + + assert p1.stdout.count("*****") >= 2 - assert "clone" not in captured.out - assert "clone" not in captured.err - assert "clone" not in p.stdout - assert "clone" not in p.stderr + # we left the context manager, so "commit" should be visible again, but "clone" should still be hidden + p2 = cli._run_git_command(["help"]) - assert "commit" not in captured.out - assert "commit" not in captured.err - assert "commit" not in p.stdout - assert "commit" not in p.stderr + captured = capfd.readouterr() - assert "clone" not in caplog.text - assert "commit" not in caplog.text + assert "clone" not in captured.out + assert "clone" not in captured.err + assert "clone" not in p2.stdout + assert "clone" not in p2.stderr - assert p.stdout.count("*****") >= 2 + assert "commit" in p2.stdout def test_git_cli_outside_repo(): From bf61bebc7a8f44308f948dcd04226ad26d277ce4 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 13 Jun 2024 20:58:45 +0200 Subject: [PATCH 14/38] add feedstock-related data to FeedstockContext --- conda_forge_tick/contexts.py | 42 +++++++++++++++++++++++++++ conda_forge_tick/migration_runner.py | 1 - conda_forge_tick/migrators/version.py | 2 +- tests/test_migrators.py | 1 - 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index 451d0cd98..ac6313c21 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -1,15 +1,20 @@ import os import typing from dataclasses import dataclass +from pathlib import Path from networkx import DiGraph from conda_forge_tick.lazy_json_backends import load +from conda_forge_tick.utils import get_keys_default if typing.TYPE_CHECKING: from conda_forge_tick.migrators_types import AttrsTypedDict +GIT_CLONE_DIR = Path("feedstocks").resolve() + + if os.path.exists("all_feedstocks.json"): with open("all_feedstocks.json") as f: DEFAULT_BRANCHES = load(f).get("default_branches", {}) @@ -58,3 +63,40 @@ def git_href(self) -> str: A link to the feedstocks GitHub repository. """ return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}" + + @property + def local_clone_dir(self) -> Path: + """ + The local path to the feedstock repository. + """ + return GIT_CLONE_DIR / self.git_repo_name + + @property + def automerge(self) -> bool | str: + """ + Get the automerge setting of the feedstock. + + Note: A better solution to implement this is to use the NodeAttributes Pydantic + model for the attrs field. This will be done in the future. + """ + return get_keys_default( + self.attrs, + ["conda-forge.yml", "bot", "automerge"], + {}, + False, + ) + + @property + def check_solvable(self) -> bool: + """ + Get the check_solvable setting of the feedstock. + + Note: A better solution to implement this is to use the NodeAttributes Pydantic + model for the attrs field. This will be done in the future. + """ + return get_keys_default( + self.attrs, + ["conda-forge.yml", "bot", "check_solvable"], + {}, + False, + ) diff --git a/conda_forge_tick/migration_runner.py b/conda_forge_tick/migration_runner.py index d7cea3aa5..c76aad201 100644 --- a/conda_forge_tick/migration_runner.py +++ b/conda_forge_tick/migration_runner.py @@ -226,7 +226,6 @@ def run_migration_local( attrs=node_attrs, ) feedstock_ctx.default_branch = default_branch - feedstock_ctx.feedstock_dir = feedstock_dir recipe_dir = os.path.join(feedstock_dir, "recipe") data = { diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index ecccc77d0..c9d1d6b41 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -340,7 +340,7 @@ def _hint_and_maybe_update_deps(self, feedstock_ctx): try: _, hint = get_dep_updates_and_hints( update_deps, - os.path.join(feedstock_ctx.feedstock_dir, "recipe"), + str(feedstock_ctx.local_clone_dir / "recipe"), feedstock_ctx.attrs, self.python_nodes, "new_version", diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 26e77196b..0db01132c 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -523,7 +523,6 @@ def run_test_migration( feedstock_name=name, attrs=pmy, ) - fctx.feedstock_dir = os.path.dirname(tmpdir) m.effective_graph.add_node(name) m.effective_graph.nodes[name]["payload"] = MockLazyJson({}) m.pr_body(fctx) From 71a2a953af100605ec54394e820337a5ec7250b3 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 13 Jun 2024 21:39:47 +0200 Subject: [PATCH 15/38] use get_bot_token instead of sensitive_env --- conda_forge_tick/git_utils.py | 56 ++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 1fcffefee..e43bc9fa7 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -86,13 +86,17 @@ } +def get_bot_token(): + with sensitive_env() as env: + return env["BOT_TOKEN"] + + def github3_client() -> github3.GitHub: """ This will be removed in the future, use the GitHubBackend class instead. """ if not hasattr(GITHUB3_CLIENT, "client"): - with sensitive_env() as env: - GITHUB3_CLIENT.client = github3.login(token=env["BOT_TOKEN"]) + GITHUB3_CLIENT.client = github3.login(token=get_bot_token()) return GITHUB3_CLIENT.client @@ -101,11 +105,10 @@ def github_client() -> github.Github: This will be removed in the future, use the GitHubBackend class instead. """ if not hasattr(GITHUB_CLIENT, "client"): - with sensitive_env() as env: - GITHUB_CLIENT.client = github.Github( - auth=github.Auth.Token(env["BOT_TOKEN"]), - per_page=100, - ) + GITHUB_CLIENT.client = github.Github( + auth=github.Auth.Token(get_bot_token()), + per_page=100, + ) return GITHUB_CLIENT.client @@ -997,8 +1000,7 @@ def github_backend() -> GitHubBackend: """ This helper method will be removed in the future, use the GitHubBackend class directly. """ - with sensitive_env() as env: - return GitHubBackend.from_token(env["BOT_TOKEN"]) + return GitHubBackend.from_token(get_bot_token()) def is_github_api_limit_reached() -> bool: @@ -1058,17 +1060,18 @@ def delete_branch(pr_json: LazyJson, dry_run: bool = False) -> None: gh = github3_client() deploy_repo = gh.me().login + "/" + name - with sensitive_env() as env: - run_command_hiding_token( - [ - "git", - "push", - f"https://{env['BOT_TOKEN']}@github.com/{deploy_repo}.git", - "--delete", - ref, - ], - token=env["BOT_TOKEN"], - ) + token = get_bot_token() + + run_command_hiding_token( + [ + "git", + "push", + f"https://{token}@github.com/{deploy_repo}.git", + "--delete", + ref, + ], + token=token, + ) # Replace ref so we know not to try again pr_json["head"]["ref"] = "this_is_not_a_branch" @@ -1135,11 +1138,10 @@ def lazy_update_pr_json( pr_json : dict-like A dict-like object with the current PR information. """ - with sensitive_env() as env: - hdrs = { - "Authorization": f"token {env['BOT_TOKEN']}", - "Accept": "application/vnd.github.v3+json", - } + hdrs = { + "Authorization": f"token {get_bot_token()}", + "Accept": "application/vnd.github.v3+json", + } if not force and "ETag" in pr_json: hdrs["If-None-Match"] = pr_json["ETag"] @@ -1306,9 +1308,9 @@ def push_repo( The dict representing the PR, can be used with `from_json` to create a PR instance. """ - with sensitive_env() as env, pushd(feedstock_dir): + with pushd(feedstock_dir): # Copyright (c) 2016 Aaron Meurer, Gil Forsyth - token = env["BOT_TOKEN"] + token = get_bot_token() gh_username = github3_client().me().login head = gh_username + ":" + branch From 56f275925bc8ed22f776a439354a597118a9359e Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 13 Jun 2024 21:43:49 +0200 Subject: [PATCH 16/38] token hiding: no context manager, automatically if token is known --- conda_forge_tick/git_utils.py | 27 ++++++--- tests/test_git_utils.py | 111 +++++++++++++++++----------------- 2 files changed, 74 insertions(+), 64 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index e43bc9fa7..98fa8b7aa 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1,6 +1,5 @@ """Utilities for managing github repos""" -import contextlib import copy import enum import logging @@ -221,14 +220,13 @@ def _run_git_command( return p - @contextlib.contextmanager - def hide_token(self, token: str): + def add_hidden_token(self, token: str) -> None: """ - Within this context manager, the given token will be hidden in the logs. + Permanently hide a token in the logs. + + :param token: The token to hide. """ self.__hidden_tokens.append(token) - yield - self.__hidden_tokens.pop() @lock_git_operation() def add(self, git_dir: Path, *pathspec: Path, all_: bool = False): @@ -727,14 +725,26 @@ class GitHubBackend(GitPlatformBackend): The number of items to fetch per page from the GitHub API. """ - def __init__(self, github3_client: github3.GitHub, pygithub_client: github.Github): + def __init__( + self, + github3_client: github3.GitHub, + pygithub_client: github.Github, + token_to_hide: str | None = None, + ): """ Create a new GitHubBackend. Note: Because we need additional response headers, we wrap the github3 session of the github3 client with our own session wrapper and replace the github3 client's session with it. + + :param github3_client: The github3 client to use for interacting with the GitHub API. + :param pygithub_client: The PyGithub client to use for interacting with the GitHub API. + :param token_to_hide: A token to hide in the CLI output. If None, no tokens are hidden. """ - super().__init__(GitCli()) + cli = GitCli() + if token_to_hide: + cli.add_hidden_token(token_to_hide) + super().__init__(cli) self.github3_client = github3_client self._github3_session = _Github3SessionWrapper(self.github3_client.session) self.github3_client.session = self._github3_session @@ -746,6 +756,7 @@ def from_token(cls, token: str): return cls( github3.login(token=token), github.Github(auth=github.Auth.Token(token), per_page=cls._GITHUB_PER_PAGE), + token_to_hide=token, ) def does_repository_exist(self, owner: str, repo_name: str) -> bool: diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index e767e537b..910289ebd 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -80,16 +80,10 @@ def test_git_cli_run_git_command_mock( working_directory = Path("TEST_DIR") - def run_command(): - cli._run_git_command( - ["COMMAND", "ARG1", "ARG2"], working_directory, check_error - ) - if token_hidden: - with cli.hide_token("TOKEN"): - run_command() - else: - run_command() + cli.add_hidden_token("TOKEN") + + cli._run_git_command(["COMMAND", "ARG1", "ARG2"], working_directory, check_error) stderr_args = {"stderr": subprocess.PIPE} if token_hidden else {} @@ -113,14 +107,9 @@ def test_git_cli_run_git_command_stdout_captured( """ cli = GitCli() - def run_command() -> subprocess.CompletedProcess: - return cli._run_git_command(["version"], check_error=check_error) - if token_hidden: - with cli.hide_token("TOKEN"): - p = run_command() - else: - p = run_command() + cli.add_hidden_token("TOKEN") + p = cli._run_git_command(["version"], check_error=check_error) captured = capfd.readouterr() @@ -146,8 +135,8 @@ def test_git_cli_run_git_command_stderr_not_captured(capfd): def test_git_cli_hide_token_stdout_no_error(capfd): cli = GitCli() - with cli.hide_token("git"): - p = cli._run_git_command(["help"]) + cli.add_hidden_token("git") + p = cli._run_git_command(["help"]) captured = capfd.readouterr() @@ -164,10 +153,10 @@ def test_git_cli_hide_token_stdout_error_check_error(caplog, capfd): caplog.set_level(logging.DEBUG) - with cli.hide_token("all"): - with pytest.raises(GitCliError): - # git help --a prints to stdout (!) and then exits with an error - cli._run_git_command(["help", "--a"]) + cli.add_hidden_token("all") + with pytest.raises(GitCliError): + # git help --a prints to stdout (!) and then exits with an error + cli._run_git_command(["help", "--a"]) captured = capfd.readouterr() @@ -183,8 +172,8 @@ def test_git_cli_hide_token_stdout_error_no_check_error(caplog, capfd): caplog.set_level(logging.DEBUG) - with cli.hide_token("all"): - p = cli._run_git_command(["help", "--a"], check_error=False) + cli.add_hidden_token("all") + p = cli._run_git_command(["help", "--a"], check_error=False) captured = capfd.readouterr() @@ -200,8 +189,8 @@ def test_git_cli_hide_token_stdout_error_no_check_error(caplog, capfd): def test_git_cli_hide_token_stderr_no_check_error(capfd): cli = GitCli() - with cli.hide_token("command"): - p = cli._run_git_command(["non-existing-command"], check_error=False) + cli.add_hidden_token("command") + p = cli._run_git_command(["non-existing-command"], check_error=False) captured = capfd.readouterr() @@ -219,9 +208,9 @@ def test_git_cli_hide_token_run_git_command_check_error(capfd, caplog): caplog.set_level(logging.INFO) - with cli.hide_token("command"): - with pytest.raises(GitCliError): - cli._run_git_command(["non-existing-command"]) + cli.add_hidden_token("command") + with pytest.raises(GitCliError): + cli._run_git_command(["non-existing-command"]) print(caplog.text) assert "Command 'git non-existing-command' failed." in caplog.text @@ -237,38 +226,26 @@ def test_git_cli_hide_token_multiple(capfd, caplog): caplog.set_level(logging.DEBUG) - with cli.hide_token("clone"): - with cli.hide_token("commit"): - p1 = cli._run_git_command(["help"]) - - captured = capfd.readouterr() + cli.add_hidden_token("clone") + cli.add_hidden_token("commit") + p1 = cli._run_git_command(["help"]) - assert "clone" not in captured.out - assert "clone" not in captured.err - assert "clone" not in p1.stdout - assert "clone" not in p1.stderr - - assert "commit" not in captured.out - assert "commit" not in captured.err - assert "commit" not in p1.stdout - assert "commit" not in p1.stderr - - assert "clone" not in caplog.text - assert "commit" not in caplog.text - - assert p1.stdout.count("*****") >= 2 + captured = capfd.readouterr() - # we left the context manager, so "commit" should be visible again, but "clone" should still be hidden - p2 = cli._run_git_command(["help"]) + assert "clone" not in captured.out + assert "clone" not in captured.err + assert "clone" not in p1.stdout + assert "clone" not in p1.stderr - captured = capfd.readouterr() + assert "commit" not in captured.out + assert "commit" not in captured.err + assert "commit" not in p1.stdout + assert "commit" not in p1.stderr - assert "clone" not in captured.out - assert "clone" not in captured.err - assert "clone" not in p2.stdout - assert "clone" not in p2.stderr + assert "clone" not in caplog.text + assert "commit" not in caplog.text - assert "commit" in p2.stdout + assert p1.stdout.count("*****") >= 2 def test_git_cli_outside_repo(): @@ -1085,6 +1062,28 @@ def test_github_backend_from_token(): # we cannot verify the pygithub token trivially +@pytest.mark.parametrize("from_token", [True, False]) +def test_github_backend_token_to_hide(caplog, capfd, from_token: bool): + caplog.set_level(logging.DEBUG) + token = "commit" + + if from_token: + backend = GitHubBackend.from_token(token) + else: + backend = GitHubBackend(MagicMock(), MagicMock(), token) + + # the token should be hidden by default, without any context manager + p = backend.cli._run_git_command(["help"]) + + captured = capfd.readouterr() + + assert token not in caplog.text + assert token not in captured.out + assert token not in captured.err + assert token not in p.stdout + assert token not in p.stderr + + @pytest.mark.parametrize("does_exist", [True, False]) def test_github_backend_does_repository_exist(does_exist: bool): github3_client = MagicMock() From bf164910190127fc56fd36c7e616e61c88f2ee39 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 16:13:43 +0200 Subject: [PATCH 17/38] hide tokens in Git Backends automatically, push to repository --- conda_forge_tick/git_utils.py | 47 +++++++++++++++++++--- tests/test_git_utils.py | 74 +++++++++++++++++++++++++++-------- 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 98fa8b7aa..86644e7ad 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -548,21 +548,38 @@ def get_remote_url( owner: str, repo_name: str, connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, + token: str | None = None, ) -> str: """ Get the URL of a remote repository. :param owner: The owner of the repository. :param repo_name: The name of the repository. :param connection_mode: The connection mode to use. + :param token: A token to use for authentication. If falsy, no token is used. Use get_authenticated_remote_url + instead if you want to use the token of the current user. :raises ValueError: If the connection mode is not supported. """ # Currently we don't need any abstraction for other platforms than GitHub, so we don't build such abstractions. match connection_mode: case GitConnectionMode.HTTPS: - return f"https://github.com/{owner}/{repo_name}.git" + return f"https://{f'{token}@' if token else ''}github.com/{owner}/{repo_name}.git" case _: raise ValueError(f"Unsupported connection mode: {connection_mode}") + @abstractmethod + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + """ + Push changes to a repository. + :param owner: The owner of the repository. + :param repo_name: The name of the repository. + :param git_dir: The directory of the git repository. + :param branch: The branch to push to. + :raises GitPlatformError: If the push fails. + """ + pass + @abstractmethod def fork(self, owner: str, repo_name: str): """ @@ -729,7 +746,7 @@ def __init__( self, github3_client: github3.GitHub, pygithub_client: github.Github, - token_to_hide: str | None = None, + token: str, ): """ Create a new GitHubBackend. @@ -739,12 +756,14 @@ def __init__( :param github3_client: The github3 client to use for interacting with the GitHub API. :param pygithub_client: The PyGithub client to use for interacting with the GitHub API. - :param token_to_hide: A token to hide in the CLI output. If None, no tokens are hidden. + :param token: The token that will be hidden in CLI outputs and used for writing to git repositories. Note that + you need to authenticate github3 and PyGithub yourself. Use the `from_token` class method to create an instance + that has all necessary clients set up. """ cli = GitCli() - if token_to_hide: - cli.add_hidden_token(token_to_hide) + cli.add_hidden_token(token) super().__init__(cli) + self.__token = token self.github3_client = github3_client self._github3_session = _Github3SessionWrapper(self.github3_client.session) self.github3_client.session = self._github3_session @@ -756,13 +775,22 @@ def from_token(cls, token: str): return cls( github3.login(token=token), github.Github(auth=github.Auth.Token(token), per_page=cls._GITHUB_PER_PAGE), - token_to_hide=token, + token=token, ) def does_repository_exist(self, owner: str, repo_name: str) -> bool: repo = self.github3_client.repository(owner, repo_name) return repo is not None + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + # we need an authenticated URL with write access + remote_url = self.get_remote_url( + owner, repo_name, GitConnectionMode.HTTPS, self.__token + ) + self.cli.push_to_url(git_dir, remote_url, branch) + @lock_git_operation() def fork(self, owner: str, repo_name: str): if self.does_repository_exist(self.user, repo_name): @@ -923,6 +951,13 @@ def does_repository_exist(self, owner: str, repo_name: str) -> bool: self.get_remote_url(owner, repo_name, GitConnectionMode.HTTPS) ) + def push_to_repository( + self, owner: str, repo_name: str, git_dir: Path, branch: str + ): + logger.debug( + f"Dry Run: Pushing changes from {git_dir} to {owner}/{repo_name} on branch {branch}." + ) + @lock_git_operation() def fork(self, owner: str, repo_name: str): if repo_name in self._repos: diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 910289ebd..d7f84eee1 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1053,6 +1053,16 @@ def test_git_platform_backend_get_remote_url_https(): assert url == f"https://github.com/{owner}/{repo}.git" +def test_git_platform_backend_get_remote_url_token(): + owner = "OWNER" + repo = "REPO" + token = "TOKEN" + + url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS, token) + + assert url == f"https://{token}@github.com/{owner}/{repo}.git" + + def test_github_backend_from_token(): token = "TOKEN" @@ -1088,7 +1098,7 @@ def test_github_backend_token_to_hide(caplog, capfd, from_token: bool): def test_github_backend_does_repository_exist(does_exist: bool): github3_client = MagicMock() - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") github3_client.repository.return_value = MagicMock() if does_exist else None @@ -1110,7 +1120,7 @@ def test_github_backend_fork_not_exists_repo_found( repository = MagicMock() github3_client.repository.return_value = repository - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") user_mock.return_value = "USER" backend.fork("UPSTREAM-OWNER", "REPO") @@ -1120,6 +1130,21 @@ def test_github_backend_fork_not_exists_repo_found( sleep_mock.assert_called_once_with(5) +@mock.patch("conda_forge_tick.git_utils.GitCli.push_to_url") +def test_github_backend_push_to_repository(push_to_url_mock: MagicMock): + backend = GitHubBackend.from_token("THIS_IS_THE_TOKEN") + + git_dir = Path("GIT_DIR") + + backend.push_to_repository("OWNER", "REPO", git_dir, "BRANCH_NAME") + + push_to_url_mock.assert_called_once_with( + git_dir, + "https://THIS_IS_THE_TOKEN@github.com/OWNER/REPO.git", + "BRANCH_NAME", + ) + + @pytest.mark.parametrize("branch_already_synced", [True, False]) @mock.patch("time.sleep", return_value=None) @mock.patch( @@ -1158,7 +1183,7 @@ def get_repo(full_name: str): upstream_repo.default_branch = "UPSTREAM_BRANCH_NAME" fork_repo.default_branch = "FORK_BRANCH_NAME" - backend = GitHubBackend(MagicMock(), pygithub_client) + backend = GitHubBackend(MagicMock(), pygithub_client, "") backend.fork("UPSTREAM-OWNER", "REPO") if not branch_already_synced: @@ -1181,7 +1206,7 @@ def test_github_backend_remote_does_not_exist( github3_client = MagicMock() github3_client.repository.return_value = None - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") user_mock.return_value = "USER" @@ -1198,7 +1223,7 @@ def test_github_backend_user(): user.login = "USER" pygithub_client.get_user.return_value = user - backend = GitHubBackend(MagicMock(), pygithub_client) + backend = GitHubBackend(MagicMock(), pygithub_client, "") for _ in range(4): # cached property @@ -1213,7 +1238,7 @@ def test_github_backend_get_api_requests_left_github_exception(caplog): "API Error" ) - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() is None assert "API error while fetching" in caplog.text @@ -1225,7 +1250,7 @@ def test_github_backend_get_api_requests_left_unexpected_response_schema(caplog) github3_client = MagicMock() github3_client.rate_limit.return_value = {"some": "gibberish data"} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() is None assert "API Error while parsing" @@ -1237,7 +1262,7 @@ def test_github_backend_get_api_requests_left_nonzero(): github3_client = MagicMock() github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 5}}} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 5 @@ -1249,7 +1274,7 @@ def test_github_backend_get_api_requests_left_zero_invalid_reset_time(caplog): github3_client.rate_limit.return_value = {"resources": {"core": {"remaining": 0}}} - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 0 @@ -1269,7 +1294,7 @@ def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): "resources": {"core": {"remaining": 0, "reset": reset_timestamp}} } - backend = GitHubBackend(github3_client, MagicMock()) + backend = GitHubBackend(github3_client, MagicMock(), "") assert backend.get_api_requests_left() == 0 @@ -1308,7 +1333,7 @@ def request_side_effect(method, _url, **_kwargs): pygithub_mock = MagicMock() pygithub_mock.get_user.return_value.login = "CURRENT_USER" - backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock) + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") pr_data = backend.create_pull_request( "conda-forge", @@ -1394,7 +1419,7 @@ def request_side_effect(method, url, **_kwargs): request_mock.side_effect = request_side_effect - backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") backend.comment_on_pull_request( "conda-forge", @@ -1426,7 +1451,7 @@ def request_side_effect(method, url, **_kwargs): request_mock.side_effect = request_side_effect - backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") with pytest.raises(RepositoryNotFoundError): backend.comment_on_pull_request( @@ -1463,7 +1488,7 @@ def request_side_effect(method, url, **_kwargs): assert False, f"Unexpected endpoint: {method} {url}" request_mock.side_effect = request_side_effect - backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") with pytest.raises( GitPlatformError, @@ -1516,7 +1541,7 @@ def request_side_effect(method, url, **_kwargs): request_mock.side_effect = request_side_effect - backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock()) + backend = GitHubBackend(github3.login(token="TOKEN"), MagicMock(), "") with pytest.raises(GitPlatformError, match="Could not comment on pull request"): backend.comment_on_pull_request( @@ -1528,7 +1553,7 @@ def request_side_effect(method, url, **_kwargs): @pytest.mark.parametrize( - "backend", [GitHubBackend(MagicMock(), MagicMock()), DryRunBackend()] + "backend", [GitHubBackend(MagicMock(), MagicMock(), ""), DryRunBackend()] ) @mock.patch( "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock @@ -1547,7 +1572,7 @@ def test_git_platform_backend_clone_fork_and_branch( user_mock.return_value = "USER" - backend = GitHubBackend(MagicMock(), MagicMock()) + backend = GitHubBackend(MagicMock(), MagicMock(), "") backend.clone_fork_and_branch( upstream_owner, repo_name, target_dir, new_branch, base_branch ) @@ -1584,6 +1609,21 @@ def test_dry_run_backend_does_repository_exist_other_repo(): ) +def test_dry_run_backend_push_to_repository(caplog): + caplog.set_level(logging.DEBUG) + + backend = DryRunBackend() + + git_dir = Path("GIT_DIR") + + backend.push_to_repository("OWNER", "REPO", git_dir, "BRANCH_NAME") + + assert ( + "Dry Run: Pushing changes from GIT_DIR to OWNER/REPO on branch BRANCH_NAME" + in caplog.text + ) + + def test_dry_run_backend_fork(caplog): caplog.set_level(logging.DEBUG) From a0d7070180b02075eb446a7d94d47ab532baaf16 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 16:20:22 +0200 Subject: [PATCH 18/38] FIX: forking should do nothing if already exists --- conda_forge_tick/git_utils.py | 3 ++- tests/test_git_utils.py | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 86644e7ad..54af823aa 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -961,7 +961,8 @@ def push_to_repository( @lock_git_operation() def fork(self, owner: str, repo_name: str): if repo_name in self._repos: - raise ValueError(f"Fork of {repo_name} already exists.") + logger.debug(f"Fork of {repo_name} already exists. Doing nothing.") + return logger.debug( f"Dry Run: Creating fork of {owner}/{repo_name} for user {self._USER}." diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index d7f84eee1..2fd371979 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1635,11 +1635,8 @@ def test_dry_run_backend_fork(caplog): in caplog.text ) - with pytest.raises(ValueError, match="Fork of REPO already exists"): - backend.fork("UPSTREAM_OWNER", "REPO") - - with pytest.raises(ValueError, match="Fork of REPO already exists"): - backend.fork("OTHER_OWNER", "REPO") + # this should not raise an error + backend.fork("UPSTREAM_OWNER", "REPO") def test_dry_run_backend_sync_default_branch(caplog): From f481f435894fdee893ba96d6cab2541a68404de1 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 16:56:59 +0200 Subject: [PATCH 19/38] get_remote_url now redirects forks to upstream (DryRunBackend) --- conda_forge_tick/git_utils.py | 33 ++++++++++++-- tests/test_git_utils.py | 82 +++++++++++++++++++++++++++-------- 2 files changed, 93 insertions(+), 22 deletions(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 54af823aa..dd2715e2b 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -543,8 +543,8 @@ def does_repository_exist(self, owner: str, repo_name: str) -> bool: """ pass - @staticmethod def get_remote_url( + self, owner: str, repo_name: str, connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, @@ -558,6 +558,8 @@ def get_remote_url( :param token: A token to use for authentication. If falsy, no token is used. Use get_authenticated_remote_url instead if you want to use the token of the current user. :raises ValueError: If the connection mode is not supported. + :raises RepositoryNotFoundError: If the repository does not exist. This is only raised if the backend relies on + the repository existing to generate the URL. """ # Currently we don't need any abstraction for other platforms than GitHub, so we don't build such abstractions. match connection_mode: @@ -937,7 +939,12 @@ class DryRunBackend(GitPlatformBackend): def __init__(self): super().__init__(GitCli()) - self._repos: set[str] = set() + self._repos: dict[str, str] = {} + """ + _repos maps from repository name to the owner of the upstream repository. + If a remote URL of a fork is requested with get_remote_url, _USER (the virtual current user) is + replaced by the owner of the upstream repository. This allows cloning the forked repository. + """ def get_api_requests_left(self) -> Bound: return Bound.INFINITY @@ -951,6 +958,26 @@ def does_repository_exist(self, owner: str, repo_name: str) -> bool: self.get_remote_url(owner, repo_name, GitConnectionMode.HTTPS) ) + def get_remote_url( + self, + owner: str, + repo_name: str, + connection_mode: GitConnectionMode = GitConnectionMode.HTTPS, + token: str | None = None, + ) -> str: + if owner != self._USER: + return super().get_remote_url(owner, repo_name, connection_mode, token) + # redirect to the upstream repository + try: + upstream_owner = self._repos[repo_name] + except KeyError: + raise RepositoryNotFoundError( + f"Repository {owner}/{repo_name} appears to be a virtual fork but does not exist. Note that dry-run " + "forks are persistent only for the duration of the backend instance." + ) + + return super().get_remote_url(upstream_owner, repo_name, connection_mode, token) + def push_to_repository( self, owner: str, repo_name: str, git_dir: Path, branch: str ): @@ -967,7 +994,7 @@ def fork(self, owner: str, repo_name: str): logger.debug( f"Dry Run: Creating fork of {owner}/{repo_name} for user {self._USER}." ) - self._repos.add(repo_name) + self._repos[repo_name] = owner def _sync_default_branch(self, upstream_owner: str, upstream_repo: str): logger.debug( diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 2fd371979..ec2535bbc 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1044,25 +1044,6 @@ def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(c assert "trying to reset hard" in caplog.text -def test_git_platform_backend_get_remote_url_https(): - owner = "OWNER" - repo = "REPO" - - url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS) - - assert url == f"https://github.com/{owner}/{repo}.git" - - -def test_git_platform_backend_get_remote_url_token(): - owner = "OWNER" - repo = "REPO" - token = "TOKEN" - - url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS, token) - - assert url == f"https://{token}@github.com/{owner}/{repo}.git" - - def test_github_backend_from_token(): token = "TOKEN" @@ -1106,6 +1087,27 @@ def test_github_backend_does_repository_exist(does_exist: bool): github3_client.repository.assert_called_once_with("OWNER", "REPO") +def test_github_backend_get_remote_url_https(): + owner = "OWNER" + repo = "REPO" + backend = GitHubBackend(MagicMock(), MagicMock(), "") + + url = backend.get_remote_url(owner, repo, GitConnectionMode.HTTPS) + + assert url == f"https://github.com/{owner}/{repo}.git" + + +def test_github_backend_get_remote_url_token(): + owner = "OWNER" + repo = "REPO" + token = "TOKEN" + backend = GitHubBackend(MagicMock(), MagicMock(), "") + + url = backend.get_remote_url(owner, repo, GitConnectionMode.HTTPS, token) + + assert url == f"https://{token}@github.com/{owner}/{repo}.git" + + @mock.patch("time.sleep", return_value=None) @mock.patch( "conda_forge_tick.git_utils.GitHubBackend.user", new_callable=mock.PropertyMock @@ -1609,6 +1611,48 @@ def test_dry_run_backend_does_repository_exist_other_repo(): ) +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_non_fork(token: str | None): + backend = DryRunBackend() + + url = backend.get_remote_url("OWNER", "REPO", GitConnectionMode.HTTPS, token) + + if token is None: + assert url == "https://github.com/OWNER/REPO.git" + else: + assert url == "https://TOKEN@github.com/OWNER/REPO.git" + + +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_non_existing_fork(token: str | None): + backend = DryRunBackend() + + with pytest.raises(RepositoryNotFoundError, match="does not exist"): + backend.get_remote_url(backend.user, "REPO", GitConnectionMode.HTTPS, token) + + backend.fork("UPSTREAM_OWNER", "REPO2") + + with pytest.raises(RepositoryNotFoundError, match="does not exist"): + backend.get_remote_url(backend.user, "REPO", GitConnectionMode.HTTPS, token) + + +@pytest.mark.parametrize("token", [None, "TOKEN"]) +def test_dry_run_backend_get_remote_url_existing_fork(token: str | None): + backend = DryRunBackend() + + backend.fork("UPSTREAM_OWNER", "pytest-feedstock") + + url = backend.get_remote_url( + backend.user, "pytest-feedstock", GitConnectionMode.HTTPS, token + ) + + # note that the URL does not indicate anymore that it is a fork + assert ( + url + == f"https://{f'{token}@' if token else ''}github.com/UPSTREAM_OWNER/pytest-feedstock.git" + ) + + def test_dry_run_backend_push_to_repository(caplog): caplog.set_level(logging.DEBUG) From 38c2e7e6b89c48764bb25b65c27e14f44b52207f Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 21:39:02 +0200 Subject: [PATCH 20/38] detect duplicate pull requests --- conda_forge_tick/git_utils.py | 28 ++- tests/github_api/create_pull_duplicate.json | 12 ++ .../create_pull_validation_error.json | 12 ++ tests/test_git_utils.py | 179 ++++++++++++++---- 4 files changed, 185 insertions(+), 46 deletions(-) create mode 100644 tests/github_api/create_pull_duplicate.json create mode 100644 tests/github_api/create_pull_validation_error.json diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index dd2715e2b..f58baf58b 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -146,6 +146,14 @@ class GitPlatformError(Exception): pass +class DuplicatePullRequestError(GitPlatformError): + """ + Raised if a pull request already exists. + """ + + pass + + class RepositoryNotFoundError(Exception): """ Raised when a repository is not found. @@ -685,6 +693,7 @@ def create_pull_request( :returns: The data of the created pull request. :raises GitPlatformError: If the pull request could not be created. + :raises DuplicatePullRequestError: If a pull request already exists and the backend checks for it. """ pass @@ -882,12 +891,19 @@ def create_pull_request( target_owner, target_repo ) - response: github3.pulls.ShortPullRequest | None = repo.create_pull( - title=title, - base=base_branch, - head=f"{self.user}:{head_branch}", - body=body, - ) + try: + response: github3.pulls.ShortPullRequest | None = repo.create_pull( + title=title, + base=base_branch, + head=f"{self.user}:{head_branch}", + body=body, + ) + except github3.exceptions.UnprocessableEntity as e: + if any("already exists" in error.get("message", "") for error in e.errors): + raise DuplicatePullRequestError( + f"Pull request from {self.user}:{head_branch} to {target_owner}:{base_branch} already exists." + ) from e + raise if response is None: raise GitPlatformError("Could not create pull request.") diff --git a/tests/github_api/create_pull_duplicate.json b/tests/github_api/create_pull_duplicate.json new file mode 100644 index 000000000..aff82e963 --- /dev/null +++ b/tests/github_api/create_pull_duplicate.json @@ -0,0 +1,12 @@ +{ + "message": "Validation Failed", + "errors": [ + { + "resource": "PullRequest", + "code": "custom", + "message": "A pull request already exists for OWNER:BRANCH." + } + ], + "documentation_url": "https://docs.github.com/rest/pulls/pulls#create-a-pull-request", + "status": "422" +} diff --git a/tests/github_api/create_pull_validation_error.json b/tests/github_api/create_pull_validation_error.json new file mode 100644 index 000000000..ae6f3df17 --- /dev/null +++ b/tests/github_api/create_pull_validation_error.json @@ -0,0 +1,12 @@ +{ + "message": "Validation Failed", + "errors": [ + { + "resource": "PullRequest", + "field": "head", + "code": "invalid" + } + ], + "documentation_url": "https://docs.github.com/rest/pulls/pulls#create-a-pull-request", + "status": "422" +} diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index ec2535bbc..073162389 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -16,6 +16,7 @@ from conda_forge_tick.git_utils import ( Bound, DryRunBackend, + DuplicatePullRequestError, GitCli, GitCliError, GitConnectionMode, @@ -1044,6 +1045,41 @@ def test_git_cli_clone_fork_and_branch_non_existing_remote_existing_target_dir(c assert "trying to reset hard" in caplog.text +def _github_api_json_fixture(name: str) -> dict: + with Path(__file__).parent.joinpath(f"github_api/{name}.json").open() as f: + return json.load(f) + + +@pytest.fixture() +def github_response_create_issue_comment() -> dict: + return _github_api_json_fixture("create_issue_comment_pytest") + + +@pytest.fixture() +def github_response_create_pull_duplicate() -> dict: + return _github_api_json_fixture("create_pull_duplicate") + + +@pytest.fixture() +def github_response_create_pull_validation_error() -> dict: + return _github_api_json_fixture("create_pull_validation_error") + + +@pytest.fixture() +def github_response_get_pull() -> dict: + return _github_api_json_fixture("get_pull_pytest") + + +@pytest.fixture() +def github_response_get_repo() -> dict: + return _github_api_json_fixture("get_repo_pytest") + + +@pytest.fixture() +def github_response_headers() -> dict: + return _github_api_json_fixture("github_response_headers") + + def test_github_backend_from_token(): token = "TOKEN" @@ -1305,28 +1341,23 @@ def test_github_backend_get_api_requests_left_zero_valid_reset_time(caplog): @mock.patch("requests.Session.request") -def test_github_backend_create_pull_request_mock(request_mock: MagicMock): - with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: - get_repo_response = json.load(f) - - with open( - Path(__file__).parent / "github_api" / "github_response_headers.json" - ) as f: - response_headers = json.load(f) - - with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: - create_pull_response = json.load(f) - +def test_github_backend_create_pull_request_mock( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_headers: dict, + github_response_get_pull: dict, +): def request_side_effect(method, _url, **_kwargs): response = requests.Response() if method == "GET": response.status_code = 200 - response.json = lambda: get_repo_response + response.json = lambda: github_response_get_repo return response if method == "POST": response.status_code = 201 - response.json = lambda: create_pull_response - response.headers = CaseInsensitiveDict(response_headers) + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_get_pull + response.headers = CaseInsensitiveDict(github_response_headers) return response assert False, f"Unexpected method: {method}" @@ -1380,18 +1411,93 @@ def request_side_effect(method, _url, **_kwargs): @mock.patch("requests.Session.request") -def test_github_backend_comment_on_pull_request_success(request_mock: MagicMock): - with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: - get_repo_response = json.load(f) +def test_github_backend_create_pull_request_duplicate( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_create_pull_duplicate: dict, +): + def request_side_effect(method, _url, **_kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if method == "POST": + response.status_code = 422 + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_create_pull_duplicate + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") + + with pytest.raises( + DuplicatePullRequestError, + match="Pull request from CURRENT_USER:HEAD_BRANCH to conda-forge:BASE_BRANCH already exists", + ): + backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) + + +@mock.patch("requests.Session.request") +def test_github_backend_create_pull_request_validation_error( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_create_pull_validation_error: dict, +): + """ + Test that other GitHub API 422 validation errors are not caught as DuplicatePullRequestError. + """ - with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: - get_pull_response = json.load(f) + def request_side_effect(method, _url, **_kwargs): + response = requests.Response() + if method == "GET": + response.status_code = 200 + response.json = lambda: github_response_get_repo + return response + if method == "POST": + response.status_code = 422 + # note that the "create pull" response body is identical to the "get pull" response body + response.json = lambda: github_response_create_pull_validation_error + return response + assert False, f"Unexpected method: {method}" + + request_mock.side_effect = request_side_effect + + pygithub_mock = MagicMock() + pygithub_mock.get_user.return_value.login = "CURRENT_USER" + + backend = GitHubBackend(github3.login(token="TOKEN"), pygithub_mock, "") + + with pytest.raises(github3.exceptions.UnprocessableEntity): + backend.create_pull_request( + "conda-forge", + "pytest-feedstock", + "BASE_BRANCH", + "HEAD_BRANCH", + "TITLE", + "BODY", + ) - with open( - Path(__file__).parent / "github_api" / "create_issue_comment_pytest.json" - ) as f: - create_comment_response = json.load(f) +@mock.patch("requests.Session.request") +def test_github_backend_comment_on_pull_request_success( + request_mock: MagicMock, + github_response_get_repo: dict, + github_response_get_pull: dict, + github_response_create_issue_comment: dict, +): def request_side_effect(method, url, **_kwargs): response = requests.Response() if ( @@ -1399,7 +1505,7 @@ def request_side_effect(method, url, **_kwargs): and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" ): response.status_code = 200 - response.json = lambda: get_repo_response + response.json = lambda: github_response_get_repo return response if ( method == "GET" @@ -1407,7 +1513,7 @@ def request_side_effect(method, url, **_kwargs): == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" ): response.status_code = 200 - response.json = lambda: get_pull_response + response.json = lambda: github_response_get_pull return response if ( method == "POST" @@ -1415,7 +1521,7 @@ def request_side_effect(method, url, **_kwargs): == "https://api.github.com/repos/conda-forge/pytest-feedstock/issues/1337/comments" ): response.status_code = 201 - response.json = lambda: create_comment_response + response.json = lambda: github_response_create_issue_comment return response assert False, f"Unexpected endpoint: {method} {url}" @@ -1467,10 +1573,8 @@ def request_side_effect(method, url, **_kwargs): @mock.patch("requests.Session.request") def test_github_backend_comment_on_pull_request_pull_request_not_found( request_mock: MagicMock, + github_response_get_repo: dict, ): - with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: - get_repo_response = json.load(f) - def request_side_effect(method, url, **_kwargs): response = requests.Response() if ( @@ -1478,7 +1582,7 @@ def request_side_effect(method, url, **_kwargs): and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" ): response.status_code = 200 - response.json = lambda: get_repo_response + response.json = lambda: github_response_get_repo return response if ( method == "GET" @@ -1507,22 +1611,17 @@ def request_side_effect(method, url, **_kwargs): @mock.patch("requests.Session.request") def test_github_backend_comment_on_pull_request_unexpected_response( request_mock: MagicMock, + github_response_get_repo: dict, + github_response_get_pull: dict, ): - with open(Path(__file__).parent / "github_api" / "get_repo_pytest.json") as f: - get_repo_response = json.load(f) - - with open(Path(__file__).parent / "github_api" / "get_pull_pytest.json") as f: - get_pull_response = json.load(f) - def request_side_effect(method, url, **_kwargs): - # noinspection DuplicatedCode response = requests.Response() if ( method == "GET" and url == "https://api.github.com/repos/conda-forge/pytest-feedstock" ): response.status_code = 200 - response.json = lambda: get_repo_response + response.json = lambda: github_response_get_repo return response if ( method == "GET" @@ -1530,7 +1629,7 @@ def request_side_effect(method, url, **_kwargs): == "https://api.github.com/repos/conda-forge/pytest-feedstock/pulls/1337" ): response.status_code = 200 - response.json = lambda: get_pull_response + response.json = lambda: github_response_get_pull return response if ( method == "POST" From 7fad40ee9a4206fbd301e9e7c7322bb0d7e91da7 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 22:28:36 +0200 Subject: [PATCH 21/38] refactor auto_tick.run using the new git backend --- conda_forge_tick/auto_tick.py | 622 +++++++++++++++++------------ conda_forge_tick/git_utils.py | 130 +----- conda_forge_tick/models/pr_json.py | 1 + 3 files changed, 375 insertions(+), 378 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index b7a42f49a..14e5ba857 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -4,12 +4,16 @@ import logging import os import random +import shutil +import textwrap import time import traceback import typing -from subprocess import CalledProcessError -from textwrap import dedent -from typing import MutableMapping, Tuple, cast +from dataclasses import dataclass +from typing import Literal, cast + +from .models.pr_info import PullRequestInfoSpecial +from .models.pr_json import PullRequestData, PullRequestState if typing.TYPE_CHECKING: from .migrators_types import MigrationUidTypedDict @@ -24,16 +28,22 @@ from conda.models.version import VersionOrder from conda_forge_tick.cli_context import CliContext -from conda_forge_tick.contexts import FeedstockContext, MigratorSessionContext from conda_forge_tick.deploy import deploy +from conda_forge_tick.contexts import ( + GIT_CLONE_DIR, + FeedstockContext, + MigratorSessionContext, +) from conda_forge_tick.feedstock_parser import BOOTSTRAP_MAPPINGS from conda_forge_tick.git_utils import ( - GIT_CLONE_DIR, - comment_on_pr, - get_repo, + DryRunBackend, + DuplicatePullRequestError, + GitCli, + GitCliError, + GitPlatformBackend, + RepositoryNotFoundError, github_backend, is_github_api_limit_reached, - push_repo, ) from conda_forge_tick.lazy_json_backends import ( LazyJson, @@ -49,7 +59,7 @@ from conda_forge_tick.migration_runner import run_migration from conda_forge_tick.migrators import MigrationYaml, Migrator, Version from conda_forge_tick.migrators.version import VersionMigrationError -from conda_forge_tick.os_utils import eval_cmd, pushd +from conda_forge_tick.os_utils import eval_cmd from conda_forge_tick.rerender_feedstock import rerender_feedstock from conda_forge_tick.solver_checks import is_recipe_solvable from conda_forge_tick.utils import ( @@ -139,19 +149,275 @@ def _get_pre_pr_migrator_attempts(attrs, migrator_name, *, is_version): return pri.get("pre_pr_migrator_attempts", {}).get(migrator_name, 0) +def _prepare_feedstock_repository( + backend: GitPlatformBackend, + context: FeedstockContext, + branch: str, + base_branch: str, +) -> bool: + """ + Prepare a feedstock repository for migration by forking and cloning it. The local clone will be present in + context.local_clone_dir. + + Any errors are written to the pr_info attribute of the feedstock context and logged. + + :param backend: The GitPlatformBackend instance to use. + :param context: The FeedstockContext instance. + :param branch: The branch to create in the forked repository. + :param base_branch: The base branch to branch from. + :return: True if the repository was successfully prepared, False otherwise. + """ + try: + backend.fork(context.git_repo_owner, context.git_repo_name) + except RepositoryNotFoundError: + logger.warning( + f"Could not fork {context.git_repo_owner}/{context.git_repo_name}: Not Found" + ) + + error_message = f"{context.feedstock_name}: Git repository not found." + logger.critical( + f"Failed to migrate {context.feedstock_name}, {error_message}", + ) + + with context.attrs["pr_info"] as pri: + pri["bad"] = error_message + + return False + + backend.clone_fork_and_branch( + upstream_owner=context.git_repo_owner, + repo_name=context.git_repo_name, + target_dir=context.local_clone_dir, + new_branch=branch, + base_branch=base_branch, + ) + return True + + +def _commit_migration( + cli: GitCli, + context: FeedstockContext, + commit_message: str, + allow_empty_commits: bool = False, + raise_commit_errors: bool = True, +) -> None: + """ + Commit a migration that has been run in the local clone of a feedstock repository. + If an error occurs during the commit, it is logged. + + :param cli: The GitCli instance to use. + :param context: The FeedstockContext instance. + :param commit_message: The commit message to use. + :param allow_empty_commits: Whether the migrator allows empty commits. + :param raise_commit_errors: Whether to raise an exception if an error occurs during the commit. + + :raises GitCliError: If an error occurs during the commit and raise_commit_errors is True. + """ + cli.add( + context.local_clone_dir, + all_=True, + ) + + try: + cli.commit( + context.local_clone_dir, commit_message, allow_empty=allow_empty_commits + ) + except GitCliError as e: + logger.info("could not commit to feedstock - likely no changes", exc_info=e) + + if raise_commit_errors: + raise + + +@dataclass(frozen=True) +class _RerenderInfo: + """ + Additional information about a rerender operation. + """ + + nontrivial_migration_yaml_changes: bool + """ + True if any files which are not in the following list were changed during the rerender, False otherwise: + 1. anything in the recipe directory + 2. anything in the migrators directory + 3. the README file + + This is useful to discard MigrationYaml migrations that only drop a file in the migrations directory. + """ + rerender_comment: str | None = None + """ + If requested, a comment to be added to the PR to indicate an issue with the rerender. + None if no comment should be added. + """ + + +def _run_rerender( + git_cli: GitCli, context: FeedstockContext, suppress_errors: bool = False +) -> _RerenderInfo: + logger.info("Rerendering the feedstock") + + try: + rerender_msg = rerender_feedstock(str(context.local_clone_dir), timeout=900) + except Exception as e: + logger.error("RERENDER ERROR", exc_info=e) + + if not suppress_errors: + raise + + rerender_comment = textwrap.dedent( + """ + Hi! This feedstock was not able to be rerendered after the version update changes. I + have pushed the version update changes anyways and am trying to rerender again with this + comment. Hopefully you all can fix this! + + @conda-forge-admin rerender + """ + ) + + return _RerenderInfo( + nontrivial_migration_yaml_changes=False, rerender_comment=rerender_comment + ) + + if rerender_msg is None: + return _RerenderInfo(nontrivial_migration_yaml_changes=False) + + git_cli.commit(context.local_clone_dir, rerender_msg, all_=True, allow_empty=True) + + # HEAD~ is the state before the last commit + changed_files = git_cli.diffed_files(context.local_clone_dir, "HEAD~") + + recipe_dir = context.local_clone_dir / "recipe" + migrators_dir = context.local_clone_dir / "migrators" + + nontrivial_migration_yaml_changes = any( + not file.is_relative_to(recipe_dir) + and not file.is_relative_to(migrators_dir) + and not file.name.startswith("README") + for file in changed_files + ) + + return _RerenderInfo(nontrivial_migration_yaml_changes) + + +def _has_automerge(migrator: Migrator, context: FeedstockContext) -> bool: + """ + Determine if a migration should be auto merged based on the feedstock and migrator settings. + + :param migrator: The migrator to check. + :param context: The feedstock context. + + :return: True if the migrator should be auto merged, False otherwise. + """ + if isinstance(migrator, Version): + return context.automerge in [True, "version"] + else: + return getattr(migrator, "automerge", False) and context.automerge in [ + True, + "migration", + ] + + +def _is_solvability_check_needed( + migrator: Migrator, context: FeedstockContext, base_branch: str +) -> bool: + migrator_check_solvable = getattr(migrator, "check_solvable", True) + pr_attempts = _get_pre_pr_migrator_attempts( + context.attrs, + migrator_name=get_migrator_name(migrator), + is_version=isinstance(migrator, Version), + ) + max_pr_attempts = getattr( + migrator, "force_pr_after_solver_attempts", MAX_SOLVER_ATTEMPTS * 2 + ) + + logger.info( + textwrap.dedent( + f""" + automerge and check_solvable status/settings: + automerge: + feedstock_automerge: {context.automerge} + migrator_automerge: {getattr(migrator, 'automerge', False)} + has_automerge: {_has_automerge(migrator, context)} (only considers feedstock if version migration) + check_solvable: + feedstock_check_solvable: {context.check_solvable} + migrator_check_solvable: {migrator_check_solvable} + pre_pr_migrator_attempts: {pr_attempts} + force_pr_after_solver_attempts: {max_pr_attempts} + """ + ) + ) + + return ( + context.feedstock_name != "conda-forge-pinning" + and (base_branch == "master" or base_branch == "main") + # feedstocks that have problematic bootstrapping will not always be solvable + and context.feedstock_name not in BOOTSTRAP_MAPPINGS + # stuff in cycles always goes + and context.attrs["name"] not in getattr(migrator, "cycles", set()) + # stuff at the top always goes + and context.attrs["name"] not in getattr(migrator, "top_level", set()) + # either the migrator or the feedstock has to request solver checks + and (migrator_check_solvable or context.check_solvable) + # we try up to MAX_SOLVER_ATTEMPTS times, and then we just skip + # the solver check and issue the PR if automerge is off + and (_has_automerge(migrator, context) or (pr_attempts < max_pr_attempts)) + ) + + +def _handle_solvability_error( + errors: list[str], context: FeedstockContext, migrator: Migrator, base_branch: str +) -> None: + ci_url = get_bot_run_url() + ci_url = f"(bot CI job)" if ci_url else "" + _solver_err_str = textwrap.dedent( + f""" + not solvable {ci_url} @ {base_branch} +
+
+ 
+        {'
'.join(sorted(set(errors)))}
+
+
+
+ """, + ).strip() + + _set_pre_pr_migrator_error( + context.attrs, + get_migrator_name(migrator), + _solver_err_str, + is_version=isinstance(migrator, Version), + ) + + # remove part of a try for solver errors to make those slightly + # higher priority next time the bot runs + if isinstance(migrator, Version): + with context.attrs["version_pr_info"] as vpri: + _new_ver = vpri["new_version"] + vpri["new_version_attempts"][_new_ver] -= 0.8 + + +def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: + return PullRequestInfoSpecial( + id=str(uuid4()), + merged_at="never issued", + state="closed", + ) + + def run( - feedstock_ctx: FeedstockContext, + context: FeedstockContext, migrator: Migrator, rerender: bool = True, base_branch: str = "main", dry_run: bool = False, **kwargs: typing.Any, -) -> Tuple["MigrationUidTypedDict", dict]: +) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration Parameters ---------- - feedstock_ctx: FeedstockContext + context: FeedstockContext The node attributes migrator: Migrator instance The migrator to run on the feedstock @@ -170,301 +436,158 @@ def run( The migration return dict used for tracking finished migrations pr_json: dict The PR json object for recreating the PR as needed + + Exceptions + ---------- + GitCliError + If an error occurs during a git command which is not suppressed """ + git_backend: GitPlatformBackend = DryRunBackend() if dry_run else github_backend() # sometimes we get weird directory issues so make sure we reset os.chdir(BOT_HOME_DIR) - # get the repo - branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[0:6] - migrator_name = get_migrator_name(migrator) is_version_migration = isinstance(migrator, Version) _increment_pre_pr_migrator_attempt( - feedstock_ctx.attrs, + context.attrs, migrator_name, is_version=is_version_migration, ) - # TODO: run this in parallel - feedstock_dir, repo = get_repo( - fctx=feedstock_ctx, branch=branch_name, base_branch=base_branch - ) - if not feedstock_dir or not repo: - logger.critical( - "Failed to migrate %s, %s", - feedstock_ctx.feedstock_name, - feedstock_ctx.attrs.get("pr_info", {}).get("bad"), - ) + branch_name = migrator.remote_branch(context) + "_h" + uuid4().hex[0:6] + if not _prepare_feedstock_repository( + git_backend, context, branch_name, base_branch + ): + # something went wrong during forking or cloning return False, False - # need to use an absolute path here - feedstock_dir = os.path.abspath(feedstock_dir) - + # feedstock_dir must be absolute migration_run_data = run_migration( migrator=migrator, - feedstock_dir=feedstock_dir, - feedstock_name=feedstock_ctx.feedstock_name, - node_attrs=feedstock_ctx.attrs, - default_branch=feedstock_ctx.default_branch, + feedstock_dir=str(context.local_clone_dir.resolve()), + feedstock_name=context.feedstock_name, + node_attrs=context.attrs, + default_branch=context.default_branch, **kwargs, ) if not migration_run_data["migrate_return_value"]: logger.critical( - "Failed to migrate %s, %s", - feedstock_ctx.feedstock_name, - feedstock_ctx.attrs.get("pr_info", {}).get("bad"), + f"Failed to migrate {context.feedstock_name}, {context.attrs.get('pr_info', {}).get('bad')}", ) - eval_cmd(["rm", "-rf", feedstock_dir]) + shutil.rmtree(context.local_clone_dir) return False, False - # rerender, maybe - diffed_files: typing.List[str] = [] - with pushd(feedstock_dir): - msg = migration_run_data["commit_message"] - try: - eval_cmd(["git", "add", "--all", "."]) - if migrator.allow_empty_commits: - eval_cmd(["git", "commit", "--allow-empty", "-am", msg]) - else: - eval_cmd(["git", "commit", "-am", msg]) - except CalledProcessError as e: - logger.info( - "could not commit to feedstock - " - "likely no changes - error is '%s'" % (repr(e)), - ) - # we bail here if we do not plan to rerender and we wanted an empty - # commit - # this prevents PRs that don't actually get made from getting marked as done - if migrator.allow_empty_commits and not rerender: - raise e - - if rerender: - head_ref = eval_cmd(["git", "rev-parse", "HEAD"]).strip() - logger.info("Rerendering the feedstock") - - try: - rerender_msg = rerender_feedstock(feedstock_dir, timeout=900) - if rerender_msg is not None: - eval_cmd(["git", "commit", "--allow-empty", "-am", rerender_msg]) - - make_rerender_comment = False - except Exception as e: - # I am trying this bit of code to force these errors - # to be surfaced in the logs at the right time. - print(f"RERENDER ERROR: {e}", flush=True) - if not isinstance(migrator, Version): - raise - else: - # for check solvable or automerge, we always raise rerender errors - if get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "check_solvable"], - {}, - False, - ) or get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "automerge"], - {}, - False, - ): - raise - else: - make_rerender_comment = True - - # If we tried to run the MigrationYaml and rerender did nothing (we only - # bumped the build number and dropped a yaml file in migrations) bail - # for instance platform specific migrations - gdiff = eval_cmd( - ["git", "diff", "--name-only", f"{head_ref.strip()}...HEAD"] - ) - - diffed_files = [ - _ - for _ in gdiff.split() - if not ( - _.startswith("recipe") - or _.startswith("migrators") - or _.startswith("README") - ) - ] - else: - make_rerender_comment = False - - feedstock_automerge = get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "automerge"], - {}, - False, + # We raise an exception if we don't plan to rerender and wanted an empty commit. + # This prevents PRs that don't actually get made from getting marked as done. + _commit_migration( + cli=git_backend.cli, + context=context, + commit_message=migration_run_data["commit_message"], + allow_empty_commits=migrator.allow_empty_commits, + raise_commit_errors=migrator.allow_empty_commits and not rerender, ) - if isinstance(migrator, Version): - has_automerge = feedstock_automerge in [True, "version"] - else: - has_automerge = getattr( - migrator, "automerge", False - ) and feedstock_automerge in [True, "migration"] - migrator_check_solvable = getattr(migrator, "check_solvable", True) - feedstock_check_solvable = get_keys_default( - feedstock_ctx.attrs, - ["conda-forge.yml", "bot", "check_solvable"], - {}, - False, - ) - pr_attempts = _get_pre_pr_migrator_attempts( - feedstock_ctx.attrs, - migrator_name, - is_version=is_version_migration, - ) - max_pr_attempts = getattr( - migrator, "force_pr_after_solver_attempts", MAX_SOLVER_ATTEMPTS * 2 - ) + if rerender: + # for version migrations, check solvable or automerge, we always raise rerender errors + suppress_errors = ( + not is_version_migration + and not context.check_solvable + and not context.automerge + ) - logger.info( - f"""automerge and check_solvable status/settings: - automerge: - feedstock_automerge: {feedstock_automerge} - migratror_automerge: {getattr(migrator, 'automerge', False)} - has_automerge: {has_automerge} (only considers feedstock if version migration) - check_solvable: - feedstock_checksolvable: {feedstock_check_solvable} - migrator_check_solvable: {migrator_check_solvable} - pre_pr_migrator_attempts: {pr_attempts} - force_pr_after_solver_attempts: {max_pr_attempts} -""" - ) + rerender_info = _run_rerender(git_backend.cli, context, suppress_errors) + else: + rerender_info = _RerenderInfo(nontrivial_migration_yaml_changes=False) - if ( - feedstock_ctx.feedstock_name != "conda-forge-pinning" - and (base_branch == "master" or base_branch == "main") - # feedstocks that have problematic bootstrapping will not always be solvable - and feedstock_ctx.feedstock_name not in BOOTSTRAP_MAPPINGS - # stuff in cycles always goes - and feedstock_ctx.attrs["name"] not in getattr(migrator, "cycles", set()) - # stuff at the top always goes - and feedstock_ctx.attrs["name"] not in getattr(migrator, "top_level", set()) - # either the migrator or the feedstock has to request solver checks - and (migrator_check_solvable or feedstock_check_solvable) - # we try up to MAX_SOLVER_ATTEMPTS times and then we just skip - # the solver check and issue the PR if automerge is off - and (has_automerge or (pr_attempts < max_pr_attempts)) - ): - solvable, errors, _ = is_recipe_solvable( - feedstock_dir, - build_platform=feedstock_ctx.attrs["conda-forge.yml"].get( + if _is_solvability_check_needed(migrator, context, base_branch): + solvable, solvability_errors, _ = is_recipe_solvable( + str(context.local_clone_dir), + build_platform=context.attrs["conda-forge.yml"].get( "build_platform", None, ), ) if not solvable: - ci_url = get_bot_run_url() - ci_url = f"(bot CI job)" if ci_url else "" - _solver_err_str = dedent( - f""" - not solvable {ci_url} @ {base_branch} -
-
- 
-                {'
'.join(sorted(set(errors)))}
-
-
-
- """, - ).strip() - - _set_pre_pr_migrator_error( - feedstock_ctx.attrs, - migrator_name, - _solver_err_str, - is_version=is_version_migration, + _handle_solvability_error( + solvability_errors, context, migrator, base_branch ) - - # remove part of a try for solver errors to make those slightly - # higher priority next time the bot runs - if isinstance(migrator, Version): - with feedstock_ctx.attrs["version_pr_info"] as vpri: - _new_ver = vpri["new_version"] - vpri["new_version_attempts"][_new_ver] -= 0.8 - - eval_cmd(["rm", "-rf", feedstock_dir]) + shutil.rmtree(context.local_clone_dir) return False, False else: _reset_pre_pr_migrator_fields( - feedstock_ctx.attrs, migrator_name, is_version=is_version_migration + context.attrs, migrator_name, is_version=is_version_migration ) - # TODO: Better annotation here - pr_json: typing.Union[MutableMapping, None, bool] + pr_data: PullRequestData | PullRequestInfoSpecial | None = None + """ + The PR data for the PR that was created. The contents of this variable will be stored in the bot's database. + None means: We don't update the PR data. + """ if ( isinstance(migrator, MigrationYaml) - and not diffed_files - and feedstock_ctx.attrs["name"] != "conda-forge-pinning" + and not rerender_info.nontrivial_migration_yaml_changes + and context.attrs["name"] != "conda-forge-pinning" ): # spoof this so it looks like the package is done - pr_json = { - "state": "closed", - "merged_at": "never issued", - "id": str(uuid4()), - } + pr_data = get_spoofed_closed_pr_info() else: - # push up + # push and PR + git_backend.push_to_repository( + owner=git_backend.user, + repo_name=context.git_repo_name, + git_dir=context.local_clone_dir, + branch=branch_name, + ) try: - # TODO: remove this hack, but for now this is the only way to get - # the feedstock dir into pr_body - feedstock_ctx.feedstock_dir = feedstock_dir - pr_json = push_repo( - fctx=feedstock_ctx, - feedstock_dir=feedstock_dir, - body=migration_run_data["pr_body"], - repo=repo, - title=migration_run_data["pr_title"], - branch=branch_name, + pr_data = git_backend.create_pull_request( + target_owner=context.git_repo_owner, + target_repo=context.git_repo_name, base_branch=base_branch, - dry_run=dry_run, + head_branch=branch_name, + title=migration_run_data["pr_title"], + body=migration_run_data["pr_body"], ) + except DuplicatePullRequestError: + # This shouldn't happen too often anymore since we won't double PR + logger.warning( + f"Attempted to create a duplicate PR for merging {git_backend.user}:{branch_name} " + f"into {context.git_repo_owner}:{base_branch}. Ignoring." + ) + # Don't update the PR data (keep pr_data as None) - # This shouldn't happen too often any more since we won't double PR - except github3.GitHubError as e: - if e.msg != "Validation Failed": - raise - else: - print(f"Error during push {e}") - # If we just push to the existing PR then do nothing to the json - pr_json = False - ljpr = False - - if pr_json and pr_json["state"] != "closed" and make_rerender_comment: - comment_on_pr( - pr_json, - """\ -Hi! This feedstock was not able to be rerendered after the version update changes. I -have pushed the version update changes anyways and am trying to rerender again with this -comment. Hopefully you all can fix this! - -@conda-forge-admin rerender""", - repo, + if ( + pr_data + and pr_data.state != PullRequestState.CLOSED + and rerender_info.rerender_comment + ): + git_backend.comment_on_pull_request( + repo_owner=context.git_repo_owner, + repo_name=context.git_repo_name, + pr_number=pr_data.number, + comment=rerender_info.rerender_comment, ) - if pr_json: - ljpr = LazyJson( - os.path.join("pr_json", str(pr_json["id"]) + ".json"), + if pr_data: + pr_lazy_json = LazyJson( + os.path.join("pr_json", f"{pr_data.id}.json"), ) - with ljpr as __ljpr: - __ljpr.update(**pr_json) + with pr_lazy_json as __edit_pr_lazy_json: + __edit_pr_lazy_json.update(**pr_data.model_dump()) else: - ljpr = False + pr_lazy_json = False # If we've gotten this far then the node is good - with feedstock_ctx.attrs["pr_info"] as pri: + with context.attrs["pr_info"] as pri: pri["bad"] = False _reset_pre_pr_migrator_fields( - feedstock_ctx.attrs, migrator_name, is_version=is_version_migration + context.attrs, migrator_name, is_version=is_version_migration ) logger.info("Removing feedstock dir") - eval_cmd(["rm", "-rf", feedstock_dir]) - return migration_run_data["migrate_return_value"], ljpr + shutil.rmtree(context.local_clone_dir) + return migration_run_data["migrate_return_value"], pr_lazy_json def _compute_time_per_migrator(mctx, migrators): @@ -558,7 +681,7 @@ def _run_migrator_on_feedstock_branch( "new_version", None ) migrator_uid, pr_json = run( - feedstock_ctx=fctx, + context=fctx, migrator=migrator, rerender=migrator.rerender, base_branch=base_branch, @@ -596,6 +719,7 @@ def _run_migrator_on_feedstock_branch( ) except (github3.GitHubError, github.GithubException) as e: + # TODO: pull this down into run() - also check the other exceptions if hasattr(e, "msg") and e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index f58baf58b..9bc8f7712 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -14,7 +14,7 @@ from email import utils from functools import cached_property from pathlib import Path -from typing import Dict, Iterator, Literal, Optional, Tuple, Union +from typing import Dict, Iterator, Optional, Union import backoff import github @@ -33,7 +33,6 @@ # and pull all the needed info from the various source classes) from conda_forge_tick.lazy_json_backends import LazyJson -from .contexts import FeedstockContext from .executors import lock_git_operation from .models.pr_json import ( GithubPullRequestBase, @@ -43,7 +42,6 @@ PullRequestInfoHead, PullRequestState, ) -from .os_utils import pushd from .utils import get_bot_run_url, replace_tokens, run_command_hiding_token logger = logging.getLogger(__name__) @@ -1104,41 +1102,6 @@ def is_github_api_limit_reached() -> bool: return backend.is_api_limit_reached() -@lock_git_operation() -def get_repo( - fctx: FeedstockContext, - branch: str, - base_branch: str = "main", -) -> Tuple[str, github3.repos.Repository] | Tuple[Literal[False], Literal[False]]: - """Get the feedstock repo - - Parameters - ---------- - fctx : FeedstockContext - Feedstock context used for constructing feedstock urls, etc. - branch : str - The branch to be made. - base_branch : str, optional - The base branch from which to make the new branch. - - Returns - ------- - recipe_dir : str - The recipe directory - repo : github3 repository - The github3 repository object. - """ - - # This is needed because we want to migrate to the new backend step-by-step - repo: github3.repos.Repository | None = github3_client().repository( - fctx.git_repo_owner, fctx.git_repo_name - ) - - assert repo is not None - - return str(feedstock_dir), repo - - @lock_git_operation() def delete_branch(pr_json: LazyJson, dry_run: bool = False) -> None: ref = pr_json["head"]["ref"] @@ -1360,97 +1323,6 @@ def close_out_labels( return None -@lock_git_operation() -def push_repo( - fctx: FeedstockContext, - feedstock_dir: str, - body: str, - repo: github3.repos.Repository, - title: str, - branch: str, - base_branch: str = "main", - dry_run: bool = False, -) -> Union[dict, bool, None]: - """Push a repo up to github - - Parameters - ---------- - fctx : FeedstockContext - Feedstock context used for constructing feedstock urls, etc. - feedstock_dir : str - The feedstock directory - body : str - The PR body. - repo : github3.repos.Repository - The feedstock repo as a github3 object. - title : str - The title of the PR. - head : str, optional - The github head for the PR in the form `username:branch`. - branch : str - The head branch of the PR. - base_branch : str, optional - The base branch or target branch of the PR. - - Returns - ------- - pr_json: dict - The dict representing the PR, can be used with `from_json` - to create a PR instance. - """ - with pushd(feedstock_dir): - # Copyright (c) 2016 Aaron Meurer, Gil Forsyth - token = get_bot_token() - gh_username = github3_client().me().login - - head = gh_username + ":" + branch - - deploy_repo = gh_username + "/" + fctx.feedstock_name + "-feedstock" - if dry_run: - repo_url = f"https://github.com/{deploy_repo}.git" - print(f"dry run: adding remote and pushing up branch for {repo_url}") - else: - ecode = run_command_hiding_token( - [ - "git", - "remote", - "add", - "regro_remote", - f"https://{token}@github.com/{deploy_repo}.git", - ], - token=token, - ) - if ecode != 0: - print("Failed to add git remote!") - return False - - ecode = run_command_hiding_token( - ["git", "push", "--set-upstream", "regro_remote", branch], - token=token, - ) - if ecode != 0: - print("Failed to push to remote!") - return False - - # lastly make a PR for the feedstock - print("Creating conda-forge feedstock pull request...") - if dry_run: - print(f"dry run: create pr with title: {title}") - return False - else: - pr = repo.create_pull(title, base_branch, head, body=body) - if pr is None: - print("Failed to create pull request!") - return False - else: - print("Pull request created at " + pr.html_url) - - # Return a json object so we can remake the PR if needed - pr_dict: dict = pr.as_dict() - - return trim_pr_json_keys(pr_dict) - - def comment_on_pr(pr_json, comment, repo): """Make a comment on a PR. diff --git a/conda_forge_tick/models/pr_json.py b/conda_forge_tick/models/pr_json.py index db7b4789f..4e1ef6f2f 100644 --- a/conda_forge_tick/models/pr_json.py +++ b/conda_forge_tick/models/pr_json.py @@ -89,6 +89,7 @@ class PullRequestDataValid(ValidatedBaseModel): """ Information about a pull request, as retrieved from the GitHub API. Refer to git_utils.PR_KEYS_TO_KEEP for the keys that are kept in the PR object. + ALSO UPDATE PR_KEYS_TO_KEEP IF YOU CHANGE THIS CLASS! GitHub documentation: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#get-a-pull-request """ From 6dcb67e9e7a99ee07f66bb914386ca6c8dc2bf38 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sun, 16 Jun 2024 16:33:51 +0200 Subject: [PATCH 22/38] refactor: solvability checks in separate method --- conda_forge_tick/auto_tick.py | 58 ++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 14e5ba857..d2b2f676c 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -397,6 +397,43 @@ def _handle_solvability_error( vpri["new_version_attempts"][_new_ver] -= 0.8 +def _check_and_process_solvability( + migrator: Migrator, context: FeedstockContext, base_branch: str +) -> bool: + """ + If the migration needs a solvability check, perform the check. If the recipe is not solvable, handle the error + by setting the corresponding fields in the feedstock attributes. + If the recipe is solvable, reset the fields that track the solvability check status. + + :param migrator: The migrator that was run + :param context: The current FeedstockContext of the feedstock that was migrated + :param base_branch: The branch of the feedstock repository that is the migration target + + :returns: True if the migration can proceed normally, False if a required solvability check failed and the migration + needs to be aborted + """ + if not _is_solvability_check_needed(migrator, context, base_branch): + return True + + solvable, solvability_errors, _ = is_recipe_solvable( + str(context.local_clone_dir), + build_platform=context.attrs["conda-forge.yml"].get( + "build_platform", + None, + ), + ) + if solvable: + _reset_pre_pr_migrator_fields( + context.attrs, + get_migrator_name(migrator), + is_version=isinstance(migrator, Version), + ) + return True + + _handle_solvability_error(solvability_errors, context, migrator, base_branch) + return False + + def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: return PullRequestInfoSpecial( id=str(uuid4()), @@ -501,24 +538,9 @@ def run( else: rerender_info = _RerenderInfo(nontrivial_migration_yaml_changes=False) - if _is_solvability_check_needed(migrator, context, base_branch): - solvable, solvability_errors, _ = is_recipe_solvable( - str(context.local_clone_dir), - build_platform=context.attrs["conda-forge.yml"].get( - "build_platform", - None, - ), - ) - if not solvable: - _handle_solvability_error( - solvability_errors, context, migrator, base_branch - ) - shutil.rmtree(context.local_clone_dir) - return False, False - else: - _reset_pre_pr_migrator_fields( - context.attrs, migrator_name, is_version=is_version_migration - ) + if not _check_and_process_solvability(migrator, context, base_branch): + logger.warning("Skipping migration due to solvability check failure") + return False, False pr_data: PullRequestData | PullRequestInfoSpecial | None = None """ From 7a5bfca6bb6616c660829a91824ce7d5ffb8f3bf Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sun, 16 Jun 2024 16:38:43 +0200 Subject: [PATCH 23/38] use temporary directory instead of managing the local feedstock dir manually --- conda_forge_tick/auto_tick.py | 53 ++++++++---- conda_forge_tick/contexts.py | 49 ++++++++---- conda_forge_tick/migration_runner.py | 10 ++- conda_forge_tick/migrators/arch.py | 6 +- conda_forge_tick/migrators/broken_rebuild.py | 3 +- conda_forge_tick/migrators/core.py | 6 +- conda_forge_tick/migrators/migration_yaml.py | 6 +- conda_forge_tick/migrators/replacement.py | 4 +- conda_forge_tick/migrators/version.py | 6 +- tests/test_contexts.py | 84 +++++++++++++++++++- 10 files changed, 177 insertions(+), 50 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index d2b2f676c..cf03bb8cc 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -4,7 +4,6 @@ import logging import os import random -import shutil import textwrap import time import traceback @@ -30,7 +29,7 @@ from conda_forge_tick.cli_context import CliContext from conda_forge_tick.deploy import deploy from conda_forge_tick.contexts import ( - GIT_CLONE_DIR, + ClonedFeedstockContext, FeedstockContext, MigratorSessionContext, ) @@ -151,7 +150,7 @@ def _get_pre_pr_migrator_attempts(attrs, migrator_name, *, is_version): def _prepare_feedstock_repository( backend: GitPlatformBackend, - context: FeedstockContext, + context: ClonedFeedstockContext, branch: str, base_branch: str, ) -> bool: @@ -196,7 +195,7 @@ def _prepare_feedstock_repository( def _commit_migration( cli: GitCli, - context: FeedstockContext, + context: ClonedFeedstockContext, commit_message: str, allow_empty_commits: bool = False, raise_commit_errors: bool = True, @@ -252,7 +251,7 @@ class _RerenderInfo: def _run_rerender( - git_cli: GitCli, context: FeedstockContext, suppress_errors: bool = False + git_cli: GitCli, context: ClonedFeedstockContext, suppress_errors: bool = False ) -> _RerenderInfo: logger.info("Rerendering the feedstock") @@ -398,7 +397,7 @@ def _handle_solvability_error( def _check_and_process_solvability( - migrator: Migrator, context: FeedstockContext, base_branch: str + migrator: Migrator, context: ClonedFeedstockContext, base_branch: str ) -> bool: """ If the migration needs a solvability check, perform the check. If the recipe is not solvable, handle the error @@ -442,20 +441,48 @@ def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: ) -def run( +def run_with_tmpdir( context: FeedstockContext, migrator: Migrator, rerender: bool = True, base_branch: str = "main", dry_run: bool = False, **kwargs: typing.Any, +) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: + """ + For a given feedstock and migration run the migration in a temporary directory that will be deleted after the + migration is complete. + + The parameters are the same as for the `run` function. The only difference is that you pass a FeedstockContext + instance instead of a ClonedFeedstockContext instance. + + The exceptions are the same as for the `run` function. + """ + with context.reserve_clone_directory() as cloned_context: + return run( + context=cloned_context, + migrator=migrator, + rerender=rerender, + base_branch=base_branch, + dry_run=dry_run, + **kwargs, + ) + + +def run( + context: ClonedFeedstockContext, + migrator: Migrator, + rerender: bool = True, + base_branch: str = "main", + dry_run: bool = False, + **kwargs: typing.Any, ) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration Parameters ---------- - context: FeedstockContext - The node attributes + context: ClonedFeedstockContext + The current feedstock context, already containing information about a temporary directory for the feedstock. migrator: Migrator instance The migrator to run on the feedstock rerender : bool @@ -513,7 +540,6 @@ def run( logger.critical( f"Failed to migrate {context.feedstock_name}, {context.attrs.get('pr_info', {}).get('bad')}", ) - shutil.rmtree(context.local_clone_dir) return False, False # We raise an exception if we don't plan to rerender and wanted an empty commit. @@ -607,8 +633,6 @@ def run( context.attrs, migrator_name, is_version=is_version_migration ) - logger.info("Removing feedstock dir") - shutil.rmtree(context.local_clone_dir) return migration_run_data["migrate_return_value"], pr_lazy_json @@ -690,7 +714,7 @@ def _run_migrator_on_feedstock_branch( attrs, base_branch, migrator, - fctx, + fctx: FeedstockContext, dry_run, mctx, migrator_name, @@ -702,7 +726,7 @@ def _run_migrator_on_feedstock_branch( fctx.attrs["new_version"] = attrs.get("version_pr_info", {}).get( "new_version", None ) - migrator_uid, pr_json = run( + migrator_uid, pr_json = run_with_tmpdir( context=fctx, migrator=migrator, rerender=migrator.rerender, @@ -1013,7 +1037,6 @@ def _run_migrator(migrator, mctx, temp, time_per, dry_run): dump_graph(mctx.graph) with filter_reprinted_lines("rm-tmp"): - eval_cmd(["rm", "-rf", f"{GIT_CLONE_DIR}/*"]) for f in glob.glob("/tmp/*"): if f not in temp: try: diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index ac6313c21..873d3f18e 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -1,5 +1,10 @@ +from __future__ import annotations + import os +import tempfile import typing +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path @@ -12,9 +17,6 @@ from conda_forge_tick.migrators_types import AttrsTypedDict -GIT_CLONE_DIR = Path("feedstocks").resolve() - - if os.path.exists("all_feedstocks.json"): with open("all_feedstocks.json") as f: DEFAULT_BRANCHES = load(f).get("default_branches", {}) @@ -32,10 +34,10 @@ class MigratorSessionContext: dry_run: bool = True -@dataclass +@dataclass(frozen=True) class FeedstockContext: feedstock_name: str - attrs: "AttrsTypedDict" + attrs: AttrsTypedDict _default_branch: str = None @property @@ -45,10 +47,6 @@ def default_branch(self): else: return self._default_branch - @default_branch.setter - def default_branch(self, v): - self._default_branch = v - @property def git_repo_owner(self) -> str: return "conda-forge" @@ -64,13 +62,6 @@ def git_href(self) -> str: """ return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}" - @property - def local_clone_dir(self) -> Path: - """ - The local path to the feedstock repository. - """ - return GIT_CLONE_DIR / self.git_repo_name - @property def automerge(self) -> bool | str: """ @@ -100,3 +91,29 @@ def check_solvable(self) -> bool: {}, False, ) + + @contextmanager + def reserve_clone_directory(self) -> Iterator[ClonedFeedstockContext]: + """ + Reserve a temporary directory for the feedstock repository that will be available within the context manager. + The returned context object will contain the path to the feedstock repository in local_clone_dir. + After the context manager exits, the temporary directory will be deleted. + """ + with tempfile.TemporaryDirectory() as tmpdir: + local_clone_dir = Path(tmpdir) / self.git_repo_name + local_clone_dir.mkdir() + yield ClonedFeedstockContext( + **self.__dict__, + local_clone_dir=local_clone_dir, + ) + + +@dataclass(frozen=True, kw_only=True) +class ClonedFeedstockContext(FeedstockContext): + """ + A FeedstockContext object that has reserved a temporary directory for the feedstock repository. + """ + + # Implementation Note: Keep this class frozen or there will be consistency issues if someone modifies + # a ClonedFeedstockContext object in place - it will not be reflected in the original FeedstockContext object. + local_clone_dir: Path diff --git a/conda_forge_tick/migration_runner.py b/conda_forge_tick/migration_runner.py index c76aad201..03c546ea9 100644 --- a/conda_forge_tick/migration_runner.py +++ b/conda_forge_tick/migration_runner.py @@ -3,8 +3,9 @@ import os import shutil import tempfile +from pathlib import Path -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.lazy_json_backends import LazyJson, dumps from conda_forge_tick.os_utils import ( chmod_plus_rwX, @@ -221,11 +222,14 @@ def run_migration_local( - pr_body: The PR body for the migration. """ - feedstock_ctx = FeedstockContext( + # it would be better if we don't re-instantiate ClonedFeedstockContext ourselves and let + # FeedstockContext.reserve_clone_directory be the only way to create a ClonedFeedstockContext + feedstock_ctx = ClonedFeedstockContext( feedstock_name=feedstock_name, attrs=node_attrs, + _default_branch=default_branch, + local_clone_dir=Path(feedstock_dir), ) - feedstock_ctx.default_branch = default_branch recipe_dir = os.path.join(feedstock_dir, "recipe") data = { diff --git a/conda_forge_tick/migrators/arch.py b/conda_forge_tick/migrators/arch.py index bfe81dd6f..274ed3ad2 100644 --- a/conda_forge_tick/migrators/arch.py +++ b/conda_forge_tick/migrators/arch.py @@ -4,7 +4,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.make_graph import ( get_deps_from_outputs_lut, make_outputs_lut_from_graph, @@ -213,7 +213,7 @@ def migrate( def pr_title(self, feedstock_ctx: FeedstockContext) -> str: return "Arch Migrator" - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( dedent( @@ -384,7 +384,7 @@ def migrate( def pr_title(self, feedstock_ctx: FeedstockContext) -> str: return "ARM OSX Migrator" - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( dedent( diff --git a/conda_forge_tick/migrators/broken_rebuild.py b/conda_forge_tick/migrators/broken_rebuild.py index 66e6b1303..fb3d13718 100644 --- a/conda_forge_tick/migrators/broken_rebuild.py +++ b/conda_forge_tick/migrators/broken_rebuild.py @@ -2,6 +2,7 @@ import networkx as nx +from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.migrators.core import Migrator BROKEN_PACKAGES = """\ @@ -380,7 +381,7 @@ def migrate(self, recipe_dir, attrs, **kwargs): self.set_build_number(os.path.join(recipe_dir, "meta.yaml")) return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( """\ diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 9f1227422..22b740426 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -10,7 +10,7 @@ import dateutil.parser import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort @@ -455,7 +455,9 @@ def migrate( """ return self.migrator_uid(attrs) - def pr_body(self, feedstock_ctx: FeedstockContext, add_label_text=True) -> str: + def pr_body( + self, feedstock_ctx: ClonedFeedstockContext, add_label_text=True + ) -> str: """Create a PR message body Returns diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index ac2c47779..ecdf06ede 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -10,7 +10,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.feedstock_parser import PIN_SEP_PAT from conda_forge_tick.make_graph import get_deps_from_outputs_lut from conda_forge_tick.migrators.core import GraphMigrator, Migrator, MiniMigrator @@ -280,7 +280,7 @@ def migrate( return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: "FeedstockContext") -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) if feedstock_ctx.feedstock_name == "conda-forge-pinning": additional_body = ( @@ -534,7 +534,7 @@ def migrate( return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: "FeedstockContext") -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = ( "This PR has been triggered in an effort to update the pin for" " **{name}**. The current pinned version is {current_pin}, " diff --git a/conda_forge_tick/migrators/replacement.py b/conda_forge_tick/migrators/replacement.py index 7df2be3f5..71ccf2dce 100644 --- a/conda_forge_tick/migrators/replacement.py +++ b/conda_forge_tick/migrators/replacement.py @@ -6,7 +6,7 @@ import networkx as nx -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator if typing.TYPE_CHECKING: @@ -127,7 +127,7 @@ def migrate( self.set_build_number(os.path.join(recipe_dir, "meta.yaml")) return super().migrate(recipe_dir, attrs) - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: body = super().pr_body(feedstock_ctx) body = body.format( "I noticed that this recipe depends on `%s` instead of \n" diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index c9d1d6b41..2d7bd5f0e 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -11,7 +11,7 @@ import networkx as nx from conda.models.version import VersionOrder -from conda_forge_tick.contexts import FeedstockContext +from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator from conda_forge_tick.models.pr_info import MigratorName from conda_forge_tick.os_utils import pushd @@ -219,7 +219,7 @@ def migrate( ) ) - def pr_body(self, feedstock_ctx: FeedstockContext) -> str: + def pr_body(self, feedstock_ctx: ClonedFeedstockContext) -> str: if feedstock_ctx.feedstock_name in self.effective_graph.nodes: pred = [ ( @@ -319,7 +319,7 @@ def pr_body(self, feedstock_ctx: FeedstockContext) -> str: return super().pr_body(feedstock_ctx, add_label_text=False).format(body) - def _hint_and_maybe_update_deps(self, feedstock_ctx): + def _hint_and_maybe_update_deps(self, feedstock_ctx: ClonedFeedstockContext): update_deps = get_keys_default( feedstock_ctx.attrs, ["conda-forge.yml", "bot", "inspection"], diff --git a/tests/test_contexts.py b/tests/test_contexts.py index 0a72dff86..34b843642 100644 --- a/tests/test_contexts.py +++ b/tests/test_contexts.py @@ -1,3 +1,5 @@ +import pytest + from conda_forge_tick.contexts import DEFAULT_BRANCHES, FeedstockContext from conda_forge_tick.migrators_types import AttrsTypedDict @@ -6,15 +8,37 @@ {"conda-forge.yml": {"provider": {"default_branch": "main"}}} ) +demo_attrs_automerge = AttrsTypedDict( + { + "conda-forge.yml": { + "provider": {"default_branch": "main"}, + "bot": {"automerge": True}, + } + } +) + +demo_attrs_check_solvable = AttrsTypedDict( + { + "conda-forge.yml": { + "provider": {"default_branch": "main"}, + "bot": {"check_solvable": True}, + } + } +) + -def test_feedstock_context_default_branch(): +def test_feedstock_context_default_branch_not_set(): context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) assert context.default_branch == "main" DEFAULT_BRANCHES["TEST-FEEDSTOCK-NAME"] = "develop" assert context.default_branch == "develop" - context.default_branch = "feature" + +def test_feedstock_context_default_branch_set(): + context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs, "feature") + + DEFAULT_BRANCHES["TEST-FEEDSTOCK-NAME"] = "develop" assert context.default_branch == "feature" # reset the default branches @@ -40,3 +64,59 @@ def test_feedstock_context_git_href(): context.git_href == "https://github.com/conda-forge/TEST-FEEDSTOCK-NAME-feedstock" ) + + +@pytest.mark.parametrize("automerge", [True, False]) +def test_feedstock_context_automerge(automerge: bool): + context = FeedstockContext( + "TEST-FEEDSTOCK-NAME", demo_attrs_automerge if automerge else demo_attrs + ) + + assert context.automerge == automerge + + +@pytest.mark.parametrize("check_solvable", [True, False]) +def test_feedstock_context_check_solvable(check_solvable: bool): + context = FeedstockContext( + "TEST-FEEDSTOCK-NAME", + demo_attrs_check_solvable if check_solvable else demo_attrs, + ) + + assert context.check_solvable == check_solvable + + +@pytest.mark.parametrize("default_branch", [None, "feature"]) +@pytest.mark.parametrize( + "attrs", [demo_attrs, demo_attrs_automerge, demo_attrs_check_solvable] +) +def test_feedstock_context_reserve_clone_directory( + attrs: AttrsTypedDict, default_branch: str +): + context = FeedstockContext("pytest", attrs, default_branch) + + with context.reserve_clone_directory() as cloned_context: + assert cloned_context.feedstock_name == "pytest" + assert cloned_context.attrs == attrs + assert ( + cloned_context.default_branch == default_branch + if default_branch + else "main" + ) + assert cloned_context.git_repo_owner == "conda-forge" + assert cloned_context.git_repo_name == "pytest-feedstock" + assert ( + cloned_context.git_href == "https://github.com/conda-forge/pytest-feedstock" + ) + assert cloned_context.automerge == context.automerge + assert cloned_context.check_solvable == context.check_solvable + + assert cloned_context.local_clone_dir.exists() + assert cloned_context.local_clone_dir.is_dir() + assert cloned_context.local_clone_dir.name == "pytest-feedstock" + + with open(cloned_context.local_clone_dir / "test.txt", "w") as f: + f.write("test") + + assert (cloned_context.local_clone_dir / "test.txt").exists() + + assert not cloned_context.local_clone_dir.exists() From b183dab725ff231f1c8a934f59e9730732ae9c60 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 11:18:51 +0200 Subject: [PATCH 24/38] dependency injection for GitPlatformBackend --- conda_forge_tick/auto_tick.py | 41 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index cf03bb8cc..4ea0dd9c7 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -444,9 +444,9 @@ def get_spoofed_closed_pr_info() -> PullRequestInfoSpecial: def run_with_tmpdir( context: FeedstockContext, migrator: Migrator, + git_backend: GitPlatformBackend, rerender: bool = True, base_branch: str = "main", - dry_run: bool = False, **kwargs: typing.Any, ) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """ @@ -462,9 +462,9 @@ def run_with_tmpdir( return run( context=cloned_context, migrator=migrator, + git_backend=git_backend, rerender=rerender, base_branch=base_branch, - dry_run=dry_run, **kwargs, ) @@ -472,9 +472,9 @@ def run_with_tmpdir( def run( context: ClonedFeedstockContext, migrator: Migrator, + git_backend: GitPlatformBackend, rerender: bool = True, base_branch: str = "main", - dry_run: bool = False, **kwargs: typing.Any, ) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration @@ -485,12 +485,12 @@ def run( The current feedstock context, already containing information about a temporary directory for the feedstock. migrator: Migrator instance The migrator to run on the feedstock + git_backend: GitPlatformBackend + The git backend to use. Use the DryRunBackend for testing. rerender : bool Whether to rerender base_branch : str, optional The base branch to which the PR will be targeted. - dry_run : bool, optional - Whether to run in dry run mode. kwargs: dict The keyword arguments to pass to the migrator. @@ -506,8 +506,6 @@ def run( GitCliError If an error occurs during a git command which is not suppressed """ - git_backend: GitPlatformBackend = DryRunBackend() if dry_run else github_backend() - # sometimes we get weird directory issues so make sure we reset os.chdir(BOT_HOME_DIR) @@ -715,7 +713,7 @@ def _run_migrator_on_feedstock_branch( base_branch, migrator, fctx: FeedstockContext, - dry_run, + git_backend: GitPlatformBackend, mctx, migrator_name, good_prs, @@ -729,9 +727,9 @@ def _run_migrator_on_feedstock_branch( migrator_uid, pr_json = run_with_tmpdir( context=fctx, migrator=migrator, + git_backend=git_backend, rerender=migrator.rerender, base_branch=base_branch, - dry_run=dry_run, hash_type=attrs.get("hash_type", "sha256"), ) finally: @@ -892,7 +890,7 @@ def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): return False -def _run_migrator(migrator, mctx, temp, time_per, dry_run): +def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBackend): _mg_start = time.time() migrator_name = get_migrator_name(migrator) @@ -1010,14 +1008,14 @@ def _run_migrator(migrator, mctx, temp, time_per, dry_run): ) ): good_prs, break_loop = _run_migrator_on_feedstock_branch( - attrs, - base_branch, - migrator, - fctx, - dry_run, - mctx, - migrator_name, - good_prs, + attrs=attrs, + base_branch=base_branch, + migrator=migrator, + fctx=fctx, + git_backend=git_backend, + mctx=mctx, + migrator_name=migrator_name, + good_prs=good_prs, ) if break_loop: break @@ -1033,8 +1031,7 @@ def _run_migrator(migrator, mctx, temp, time_per, dry_run): os.chdir(BOT_HOME_DIR) # Write graph partially through - if not dry_run: - dump_graph(mctx.graph) + dump_graph(mctx.graph) with filter_reprinted_lines("rm-tmp"): for f in glob.glob("/tmp/*"): @@ -1267,13 +1264,15 @@ def main(ctx: CliContext) -> None: flush=True, ) + git_backend = github_backend() if not ctx.dry_run else DryRunBackend() + for mg_ind, migrator in enumerate(migrators): good_prs = _run_migrator( migrator, mctx, temp, time_per_migrator[mg_ind], - ctx.dry_run, + git_backend, ) if good_prs > 0: pass From fc338227606d93c1b23702259cb98692b38c668f Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 25 Jun 2024 16:12:20 +0200 Subject: [PATCH 25/38] small fixes --- conda_forge_tick/auto_tick.py | 4 ++-- conda_forge_tick/git_utils.py | 5 +++-- conda_forge_tick/models/common.py | 24 +++++++++--------------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 4ea0dd9c7..cea5152a6 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -27,12 +27,12 @@ from conda.models.version import VersionOrder from conda_forge_tick.cli_context import CliContext -from conda_forge_tick.deploy import deploy from conda_forge_tick.contexts import ( ClonedFeedstockContext, FeedstockContext, MigratorSessionContext, ) +from conda_forge_tick.deploy import deploy from conda_forge_tick.feedstock_parser import BOOTSTRAP_MAPPINGS from conda_forge_tick.git_utils import ( DryRunBackend, @@ -620,7 +620,7 @@ def run( os.path.join("pr_json", f"{pr_data.id}.json"), ) with pr_lazy_json as __edit_pr_lazy_json: - __edit_pr_lazy_json.update(**pr_data.model_dump()) + __edit_pr_lazy_json.update(**pr_data.model_dump(mode="json")) else: pr_lazy_json = False diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 9bc8f7712..d5e7a349b 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1037,12 +1037,13 @@ def create_pull_request( Target Repository: {target_owner}/{target_repo} Branches: {self.user}:{head_branch} -> {target_owner}:{base_branch} Body: - {body} - ============================================================== """ ) ) + logger.debug(body) + logger.debug("==============================================================") + now = datetime.now() return PullRequestData.model_validate( { diff --git a/conda_forge_tick/models/common.py b/conda_forge_tick/models/common.py index b07e33f8f..77b548fae 100644 --- a/conda_forge_tick/models/common.py +++ b/conda_forge_tick/models/common.py @@ -10,6 +10,7 @@ BeforeValidator, ConfigDict, Field, + PlainSerializer, UrlConstraints, ) from pydantic_core import Url @@ -77,7 +78,7 @@ def none_to_empty_dict(value: T | None) -> T | dict[Never]: return value -NoneIsEmptyDict = Annotated[dict[T], BeforeValidator(none_to_empty_dict)] +NoneIsEmptyDict = Annotated[dict[K, V], BeforeValidator(none_to_empty_dict)] """ A generic dict type that converts `None` to an empty dict. This should not be needed if this proper data model is used in production. @@ -151,22 +152,15 @@ def parse_rfc_2822_date(value: str) -> datetime: return email.utils.parsedate_to_datetime(value) -RFC2822Date = Annotated[datetime, BeforeValidator(parse_rfc_2822_date)] - - -def none_to_empty_dict(value: T | None) -> T | dict[Never, Never]: - """ - Convert `None` to an empty dictionary f, otherwise keep the value as is. - """ - if value is None: - return {} - return value +def serialize_rfc_2822_date(value: datetime) -> str: + return email.utils.format_datetime(value) -NoneIsEmptyDict = Annotated[dict[K, V], BeforeValidator(none_to_empty_dict)] -""" -A generic dict type that converts `None` to an empty dict. -""" +RFC2822Date = Annotated[ + datetime, + BeforeValidator(parse_rfc_2822_date), + PlainSerializer(serialize_rfc_2822_date), +] GitUrl = Annotated[Url, UrlConstraints(allowed_schemes=["git"])] From de76a1eb4b243fc6a57d28f7098e17762c4569d6 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Wed, 29 May 2024 18:08:12 +0200 Subject: [PATCH 26/38] add feedstock attributes to FeedstockContext --- conda_forge_tick/auto_tick.py | 6 +++--- conda_forge_tick/contexts.py | 15 +++++++++++++++ tests/test_contexts.py | 1 - 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index cea5152a6..3a719195d 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -572,9 +572,9 @@ def run( None means: We don't update the PR data. """ if ( - isinstance(migrator, MigrationYaml) - and not rerender_info.nontrivial_migration_yaml_changes - and context.attrs["name"] != "conda-forge-pinning" + isinstance(migrator, MigrationYaml) + and not rerender_info.nontrivial_migration_yaml_changes + and context.attrs["name"] != "conda-forge-pinning" ): # spoof this so it looks like the package is done pr_data = get_spoofed_closed_pr_info() diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index 873d3f18e..cb491856f 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -117,3 +117,18 @@ class ClonedFeedstockContext(FeedstockContext): # Implementation Note: Keep this class frozen or there will be consistency issues if someone modifies # a ClonedFeedstockContext object in place - it will not be reflected in the original FeedstockContext object. local_clone_dir: Path + + @property + def git_repo_owner(self) -> str: + return "conda-forge" + + @property + def git_repo_name(self) -> str: + return f"{self.feedstock_name}-feedstock" + + @property + def git_href(self) -> str: + """ + A link to the feedstocks GitHub repository. + """ + return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}" diff --git a/tests/test_contexts.py b/tests/test_contexts.py index 34b843642..dbab1417e 100644 --- a/tests/test_contexts.py +++ b/tests/test_contexts.py @@ -26,7 +26,6 @@ } ) - def test_feedstock_context_default_branch_not_set(): context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) assert context.default_branch == "main" From 359379b7796c53bf23e808cdde9a61eaca6038bb Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 30 May 2024 16:08:36 +0200 Subject: [PATCH 27/38] add create_pull_request to git backend --- conda_forge_tick/git_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index d5e7a349b..0d14fd8b3 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1062,7 +1062,7 @@ def create_pull_request( "base": GithubPullRequestBase(repo=GithubRepository(name=target_repo)), } ) - + def comment_on_pull_request( self, repo_owner: str, repo_name: str, pr_number: int, comment: str ): From a6755e023814097391d070113c1e083454cc0393 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sat, 15 Jun 2024 16:13:43 +0200 Subject: [PATCH 28/38] hide tokens in Git Backends automatically, push to repository --- conda_forge_tick/git_utils.py | 2 +- tests/test_git_utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index 0d14fd8b3..d5e7a349b 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -1062,7 +1062,7 @@ def create_pull_request( "base": GithubPullRequestBase(repo=GithubRepository(name=target_repo)), } ) - + def comment_on_pull_request( self, repo_owner: str, repo_name: str, pr_number: int, comment: str ): diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 073162389..2cd8dedf8 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -1080,6 +1080,16 @@ def github_response_headers() -> dict: return _github_api_json_fixture("github_response_headers") +def test_git_platform_backend_get_remote_url_token(): + owner = "OWNER" + repo = "REPO" + token = "TOKEN" + + url = GitPlatformBackend.get_remote_url(owner, repo, GitConnectionMode.HTTPS, token) + + assert url == f"https://{token}@github.com/{owner}/{repo}.git" + + def test_github_backend_from_token(): token = "TOKEN" From 320f62d9288688a069dcd133618681cf5cdfadc6 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sun, 16 Jun 2024 16:38:43 +0200 Subject: [PATCH 29/38] use temporary directory instead of managing the local feedstock dir manually --- conda_forge_tick/auto_tick.py | 28 ++++++++++++++++++++++++++++ tests/test_contexts.py | 1 + 2 files changed, 29 insertions(+) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 3a719195d..d30e59f55 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -476,6 +476,34 @@ def run( rerender: bool = True, base_branch: str = "main", **kwargs: typing.Any, +) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: + """ + For a given feedstock and migration run the migration in a temporary directory that will be deleted after the + migration is complete. + + The parameters are the same as for the `run` function. The only difference is that you pass a FeedstockContext + instance instead of a ClonedFeedstockContext instance. + + The exceptions are the same as for the `run` function. + """ + with context.reserve_clone_directory() as cloned_context: + return run( + context=cloned_context, + migrator=migrator, + rerender=rerender, + base_branch=base_branch, + dry_run=dry_run, + **kwargs, + ) + + +def run( + context: ClonedFeedstockContext, + migrator: Migrator, + rerender: bool = True, + base_branch: str = "main", + dry_run: bool = False, + **kwargs: typing.Any, ) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration diff --git a/tests/test_contexts.py b/tests/test_contexts.py index dbab1417e..34b843642 100644 --- a/tests/test_contexts.py +++ b/tests/test_contexts.py @@ -26,6 +26,7 @@ } ) + def test_feedstock_context_default_branch_not_set(): context = FeedstockContext("TEST-FEEDSTOCK-NAME", demo_attrs) assert context.default_branch == "main" From ada4150239f664a0816e8e0632b6c899a8f1f84e Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 11:18:51 +0200 Subject: [PATCH 30/38] dependency injection for GitPlatformBackend --- conda_forge_tick/auto_tick.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index d30e59f55..5f0a087c8 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -490,9 +490,9 @@ def run( return run( context=cloned_context, migrator=migrator, + git_backend=git_backend, rerender=rerender, base_branch=base_branch, - dry_run=dry_run, **kwargs, ) @@ -500,9 +500,9 @@ def run( def run( context: ClonedFeedstockContext, migrator: Migrator, + git_backend: GitPlatformBackend, rerender: bool = True, base_branch: str = "main", - dry_run: bool = False, **kwargs: typing.Any, ) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: """For a given feedstock and migration run the migration From 56f64ed1199bd091d25ad98a23abdf8374805d68 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 14:11:58 +0200 Subject: [PATCH 31/38] add single package support for auto_tick, clean up dry run option --- conda_forge_tick/auto_tick.py | 110 ++++++++++++++++++++----- conda_forge_tick/cli.py | 13 ++- conda_forge_tick/contexts.py | 1 - conda_forge_tick/lazy_json_backends.py | 11 +++ conda_forge_tick/status_report.py | 1 - 5 files changed, 112 insertions(+), 24 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 5f0a087c8..4907cc54a 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -9,7 +9,7 @@ import traceback import typing from dataclasses import dataclass -from typing import Literal, cast +from typing import AnyStr, Literal, cast from .models.pr_info import PullRequestInfoSpecial from .models.pr_json import PullRequestData, PullRequestState @@ -46,6 +46,7 @@ ) from conda_forge_tick.lazy_json_backends import ( LazyJson, + does_key_exist_in_hashmap, get_all_keys_for_hashmap, lazy_json_transaction, remove_key_for_hashmap, @@ -662,7 +663,7 @@ def run( return migration_run_data["migrate_return_value"], pr_lazy_json -def _compute_time_per_migrator(mctx, migrators): +def _compute_time_per_migrator(migrators): # we weight each migrator by the number of available nodes to migrate num_nodes = [] for migrator in tqdm.tqdm(migrators, ncols=80, desc="computing time per migrator"): @@ -918,7 +919,26 @@ def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): return False -def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBackend): +def _run_migrator( + migrator: Migrator, + mctx: MigratorSessionContext, + temp: list[AnyStr], + time_per: float, + git_backend: GitPlatformBackend, + package: str | None = None, +) -> int: + """ + Run a migrator. + + :param migrator: The migrator to run. + :param mctx: The migrator session context. + :param temp: The list of temporary files. + :param time_per: The time limit of this migrator. + :param git_backend: The GitPlatformBackend instance to use. + :param package: The package to update, if None, all packages are updated. + + :return: The number of "good" PRs created by the migrator. + """ _mg_start = time.time() migrator_name = get_migrator_name(migrator) @@ -940,6 +960,14 @@ def _run_migrator(migrator, mctx, temp, time_per, git_backend: GitPlatformBacken possible_nodes = list(migrator.order(effective_graph, mctx.graph)) + if package: + if package not in possible_nodes: + logger.warning( + f"Package {package} is not a candidate for migration of {migrator_name}" + ) + return 0 + possible_nodes = [package] + # version debugging info if isinstance(migrator, Version): print("possible version migrations:", flush=True) @@ -1084,18 +1112,26 @@ def _setup_limits(): resource.setrlimit(resource.RLIMIT_AS, (limit_int, limit_int)) -def _update_nodes_with_bot_rerun(gx: nx.DiGraph): - """Go through all the open PRs and check if they are rerun""" +def _update_nodes_with_bot_rerun(gx: nx.DiGraph, package: str | None = None): + """ + Go through all the open PRs and check if they are rerun + + :param gx: the dependency graph + :param package: the package to update, if None, all packages are updated + """ print("processing bot-rerun labels", flush=True) - for i, (name, node) in enumerate(gx.nodes.items()): + nodes = gx.nodes.items() if not package else [(package, gx.nodes[package])] + + for i, (name, node) in nodes: # logger.info( # f"node: {i} memory usage: " # f"{psutil.Process().memory_info().rss // 1024 ** 2}MB", # ) with node["payload"] as payload: if payload.get("archived", False): + logger.debug(f"skipping archived package {name}") continue with payload["pr_info"] as pri, payload["version_pr_info"] as vpri: # reset bad @@ -1145,12 +1181,21 @@ def _filter_ignored_versions(attrs, version): return version -def _update_nodes_with_new_versions(gx): - """Updates every node with it's new version (when available)""" +def _update_nodes_with_new_versions(gx: nx.DiGraph, package: str | None = None): + """ + Updates every node with its new version (when available) + + :param gx: the dependency graph + :param package: the package to update, if None, all packages are updated + """ print("updating nodes with new versions", flush=True) - version_nodes = get_all_keys_for_hashmap("versions") + if package and not does_key_exist_in_hashmap("versions", package): + logger.warning(f"Package {package} not found in versions hashmap") + return + + version_nodes = get_all_keys_for_hashmap("versions") if not package else [package] for node in version_nodes: version_data = LazyJson(f"versions/{node}.json").data @@ -1176,13 +1221,35 @@ def _update_nodes_with_new_versions(gx): vpri["new_version"] = version_from_data -def _remove_closed_pr_json(): +def _remove_closed_pr_json(package: str | None = None): + """ + Remove the pull request information for closed PRs. + + :param package: The package to remove the PR information for. If None, all PR information is removed. If you pass + a package, closed pr_json files are not removed because this would require iterating all pr_json files. + """ print("collapsing closed PR json", flush=True) + if package: + pr_info_nodes = ( + [package] if does_key_exist_in_hashmap("pr_info", package) else [] + ) + version_pr_info_nodes = ( + [package] if does_key_exist_in_hashmap("version_pr_info", package) else [] + ) + + if not pr_info_nodes: + logger.warning(f"Package {package} not found in pr_info hashmap") + if not version_pr_info_nodes: + logger.warning(f"Package {package} not found in version_pr_info hashmap") + else: + pr_info_nodes = get_all_keys_for_hashmap("pr_info") + version_pr_info_nodes = get_all_keys_for_hashmap("version_pr_info") + # first we go from nodes to pr json and update the pr info and remove the data name_nodes = [ - ("pr_info", get_all_keys_for_hashmap("pr_info")), - ("version_pr_info", get_all_keys_for_hashmap("version_pr_info")), + ("pr_info", pr_info_nodes), + ("version_pr_info", version_pr_info_nodes), ] for name, nodes in name_nodes: for node in nodes: @@ -1215,6 +1282,11 @@ def _remove_closed_pr_json(): # at this point, any json blob referenced in the pr info is state != closed # so we can remove anything that is empty or closed + if package: + logger.info( + "Since you requested a run for a specific package, we are not removing closed pr_json files." + ) + return nodes = get_all_keys_for_hashmap("pr_json") for node in nodes: pr = LazyJson(f"pr_json/{node}.json") @@ -1225,22 +1297,22 @@ def _remove_closed_pr_json(): ) -def _update_graph_with_pr_info(): - _remove_closed_pr_json() +def _update_graph_with_pr_info(package: str | None = None): + _remove_closed_pr_json(package) gx = load_existing_graph() - _update_nodes_with_bot_rerun(gx) - _update_nodes_with_new_versions(gx) + _update_nodes_with_bot_rerun(gx, package) + _update_nodes_with_new_versions(gx, package) dump_graph(gx) -def main(ctx: CliContext) -> None: +def main(ctx: CliContext, package: str | None = None) -> None: global START_TIME START_TIME = time.time() _setup_limits() with fold_log_lines("updating graph with PR info"): - _update_graph_with_pr_info() + _update_graph_with_pr_info(package) deploy(ctx, dirs_to_deploy=["version_pr_info", "pr_json", "pr_info"]) # record tmp dir so we can be sure to clean it later @@ -1259,7 +1331,6 @@ def main(ctx: CliContext) -> None: graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, - dry_run=ctx.dry_run, ) migrators = load_migrators() @@ -1271,7 +1342,6 @@ def main(ctx: CliContext) -> None: time_per_migrator, tot_time_per_migrator, ) = _compute_time_per_migrator( - mctx, migrators, ) for i, migrator in enumerate(migrators): diff --git a/conda_forge_tick/cli.py b/conda_forge_tick/cli.py index 6afe95efa..3d67ed7b2 100644 --- a/conda_forge_tick/cli.py +++ b/conda_forge_tick/cli.py @@ -149,11 +149,20 @@ def update_upstream_versions( @main.command(name="auto-tick") +@click.argument( + "package", + required=False, +) @pass_context -def auto_tick(ctx: CliContext) -> None: +def auto_tick(ctx: CliContext, package: str | None) -> None: + """ + Run the main bot logic that runs all migrations, updates the graph accordingly, and opens the corresponding PRs. + + If PACKAGE is given, only run the bot for that package, otherwise run the bot for all packages. + """ from . import auto_tick - auto_tick.main(ctx) + auto_tick.main(ctx, package=package) @main.command(name="make-status-report") diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index cb491856f..222af7d42 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -31,7 +31,6 @@ class MigratorSessionContext: graph: DiGraph = None smithy_version: str = "" pinning_version: str = "" - dry_run: bool = True @dataclass(frozen=True) diff --git a/conda_forge_tick/lazy_json_backends.py b/conda_forge_tick/lazy_json_backends.py index 4619a83ff..398409317 100644 --- a/conda_forge_tick/lazy_json_backends.py +++ b/conda_forge_tick/lazy_json_backends.py @@ -630,6 +630,17 @@ def get_all_keys_for_hashmap(name): return backend.hkeys(name) +def does_key_exist_in_hashmap(name: str, key: str) -> bool: + """ + Check if a key exists in a hashmap, using the primary backend. + :param name: The hashmap name. + :param key: The key to check. + :return: True if the key exists, False otherwise. + """ + backend = LAZY_JSON_BACKENDS[CF_TICK_GRAPH_DATA_PRIMARY_BACKEND]() + return backend.hexists(name, name) + + @contextlib.contextmanager def lazy_json_transaction(): try: diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 54c2dae46..bd6d13a41 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -398,7 +398,6 @@ def main() -> None: graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, - dry_run=False, ) migrators = load_migrators() From 32737f7517ac24e1850169a137b054bc5d89e0ca Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 14:29:23 +0200 Subject: [PATCH 32/38] fixes and TODOs --- conda_forge_tick/auto_tick.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 4907cc54a..637c11483 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -10,13 +10,6 @@ import typing from dataclasses import dataclass from typing import AnyStr, Literal, cast - -from .models.pr_info import PullRequestInfoSpecial -from .models.pr_json import PullRequestData, PullRequestState - -if typing.TYPE_CHECKING: - from .migrators_types import MigrationUidTypedDict - from urllib.error import URLError from uuid import uuid4 @@ -76,6 +69,10 @@ sanitize_string, ) +from .migrators_types import MigrationUidTypedDict +from .models.pr_info import PullRequestInfoSpecial +from .models.pr_json import PullRequestData, PullRequestState + logger = logging.getLogger(__name__) BOT_HOME_DIR: str = os.getcwd() @@ -793,6 +790,7 @@ def _run_migrator_on_feedstock_branch( except (github3.GitHubError, github.GithubException) as e: # TODO: pull this down into run() - also check the other exceptions + # TODO: continue here, after that run locally and add tests, backend should be injected into run if hasattr(e, "msg") and e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: @@ -1124,7 +1122,7 @@ def _update_nodes_with_bot_rerun(gx: nx.DiGraph, package: str | None = None): nodes = gx.nodes.items() if not package else [(package, gx.nodes[package])] - for i, (name, node) in nodes: + for i, (name, node) in enumerate(nodes): # logger.info( # f"node: {i} memory usage: " # f"{psutil.Process().memory_info().rss // 1024 ** 2}MB", @@ -1332,6 +1330,7 @@ def main(ctx: CliContext, package: str | None = None) -> None: smithy_version=smithy_version, pinning_version=pinning_version, ) + # TODO: this does not support --online migrators = load_migrators() # compute the time per migrator From e734fdce913ed06c397b280aab60b898c1d5d4e8 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 15:40:36 +0200 Subject: [PATCH 33/38] fix rebasing, fix running locally --- conda_forge_tick/auto_tick.py | 55 ++++++++++------------------------- conda_forge_tick/git_utils.py | 3 +- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 637c11483..5cc09999e 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -29,13 +29,12 @@ from conda_forge_tick.feedstock_parser import BOOTSTRAP_MAPPINGS from conda_forge_tick.git_utils import ( DryRunBackend, - DuplicatePullRequestError, GitCli, GitCliError, GitPlatformBackend, RepositoryNotFoundError, github_backend, - is_github_api_limit_reached, + is_github_api_limit_reached, DuplicatePullRequestError, ) from conda_forge_tick.lazy_json_backends import ( LazyJson, @@ -49,8 +48,7 @@ PR_LIMIT, load_migrators, ) -from conda_forge_tick.migration_runner import run_migration -from conda_forge_tick.migrators import MigrationYaml, Migrator, Version +from conda_forge_tick.migrators import Migrator, Version, MigrationYaml from conda_forge_tick.migrators.version import VersionMigrationError from conda_forge_tick.os_utils import eval_cmd from conda_forge_tick.rerender_feedstock import rerender_feedstock @@ -68,6 +66,7 @@ load_existing_graph, sanitize_string, ) +from .migration_runner import run_migration from .migrators_types import MigrationUidTypedDict from .models.pr_info import PullRequestInfoSpecial @@ -467,34 +466,6 @@ def run_with_tmpdir( ) -def run( - context: ClonedFeedstockContext, - migrator: Migrator, - git_backend: GitPlatformBackend, - rerender: bool = True, - base_branch: str = "main", - **kwargs: typing.Any, -) -> tuple[MigrationUidTypedDict, dict] | tuple[Literal[False], Literal[False]]: - """ - For a given feedstock and migration run the migration in a temporary directory that will be deleted after the - migration is complete. - - The parameters are the same as for the `run` function. The only difference is that you pass a FeedstockContext - instance instead of a ClonedFeedstockContext instance. - - The exceptions are the same as for the `run` function. - """ - with context.reserve_clone_directory() as cloned_context: - return run( - context=cloned_context, - migrator=migrator, - git_backend=git_backend, - rerender=rerender, - base_branch=base_branch, - **kwargs, - ) - - def run( context: ClonedFeedstockContext, migrator: Migrator, @@ -879,10 +850,11 @@ def _run_migrator_on_feedstock_branch( return good_prs, break_loop -def _is_migrator_done(_mg_start, good_prs, time_per, pr_limit): +def _is_migrator_done( + _mg_start, good_prs, time_per, pr_limit, git_backend: GitPlatformBackend +): curr_time = time.time() - backend = github_backend() - api_req = backend.get_api_requests_left() + api_req = git_backend.get_api_requests_left() if curr_time - START_TIME > TIMEOUT: logger.info( @@ -960,7 +932,7 @@ def _run_migrator( if package: if package not in possible_nodes: - logger.warning( + logger.info( f"Package {package} is not a candidate for migration of {migrator_name}" ) return 0 @@ -998,7 +970,9 @@ def _run_migrator( flush=True, ) - if _is_migrator_done(_mg_start, good_prs, time_per, migrator.pr_limit): + if _is_migrator_done( + _mg_start, good_prs, time_per, migrator.pr_limit, git_backend + ): return 0 for node_name in possible_nodes: @@ -1015,7 +989,9 @@ def _run_migrator( ): # Don't let CI timeout, break ahead of the timeout so we make certain # to write to the repo - if _is_migrator_done(_mg_start, good_prs, time_per, migrator.pr_limit): + if _is_migrator_done( + _mg_start, good_prs, time_per, migrator.pr_limit, git_backend + ): break base_branches = migrator.get_possible_feedstock_branches(attrs) @@ -1370,6 +1346,7 @@ def main(ctx: CliContext, package: str | None = None) -> None: temp, time_per_migrator[mg_ind], git_backend, + package, ) if good_prs > 0: pass @@ -1384,5 +1361,5 @@ def main(ctx: CliContext, package: str | None = None) -> None: # ], # ) - logger.info("API Calls Remaining: %d", github_backend().get_api_requests_left()) + logger.info(f"API Calls Remaining: {git_backend.get_api_requests_left()}") logger.info("Done") diff --git a/conda_forge_tick/git_utils.py b/conda_forge_tick/git_utils.py index d5e7a349b..ad51b2bab 100644 --- a/conda_forge_tick/git_utils.py +++ b/conda_forge_tick/git_utils.py @@ -39,6 +39,7 @@ GithubPullRequestMergeableState, GithubRepository, PullRequestData, + PullRequestDataValid, PullRequestInfoHead, PullRequestState, ) @@ -1045,7 +1046,7 @@ def create_pull_request( logger.debug("==============================================================") now = datetime.now() - return PullRequestData.model_validate( + return PullRequestDataValid.model_validate( { "ETag": "GITHUB_PR_ETAG", "Last-Modified": utils.format_datetime(now), From 2ed9925a4fc7ae565a7c315c8cb38a7594b400e3 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 16:39:20 +0200 Subject: [PATCH 34/38] remove unused parameter, add helpful debug info --- conda_forge_tick/auto_tick.py | 3 ++- conda_forge_tick/make_migrators.py | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 5cc09999e..10890769b 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -933,7 +933,8 @@ def _run_migrator( if package: if package not in possible_nodes: logger.info( - f"Package {package} is not a candidate for migration of {migrator_name}" + f"Package {package} is not a candidate for migration of {migrator_name}. " + f"If you want to investigate this, run the make-migrators command." ) return 0 possible_nodes = [package] diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index 519a88df2..251b984c1 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -692,7 +692,6 @@ def create_migration_yaml_creator( def initialize_migrators( gx: nx.DiGraph, - dry_run: bool = False, ) -> MutableSequence[Migrator]: migrators: List[Migrator] = [] @@ -811,10 +810,7 @@ def load_migrators() -> MutableSequence[Migrator]: def main(ctx: CliContext) -> None: gx = load_existing_graph() - migrators = initialize_migrators( - gx, - dry_run=ctx.dry_run, - ) + migrators = initialize_migrators(gx) with ( fold_log_lines("dumping migrators to JSON"), lazy_json_override_backends( From 062d9f2f29580bd8728370d8b713fbe7757b4ba0 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 16:59:18 +0200 Subject: [PATCH 35/38] add --version-only option to make-migrators --- conda_forge_tick/cli.py | 8 ++- conda_forge_tick/make_migrators.py | 78 +++++++++++++++++------------- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/conda_forge_tick/cli.py b/conda_forge_tick/cli.py index 3d67ed7b2..2eb9f7d5a 100644 --- a/conda_forge_tick/cli.py +++ b/conda_forge_tick/cli.py @@ -245,16 +245,22 @@ def make_import_to_package_mapping( @main.command(name="make-migrators") +@click.option( + "--version-only/--all", + default=False, + help="If given, only initialize the Version migrator.", +) @pass_context def make_migrators( ctx: CliContext, + version_only: bool, ) -> None: """ Make the migrators. """ from . import make_migrators as _make_migrators - _make_migrators.main(ctx) + _make_migrators.main(ctx, version_only=version_only) if __name__ == "__main__": diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index 251b984c1..e3d72361c 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -690,36 +690,9 @@ def create_migration_yaml_creator( continue -def initialize_migrators( +def initialize_version_migrator( gx: nx.DiGraph, -) -> MutableSequence[Migrator]: - migrators: List[Migrator] = [] - - add_arch_migrate(migrators, gx) - - add_replacement_migrator( - migrators, - gx, - cast("PackageName", "build"), - cast("PackageName", "python-build"), - "The conda package name 'build' is deprecated " - "and too generic. Use 'python-build instead.'", - ) - - pinning_migrators: List[Migrator] = [] - migration_factory(pinning_migrators, gx) - create_migration_yaml_creator(migrators=pinning_migrators, gx=gx) - - with fold_log_lines("migration graph sizes"): - print("rebuild migration graph sizes:", flush=True) - for m in migrators + pinning_migrators: - if isinstance(m, GraphMigrator): - print( - f' {getattr(m, "name", m)} graph size: ' - f'{len(getattr(m, "graph", []))}', - flush=True, - ) - +) -> Version: with fold_log_lines("making version migrator"): print("building package import maps and version migrator", flush=True) python_nodes = { @@ -754,8 +727,43 @@ def initialize_migrators( ], ) - random.shuffle(pinning_migrators) - migrators = [version_migrator] + migrators + pinning_migrators + return version_migrator + + +def initialize_migrators( + gx: nx.DiGraph, +) -> MutableSequence[Migrator]: + migrators: List[Migrator] = [] + + add_arch_migrate(migrators, gx) + + add_replacement_migrator( + migrators, + gx, + cast("PackageName", "build"), + cast("PackageName", "python-build"), + "The conda package name 'build' is deprecated " + "and too generic. Use 'python-build instead.'", + ) + + pinning_migrators: List[Migrator] = [] + migration_factory(pinning_migrators, gx) + create_migration_yaml_creator(migrators=pinning_migrators, gx=gx) + + with fold_log_lines("migration graph sizes"): + print("rebuild migration graph sizes:", flush=True) + for m in migrators + pinning_migrators: + if isinstance(m, GraphMigrator): + print( + f' {getattr(m, "name", m)} graph size: ' + f'{len(getattr(m, "graph", []))}', + flush=True, + ) + + version_migrator = initialize_version_migrator(gx) + + random.shuffle(pinning_migrators) + migrators = [version_migrator] + migrators + pinning_migrators return migrators @@ -808,9 +816,13 @@ def load_migrators() -> MutableSequence[Migrator]: return migrators -def main(ctx: CliContext) -> None: +def main(ctx: CliContext, version_only: bool = False) -> None: gx = load_existing_graph() - migrators = initialize_migrators(gx) + migrators = ( + initialize_migrators(gx) + if not version_only + else [initialize_version_migrator(gx)] + ) with ( fold_log_lines("dumping migrators to JSON"), lazy_json_override_backends( From a5dcfea6d942453eb6848e1f45d9fa9de4e2539e Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Mon, 24 Jun 2024 17:36:07 +0200 Subject: [PATCH 36/38] FIX does_key_exist_in_hashmap --- conda_forge_tick/lazy_json_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/lazy_json_backends.py b/conda_forge_tick/lazy_json_backends.py index 398409317..8b19d0ac1 100644 --- a/conda_forge_tick/lazy_json_backends.py +++ b/conda_forge_tick/lazy_json_backends.py @@ -638,7 +638,7 @@ def does_key_exist_in_hashmap(name: str, key: str) -> bool: :return: True if the key exists, False otherwise. """ backend = LAZY_JSON_BACKENDS[CF_TICK_GRAPH_DATA_PRIMARY_BACKEND]() - return backend.hexists(name, name) + return backend.hexists(name, key) @contextlib.contextmanager From c752ddddf13aab05bc8488d2e79d653f36356b82 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 25 Jun 2024 14:30:39 +0200 Subject: [PATCH 37/38] fix rebasing issue --- conda_forge_tick/auto_tick.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index 10890769b..7917ebc89 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -29,12 +29,13 @@ from conda_forge_tick.feedstock_parser import BOOTSTRAP_MAPPINGS from conda_forge_tick.git_utils import ( DryRunBackend, + DuplicatePullRequestError, GitCli, GitCliError, GitPlatformBackend, RepositoryNotFoundError, github_backend, - is_github_api_limit_reached, DuplicatePullRequestError, + is_github_api_limit_reached, ) from conda_forge_tick.lazy_json_backends import ( LazyJson, @@ -48,7 +49,7 @@ PR_LIMIT, load_migrators, ) -from conda_forge_tick.migrators import Migrator, Version, MigrationYaml +from conda_forge_tick.migrators import MigrationYaml, Migrator, Version from conda_forge_tick.migrators.version import VersionMigrationError from conda_forge_tick.os_utils import eval_cmd from conda_forge_tick.rerender_feedstock import rerender_feedstock @@ -569,9 +570,9 @@ def run( None means: We don't update the PR data. """ if ( - isinstance(migrator, MigrationYaml) - and not rerender_info.nontrivial_migration_yaml_changes - and context.attrs["name"] != "conda-forge-pinning" + isinstance(migrator, MigrationYaml) + and not rerender_info.nontrivial_migration_yaml_changes + and context.attrs["name"] != "conda-forge-pinning" ): # spoof this so it looks like the package is done pr_data = get_spoofed_closed_pr_info() From 2a55da663fea1a410de8866b444d243cae463510 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Tue, 25 Jun 2024 14:31:00 +0200 Subject: [PATCH 38/38] no-op in ClonedFeedstockContext.reserve_clone_directory --- conda_forge_tick/contexts.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index 222af7d42..baea644bf 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -117,6 +117,13 @@ class ClonedFeedstockContext(FeedstockContext): # a ClonedFeedstockContext object in place - it will not be reflected in the original FeedstockContext object. local_clone_dir: Path + @contextmanager + def reserve_clone_directory(self) -> Iterator[ClonedFeedstockContext]: + """ + This method is a no-op for ClonedFeedstockContext objects because the directory has already been reserved. + """ + yield self + @property def git_repo_owner(self) -> str: return "conda-forge"