diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 6ad6b7b157..fdd2fbb4c0 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -105,10 +105,10 @@ runs: - name: nf-core modules list remote shell: bash - run: nf-core --log-file log.txt modules list remote + run: nf-core --log-file log.txt modules list remote --dir nf-core-testpipeline/ working-directory: create-lint-wf - name: nf-core modules list remote gitlab shell: bash - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote --dir nf-core-testpipeline/ working-directory: create-lint-wf diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2d67b1ce98..ae9b42f70c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,13 +72,19 @@ jobs: tests: ${{ steps.list_tests.outputs.tests }} test: - name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} + name: Run ${{matrix.tests.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: [setup, list_tests] if: ${{ needs.setup.outputs.run-tests }} # run on self-hosted runners for test_components.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default - runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} + runs-on: ${{ matrix.tests.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} strategy: - matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} + matrix: + tests: ${{ fromJson(needs.list_tests.outputs.tests) }} + xdist: [""] + include: + - tests: "test_components.py" + xdist: "-n auto --maxfail 1" + fail-fast: false # run all tests even if one fails steps: - name: go to subdirectory and change nextflow workdir @@ -111,7 +117,7 @@ jobs: sudo apt install -y git - name: Set up Singularity - if: ${{ matrix.test == 'test_download.py'}} + if: ${{ matrix.tests.test == 'test_download.py'}} uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: singularity-version: 3.8.3 @@ -132,7 +138,7 @@ jobs: - name: Test with pytest run: | - python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$? + python3 -m pytest tests/${{matrix.tests.test}} --color=yes --cov --durations=0 ${{matrix.xdist}} && exit_code=0|| exit_code=$? # don't fail if no tests were collected, e.g. for test_licence.py if [ "${exit_code}" -eq 5 ]; then echo "No tests were collected" @@ -144,7 +150,7 @@ jobs: - name: remove slashes from test name run: | - test=$(echo ${{ matrix.test }} | sed 's/\//__/g') + test=$(echo ${{ matrix.tests.test }} | sed 's/\//__/g') echo "test=${test}" >> $GITHUB_ENV - name: Store snapshot report diff --git a/CHANGELOG.md b/CHANGELOG.md index 36ea940844..0525877a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) +- Fix usage of cache path and implement parallized testing ([#3291](https://github.com/nf-core/tools/pull/3291)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.3 ([#3275](https://github.com/nf-core/tools/pull/3275)) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 81d088e13d..7a3161ad62 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -115,6 +115,10 @@ "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], } +click.rich_click.OPTION_GROUPS = { + "nf-core modules list remote": [{"options": ["--dir", "--json", "--help"]}], +} + # Set up rich stderr console stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) stdout = rich.console.Console(force_terminal=rich_force_colors()) @@ -874,11 +878,19 @@ def modules_list(ctx): @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def command_modules_list_remote(ctx, keywords, json): +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", +) +def command_modules_list_remote(ctx, keywords, json, directory): # pylint: disable=redefined-builtin """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ - modules_list_remote(ctx, keywords, json) + modules_list_remote(ctx, keywords, json, directory) # nf-core modules list local @@ -1432,11 +1444,19 @@ def subworkflows_list(ctx): @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def command_subworkflows_list_remote(ctx, keywords, json): +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", +) +def command_subworkflows_list_remote(ctx, keywords, json, directory): # pylint: disable=redefined-builtin """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ - subworkflows_list_remote(ctx, keywords, json) + subworkflows_list_remote(ctx, keywords, json, directory) # nf-core subworkflows list local diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 33b1f75160..7deb565e8d 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -9,7 +9,7 @@ stdout = rich.console.Console(force_terminal=rich_force_colors()) -def modules_list_remote(ctx, keywords, json): +def modules_list_remote(ctx, keywords, json, directory): # pylint: disable=redefined-builtin """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -17,11 +17,11 @@ def modules_list_remote(ctx, keywords, json): try: module_list = ModuleList( - ".", - True, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], + directory=directory, + remote=True, + remote_url=ctx.obj["modules_repo_url"], + branch=ctx.obj["modules_repo_branch"], + no_pull=ctx.obj["modules_repo_no_pull"], ) stdout.print(module_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: @@ -37,11 +37,11 @@ def modules_list_local(ctx, keywords, json, directory): # pylint: disable=redef try: module_list = ModuleList( - directory, - False, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], + directory=directory, + remote=False, + remote_url=ctx.obj["modules_repo_url"], + branch=ctx.obj["modules_repo_branch"], + no_pull=ctx.obj["modules_repo_no_pull"], ) stdout.print(module_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: diff --git a/nf_core/commands_subworkflows.py b/nf_core/commands_subworkflows.py index 8e90a8116b..b1fdddac75 100644 --- a/nf_core/commands_subworkflows.py +++ b/nf_core/commands_subworkflows.py @@ -63,7 +63,7 @@ def subworkflows_test(ctx, subworkflow, directory, no_prompts, update, once, pro sys.exit(1) -def subworkflows_list_remote(ctx, keywords, json): +def subworkflows_list_remote(ctx, keywords, json, directory): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -71,11 +71,11 @@ def subworkflows_list_remote(ctx, keywords, json): try: subworkflow_list = SubworkflowList( - ".", - True, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], + directory=directory, + remote=True, + modules_repo_url=ctx.obj["modules_repo_url"], + modules_repo_branch=ctx.obj["modules_repo_branch"], + modules_repo_no_pull=ctx.obj["modules_repo_no_pull"], ) stdout.print(subworkflow_list.list_components(keywords, json)) @@ -92,11 +92,11 @@ def subworkflows_list_local(ctx, keywords, json, directory): # pylint: disable= try: subworkflow_list = SubworkflowList( - directory, - False, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], + directory=directory, + remote=False, + modules_repo_url=ctx.obj["modules_repo_url"], + modules_repo_branch=ctx.obj["modules_repo_branch"], + modules_repo_no_pull=ctx.obj["modules_repo_no_pull"], ) stdout.print(subworkflow_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index f25fb33a6f..10937d1271 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Union import nf_core.utils +from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo @@ -33,7 +34,9 @@ def __init__( Initialise the ComponentClass object """ self.component_type: str = component_type - self.directory: Path = Path(directory) + self.directory: Path = Path(directory).absolute() + if remote_url is None: + remote_url = NF_CORE_MODULES_REMOTE self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress: bool = hide_progress self.no_prompts: bool = no_prompts @@ -60,10 +63,10 @@ def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: except FileNotFoundError: raise - self.default_modules_path = Path("modules", self.org) - self.default_tests_path = Path("tests", "modules", self.org) - self.default_subworkflows_path = Path("subworkflows", self.org) - self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org) + self.default_modules_path = Path("modules", self.org).absolute() + self.default_tests_path = Path("tests", "modules", self.org).absolute() + self.default_subworkflows_path = Path("subworkflows", self.org).absolute() + self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org).absolute() def get_local_components(self) -> List[str]: """ @@ -212,6 +215,8 @@ def check_modules_structure(self) -> None: self.modules_repo.setup_local_repo( self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress ) + if self.modules_repo.repo_path is None: + raise ValueError("Modules repo path is None") # Move wrong modules to the right directory for module in wrong_location_modules: modules_dir = Path("modules").resolve() diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 57c0034ba4..76e49a029b 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -122,9 +122,11 @@ def check_inputs(self) -> None: ).unsafe_ask() except LookupError: raise - - self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) - + if self.modules_repo.repo_path is None: + raise ValueError("modules_repo.repo_path is None") + self.component_dir = Path( + self.component_type, self.modules_repo.repo_path, *self.component_name.split("/") + ).absolute() # First, sanity check that the module directory exists if not Path(self.directory, self.component_dir).is_dir(): raise UserWarning( @@ -194,7 +196,6 @@ def generate_snapshot(self) -> bool: self.update = False # reset self.update to False to test if the new snapshot is stable tag = f"subworkflows/{self.component_name}" if self.component_type == "subworkflows" else self.component_name profile = self.profile if self.profile else os.environ["PROFILE"] - result = nf_core.utils.run_cmd( "nf-test", f"test --tag {tag} --profile {profile} {verbose} {update}", @@ -208,7 +209,6 @@ def generate_snapshot(self) -> bool: obsolete_snapshots = compiled_pattern.search(nftest_out.decode()) if obsolete_snapshots: self.obsolete_snapshots = True - # check if nf-test was successful if "Assertion failed:" in nftest_out.decode(): return False @@ -216,6 +216,9 @@ def generate_snapshot(self) -> bool: log.error("Test file 'main.nf.test' not found") self.errors.append("Test file 'main.nf.test' not found") return False + elif "No tests to execute" in nftest_out.decode(): + log.debug("No tests to execute") # no tests to execute is not an error anymore in nf-test v0.9.2 + return True else: log.debug("nf-test successful") return True diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c781905618..e1ec330248 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -132,7 +132,7 @@ def create(self) -> bool: if self.subtool: self.component_name = f"{self.component}/{self.subtool}" - self.component_dir = Path(self.component, self.subtool) + self.component_dir = Path(self.component, self.subtool).absolute() self.component_name_underscore = self.component_name.replace("/", "_") diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 4c20e60864..7c4ba048d4 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -1,6 +1,5 @@ import json import logging -from pathlib import Path from typing import Dict, List, Optional, Union, cast import rich.table @@ -13,17 +12,9 @@ class ComponentList(ComponentCommand): - def __init__( - self, - component_type: str, - pipeline_dir: Union[str, Path] = ".", - remote: bool = True, - remote_url: Optional[str] = None, - branch: Optional[str] = None, - no_pull: bool = False, - ) -> None: + def __init__(self, component_type: str, remote: bool = True, **kwargs) -> None: self.remote = remote - super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + super().__init__(component_type, **kwargs) def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: """ diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 901a7f02fe..f71b567438 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -414,7 +414,7 @@ def get_single_component_info(self, component): config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component) if config_entry is not None and config_entry is not True: if config_entry is False: - log.warn( + log.warning( f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' for '{component}' is set to False" ) return (self.modules_repo, None, None, None) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index d98eac7cd6..0bfb8657c4 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -19,7 +19,6 @@ from rich.table import Table import nf_core.modules.modules_utils -import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.nfcore_component import NFCoreComponent from nf_core.utils import NFCoreYamlConfig, custom_yaml_dumper, rich_force_colors @@ -29,14 +28,8 @@ class ModuleVersionBumper(ComponentCommand): - def __init__( - self, - pipeline_dir: Union[str, Path], - remote_url: Optional[str] = None, - branch: Optional[str] = None, - no_pull: bool = False, - ): - super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) + def __init__(self, **kwargs) -> None: + super().__init__("modules", **kwargs) self.up_to_date: List[Tuple[str, str]] = [] self.updated: List[Tuple[str, str]] = [] diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 49012cff40..5e1d7c1a74 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -20,7 +20,7 @@ import nf_core.components.nfcore_component import nf_core.modules.modules_utils import nf_core.utils -from nf_core.components.components_utils import get_biotools_id +from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE, get_biotools_id from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult from nf_core.components.nfcore_component import NFCoreComponent from nf_core.pipelines.lint_utils import console, run_prettier_on_file @@ -58,28 +58,8 @@ class ModuleLint(ComponentLint): module_todos = module_todos module_version = module_version - def __init__( - self, - directory: Union[str, Path], - fail_warned: bool = False, - fix: bool = False, - remote_url: Optional[str] = None, - branch: Optional[str] = None, - no_pull: bool = False, - registry: Optional[str] = None, - hide_progress: bool = False, - ): - super().__init__( - component_type="modules", - directory=directory, - fail_warned=fail_warned, - fix=fix, - remote_url=remote_url, - branch=branch, - no_pull=no_pull, - registry=registry, - hide_progress=hide_progress, - ) + def __init__(self, directory, **kwargs) -> None: + super().__init__("modules", directory=directory, **kwargs) def lint( self, @@ -312,11 +292,11 @@ def update_meta_yml_file(self, mod): for x, meta_ch_element in enumerate(meta_element): if element_name in meta_ch_element.keys(): # Copy current features of that input element form meta.yml - for feature in meta_element[x][element_name].keys(): + for feature in meta_ch_element[x][element_name].keys(): if feature not in element[element_name].keys(): - corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[x][ - element_name - ][feature] + corrected_meta_yml["input"][i][j][element_name][feature] = meta_ch_element[ + x + ][element_name][feature] break if "output" in meta_yml and correct_outputs != meta_outputs: diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index 68da570f67..85ba2dfb40 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -1,6 +1,4 @@ import logging -from pathlib import Path -from typing import Optional, Union from nf_core.components.list import ComponentList @@ -8,12 +6,5 @@ class ModuleList(ComponentList): - def __init__( - self, - pipeline_dir: Union[str, Path] = ".", - remote: bool = True, - remote_url: Optional[str] = None, - branch: Optional[str] = None, - no_pull: bool = False, - ): - super().__init__("modules", pipeline_dir, remote, remote_url, branch, no_pull) + def __init__(self, **kwargs) -> None: + super().__init__("modules", **kwargs) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 357fc49cc5..641399d804 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -47,16 +47,11 @@ def __init__( # This allows us to set this one time and then keep track of the user's choice ModulesRepo.no_pull_global |= no_pull - - # Check if the remote seems to be well formed - if remote_url is None: - remote_url = NF_CORE_MODULES_REMOTE - - self.remote_url = remote_url + self.remote_url = remote_url or NF_CORE_MODULES_REMOTE self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, branch, hide_progress) + self.setup_local_repo(self.remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) if config_fn is None or repo_config is None: @@ -83,7 +78,9 @@ def gitless_repo(self): gitless_repo_url = gitless_repo_url[:-4] return gitless_repo_url - def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): + def setup_local_repo( + self, remote_url: str, branch: Optional[str] = None, hide_progress: bool = True, in_cache: bool = False + ) -> None: """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -95,8 +92,9 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): Sets self.repo """ self.local_repo_dir = Path(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) + try: - if not os.path.exists(self.local_repo_dir): + if not self.local_repo_dir.exists(): try: pbar = rich.progress.Progress( "[bold blue]{task.description}", @@ -107,13 +105,13 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): ) with pbar: self.repo = git.Repo.clone_from( - remote, + remote_url, self.local_repo_dir, - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + progress=RemoteProgressbar(pbar, self.fullname, remote_url, "Cloning"), ) ModulesRepo.update_local_repo_status(self.fullname, True) except GitCommandError: - raise LookupError(f"Failed to clone from the remote: `{remote}`") + raise LookupError(f"Failed to clone from the remote: `{remote_url}`") # Verify that the requested branch exists by checking it out self.setup_branch(branch) else: @@ -132,7 +130,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): ) with pbar: self.repo.remotes.origin.fetch( - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + progress=RemoteProgressbar(pbar, self.fullname, remote_url, "Pulling") ) ModulesRepo.update_local_repo_status(self.fullname, True) @@ -143,13 +141,13 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): # Now merge the changes tracking_branch = self.repo.active_branch.tracking_branch() if tracking_branch is None: - raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") + raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{remote_url}'") self.repo.git.merge(tracking_branch.name) except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): log.info(f"Removing '{self.local_repo_dir}'") shutil.rmtree(self.local_repo_dir) - self.setup_local_repo(remote, branch, hide_progress) + self.setup_local_repo(remote_url, branch, hide_progress) else: raise LookupError("Exiting due to error with local modules git repo") diff --git a/nf_core/pipelines/create/githubrepo.py b/nf_core/pipelines/create/githubrepo.py index b37dfb6170..b688eee68f 100644 --- a/nf_core/pipelines/create/githubrepo.py +++ b/nf_core/pipelines/create/githubrepo.py @@ -150,7 +150,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: f"Repo will be created in the GitHub organisation account '{github_variables['repo_org']}'" ) except UnknownObjectException: - log.warn(f"Provided organisation '{github_variables['repo_org']}' not found. ") + log.warning(f"Provided organisation '{github_variables['repo_org']}' not found. ") # Create the repo try: diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py index 9e84d6cbe0..5668fe7528 100644 --- a/nf_core/subworkflows/list.py +++ b/nf_core/subworkflows/list.py @@ -1,6 +1,4 @@ import logging -from pathlib import Path -from typing import Optional, Union from nf_core.components.list import ComponentList @@ -8,12 +6,5 @@ class SubworkflowList(ComponentList): - def __init__( - self, - pipeline_dir: Union[str, Path] = ".", - remote: bool = True, - remote_url: Optional[str] = None, - branch: Optional[str] = None, - no_pull: bool = False, - ) -> None: - super().__init__("subworkflows", pipeline_dir, remote, remote_url, branch, no_pull) + def __init__(self, **kwargs) -> None: + super().__init__("subworkflows", **kwargs) diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py index 9b6bf16928..70f00cc748 100644 --- a/nf_core/subworkflows/update.py +++ b/nf_core/subworkflows/update.py @@ -2,33 +2,5 @@ class SubworkflowUpdate(ComponentUpdate): - def __init__( - self, - pipeline_dir, - force=False, - prompt=False, - sha=None, - update_all=False, - show_diff=None, - save_diff_fn=None, - update_deps=False, - remote_url=None, - branch=None, - no_pull=False, - limit_output=False, - ): - super().__init__( - pipeline_dir, - "subworkflows", - force, - prompt, - sha, - update_all, - show_diff, - save_diff_fn, - update_deps, - remote_url, - branch, - no_pull, - limit_output, - ) + def __init__(self, pipeline_dir, **kwargs) -> None: + super().__init__(pipeline_dir, "subworkflows", **kwargs) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index dd61b72a2b..ecbd2aaf54 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -175,7 +175,7 @@ def verify_sha(self, prompt, sha): return True - def setup_branch(self, branch): + def setup_branch(self, branch: Optional[str] = None) -> None: """ Verify that we have a branch and otherwise use the default one. The branch is then checked out to verify that it exists in the repo. @@ -195,7 +195,7 @@ def setup_branch(self, branch): # Verify that the branch exists by checking it out self.branch_exists() - def get_default_branch(self): + def get_default_branch(self) -> str: """ Gets the default branch for the repo (the branch origin/HEAD is pointing to) """ @@ -203,16 +203,16 @@ def get_default_branch(self): _, branch = origin_head.ref.name.split("/") return branch - def branch_exists(self): + def branch_exists(self) -> None: """ Verifies that the branch exists in the repository by trying to check it out """ try: self.checkout_branch() - except GitCommandError: - raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") + except GitCommandError as e: + raise LookupError(e.stderr) - def verify_branch(self): + def verify_branch(self) -> None: """ Verifies the active branch conforms to the correct directory structure """ @@ -225,7 +225,7 @@ def verify_branch(self): ) raise LookupError(err_str) - def checkout_branch(self): + def checkout_branch(self) -> None: """ Checks out the specified branch of the repository """ @@ -242,7 +242,7 @@ def checkout_branch(self): else: raise e - def checkout(self, commit): + def checkout(self, commit: str) -> None: """ Checks out the repository at the requested commit diff --git a/nf_core/utils.py b/nf_core/utils.py index 16125aed33..5085bb5ee5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -67,10 +67,10 @@ ) NFCORE_CACHE_DIR = Path( - os.environ.get("XDG_CACHE_HOME", Path(os.getenv("HOME") or "", ".cache")), - "nfcore", + os.getenv("XDG_CACHE_HOME", Path(os.getenv("HOME") or "", ".cache")), + "nf-core", ) -NFCORE_DIR = Path(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore") +NFCORE_DIR = Path(os.getenv("XDG_CONFIG_HOME", Path(os.getenv("HOME") or "", ".config")), "nf-core") def fetch_remote_version(source_url): @@ -305,6 +305,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: try: k, v = ul.split(" = ", 1) config[k] = v.strip("'\"") + log.debug(f"Added config key:value pair: {k}={v}") except ValueError: log.debug(f"Couldn't find key=value config pair:\n {ul}") @@ -360,11 +361,9 @@ def run_cmd(executable: str, cmd: str) -> Union[Tuple[bytes, bytes], None]: def setup_nfcore_dir() -> bool: """Creates a directory for files that need to be kept between sessions - Currently only used for keeping local copies of modules repos """ - if not NFCORE_DIR.exists(): - NFCORE_DIR.mkdir(parents=True) + NFCORE_DIR.mkdir(parents=True, exist_ok=True) return True @@ -378,28 +377,44 @@ def setup_requests_cachedir() -> Dict[str, Union[Path, datetime.timedelta, str]] Also returns the config dict so that we can use the same setup with a Session. """ pyversion: str = ".".join(str(v) for v in sys.version_info[0:3]) - cachedir: Path = setup_nfcore_cachedir(f"cache_{pyversion}") - config: Dict[str, Union[Path, datetime.timedelta, str]] = { - "cache_name": Path(cachedir, "github_info"), - "expire_after": datetime.timedelta(hours=1), - "backend": "sqlite", - } - - logging.getLogger("requests_cache").setLevel(logging.WARNING) - return config + try: + # Create cache directory with exist_ok=True + cachedir: Path = setup_nfcore_cachedir(f"cache_{pyversion}") + cachedir.mkdir(parents=True, exist_ok=True) + + # Ensure the github_info subdirectory exists + github_cache = cachedir / "github_info" + github_cache.mkdir(parents=True, exist_ok=True) + + config: Dict[str, Union[Path, datetime.timedelta, str]] = { + "cache_name": github_cache, + "expire_after": datetime.timedelta(hours=1), + "backend": "sqlite", + } + + logging.getLogger("requests_cache").setLevel(logging.WARNING) + return config + except PermissionError: + # Fallback to temporary directory if we can't create in home + import tempfile + + tmp_dir = Path(tempfile.gettempdir()) / "nf-core" + tmp_dir.mkdir(parents=True, exist_ok=True) + return { + "cache_name": tmp_dir / "github_info", + "expire_after": datetime.timedelta(hours=1), + "backend": "sqlite", + } def setup_nfcore_cachedir(cache_fn: Union[str, Path]) -> Path: """Sets up local caching for caching files between sessions.""" - cachedir = Path(NFCORE_CACHE_DIR, cache_fn) - try: - if not Path(cachedir).exists(): - Path(cachedir).mkdir(parents=True) + # Create directory with parents=True and exist_ok=True to handle race conditions + Path(cachedir).mkdir(parents=True, exist_ok=True) except PermissionError: - log.warn(f"Could not create cache directory: {cachedir}") - + log.warning(f"Could not create cache directory: {cachedir}") return cachedir @@ -1360,8 +1375,10 @@ def set_wd(path: Path) -> Generator[None, None, None]: Path to the working directory to be used inside this context. """ start_wd = Path().absolute() - os.chdir(Path(path).resolve()) try: + os.chdir(Path(path).resolve()) yield + except Exception as e: + log.error(f"Error setting working directory to {path}: {e}") finally: os.chdir(start_wd) diff --git a/pyproject.toml b/pyproject.toml index 775f04c9a1..7a7144debf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ norecursedirs = [ "nf_core", "docs", ] +asyncio_default_fixture_loop_scope = "loop" [tool.ruff] line-length = 120 diff --git a/requirements-dev.txt b/requirements-dev.txt index aab9b1e5d7..1909c40637 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,5 +18,7 @@ typing_extensions >=4.0.0 pytest-asyncio pytest-textual-snapshot==1.0.0 pytest-workflow>=2.0.0 +pytest-xdist pytest>=8.0.0 ruff + diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index a5a8eaba39..55d5870ce0 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,6 +1,8 @@ """Test generate a snapshot""" import json +import sys +from io import StringIO from pathlib import Path from unittest.mock import MagicMock @@ -120,11 +122,19 @@ def test_test_not_found(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") + test_file = Path(snap_generator.default_modules_path, snap_generator.component_name, "tests", "main.nf.test") test_file.rename(test_file.parent / "main.nf.test.bak") - with pytest.raises(UserWarning) as e: - snap_generator.run() - assert "Test file 'main.nf.test' not found" in str(e.value) + + captured_output = StringIO() + sys.stdout = captured_output + + try: + assert snap_generator.run() + output = captured_output.getvalue() + assert "No tests to execute" in output + finally: + sys.stdout = sys.__stdout__ # Restore stdout + Path(test_file.parent / "main.nf.test.bak").rename(test_file) diff --git a/tests/test_components.py b/tests/test_components.py index eaf999c3c3..f8d75671c0 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -8,6 +8,8 @@ from git.repo import Repo +# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir +os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp() from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL @@ -16,9 +18,8 @@ class TestComponents(unittest.TestCase): def setUp(self): """Clone a testing version the nf-core/modules repo""" - self.tmp_dir = Path(tempfile.mkdtemp()) - self.nfcore_modules = Path(self.tmp_dir, "modules-test") - + self.nfcore_modules = Path(tempfile.mkdtemp(), "modules-test") + self.nfcore_modules.mkdir(parents=True, exist_ok=True) Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_NFTEST_BRANCH) # Set $PROFILE environment variable to docker - tests will run with Docker @@ -28,9 +29,8 @@ def setUp(self): def tearDown(self): """Clean up temporary files and folders""" - # Clean up temporary files - if self.tmp_dir.is_dir(): - shutil.rmtree(self.tmp_dir) + if self.nfcore_modules.exists(): + shutil.rmtree(self.nfcore_modules) ############################################ # Test of the individual components commands. # diff --git a/tests/test_modules.py b/tests/test_modules.py index d0692236e8..195574332b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,6 +1,8 @@ """Tests covering the modules commands""" import json +import os +import tempfile import unittest from pathlib import Path @@ -17,6 +19,9 @@ import nf_core.pipelines.create.create from nf_core import __version__ from nf_core.pipelines.lint_utils import run_prettier_on_file + +# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir +os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp() from nf_core.utils import NFCoreYamlConfig from .utils import ( @@ -35,6 +40,7 @@ def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" + yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True yaml.indent(mapping=2, sequence=2, offset=0) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 455b2b71c2..e2c1d4d768 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -1,8 +1,13 @@ +import os import shutil +import tempfile from unittest import TestCase import pytest +# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir +os.environ["NXF_HOME"] = tempfile.mkdtemp() +os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp() from nf_core.utils import Pipeline from .utils import create_tmp_pipeline @@ -15,10 +20,6 @@ def setUp(self) -> None: self.pipeline_obj = Pipeline(self.pipeline_dir) self.pipeline_obj._load() - def tearDown(self) -> None: - """Remove the test pipeline directory""" - shutil.rmtree(self.tmp_dir) - def _make_pipeline_copy(self): """Make a copy of the test pipeline that can be edited diff --git a/tests/test_utils.py b/tests/test_utils.py index b13c8eb37d..6a8fa4e85f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -202,6 +202,6 @@ def test_set_wd(self): def test_set_wd_revert_on_raise(self): wd_before_context = Path().resolve() with pytest.raises(Exception): - with nf_core.utils.set_wd(self.tmp_dir): + with nf_core.utils.set_wd(self.tmp_dir / "non_existent_dir"): raise Exception assert wd_before_context == Path().resolve()