From d4c653c827e855708eaf5fcba9bd89bacc571ca2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:09:33 -0400 Subject: [PATCH 1/4] [pre-commit.ci] pre-commit autoupdate (#1001) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 221ddd904..d814d507f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ files: ^(.*\.(py|yaml))$ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.6.8 hooks: - id: ruff args: ["--fix"] From 02cdc2052351e95ba3914d589e40623135c4c4ba Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 4 Oct 2024 11:35:55 -0500 Subject: [PATCH 2/4] allow missing sessions (#1000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Eric Larson Co-authored-by: Richard Höchenberger --- .github/workflows/run-tests.yml | 4 +- docs/source/v1.10.md.inc | 1 + mne_bids_pipeline/_config.py | 6 ++ mne_bids_pipeline/_config_utils.py | 23 ++++++++ mne_bids_pipeline/_docs.py | 18 ++++++ .../steps/init/_01_init_derivatives_dir.py | 6 +- .../steps/init/_02_find_empty_room.py | 7 +-- .../steps/preprocessing/_01_data_quality.py | 7 +-- .../steps/preprocessing/_02_head_pos.py | 6 +- .../steps/preprocessing/_03_maxfilter.py | 11 ++-- .../preprocessing/_04_frequency_filter.py | 6 +- .../preprocessing/_05_regress_artifact.py | 6 +- .../steps/preprocessing/_06a1_fit_ica.py | 7 +-- .../preprocessing/_06a2_find_ica_artifacts.py | 7 +-- .../steps/preprocessing/_06b_run_ssp.py | 7 +-- .../steps/preprocessing/_07_make_epochs.py | 7 +-- .../steps/preprocessing/_08a_apply_ica.py | 10 ++-- .../steps/preprocessing/_08b_apply_ssp.py | 11 ++-- .../steps/preprocessing/_09_ptp_reject.py | 6 +- .../steps/sensor/_01_make_evoked.py | 7 +-- .../steps/sensor/_02_decoding_full_epochs.py | 7 +-- .../steps/sensor/_03_decoding_time_by_time.py | 7 +-- .../steps/sensor/_04_time_frequency.py | 7 +-- .../steps/sensor/_05_decoding_csp.py | 7 +-- .../steps/sensor/_06_make_cov.py | 7 +-- .../steps/source/_04_make_forward.py | 7 +-- .../steps/source/_05_make_inverse.py | 7 +-- .../steps/source/_99_group_average.py | 9 +-- mne_bids_pipeline/tests/test_run.py | 59 +++++++++++++++++++ 29 files changed, 186 insertions(+), 94 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 879eab4f9..7979df854 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - run: pip install --upgrade pip - run: pip install -ve .[tests] codespell tomli - run: make codespell-error @@ -49,7 +49,7 @@ jobs: pyvista: false - uses: actions/setup-python@v5 with: - python-version: "3.11" # no "multidict" wheels on 3.12 yet + python-version: "3.12" - run: pip install -ve .[tests] - uses: actions/cache@v4 with: diff --git a/docs/source/v1.10.md.inc b/docs/source/v1.10.md.inc index fb84978b2..0ad95e41a 100644 --- a/docs/source/v1.10.md.inc +++ b/docs/source/v1.10.md.inc @@ -3,6 +3,7 @@ ### :new: New features & enhancements - It is now possible to use separate MRIs for each session within a subject, as in longitudinal studies. This is achieved by creating separate "subject" folders for each subject-session combination, with the naming convention `sub-XXX_ses-YYY`, in the freesurfer `SUBJECTS_DIR`. (#987 by @drammock) +- New config option [`allow_missing_sessions`][mne_bids_pipeline._config.allow_missing_sessions] allows to continue when not all sessions are present for all subjects. (#1000 by @drammock) [//]: # (### :warning: Behavior changes) diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py index 9a1f792d2..0c0f7210a 100644 --- a/mne_bids_pipeline/_config.py +++ b/mne_bids_pipeline/_config.py @@ -79,6 +79,12 @@ BIDS dataset. """ +allow_missing_sessions: bool = False +""" +Whether to continue processing the dataset if some combinations of `subjects` and +`sessions` are missing. +""" + task: str = "" """ The task to process. diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index b52ab95bc..9c5a1622a 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -132,6 +132,29 @@ def get_sessions(config: SimpleNamespace) -> list[None] | list[str]: return sessions +def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str]]: + subj_sessions = dict() + cfg_sessions = get_sessions(config) + for subject in get_subjects(config): + # Only traverse through the current subject's directory + valid_sessions_subj = _get_entity_vals_cached( + config.bids_root / f"sub-{subject}", + entity_key="session", + ignore_datatypes=_get_ignore_datatypes(config), + ) + # use [None] instead of [] like is done in `get_sessions()` + valid_sessions_subj = valid_sessions_subj or [None] + missing_sessions = set(cfg_sessions) - set(valid_sessions_subj) + if missing_sessions and not config.allow_missing_sessions: + raise RuntimeError( + f"Subject {subject} is missing session{_pl(missing_sessions)} " + f"{tuple(sorted(missing_sessions))}, and " + "`config.allow_missing_sessions` is False" + ) + subj_sessions[subject] = sorted(set(cfg_sessions) & set(valid_sessions_subj)) + return subj_sessions + + def get_runs_all_subjects( config: SimpleNamespace, ) -> dict[str, list[None] | list[str]]: diff --git a/mne_bids_pipeline/_docs.py b/mne_bids_pipeline/_docs.py index 440b34690..2069fdf69 100644 --- a/mne_bids_pipeline/_docs.py +++ b/mne_bids_pipeline/_docs.py @@ -94,6 +94,7 @@ "ch_types", "task_is_rest", "data_type", + "allow_missing_sessions", ) # Eventually we could parse AST to get these, but this is simple enough _EXTRA_FUNCS = { @@ -105,6 +106,10 @@ class _ParseConfigSteps: def __init__(self, force_empty=None): + """Build a mapping from config options to tuples of steps that use each option. + + The mapping is stored in `self.steps`. + """ self._force_empty = _FORCE_EMPTY if force_empty is None else force_empty self.steps = defaultdict(list) # Add a few helper functions @@ -116,6 +121,7 @@ def __init__(self, force_empty=None): _config_utils.get_fs_subjects_dir, _config_utils.get_mf_cal_fname, _config_utils.get_mf_ctc_fname, + _config_utils.get_subjects_sessions, ): this_list = [] for attr in ast.walk(ast.parse(inspect.getsource(func))): @@ -152,6 +158,18 @@ def __init__(self, force_empty=None): if keyword.value.attr in ("exec_params",): continue self._add_step_option(step, keyword.value.attr) + for arg in call.args: + if not isinstance(arg, ast.Name): + continue + if arg.id != "config": + continue + key = call.func.id + # e.g., get_subjects_sessions(config) + if key in _MANUAL_KWS: + for option in _MANUAL_KWS[key]: + self._add_step_option(step, option) + break + # Also look for root-level conditionals like use_maxwell_filter # or spatial_filter for cond in ast.iter_child_nodes(func): diff --git a/mne_bids_pipeline/steps/init/_01_init_derivatives_dir.py b/mne_bids_pipeline/steps/init/_01_init_derivatives_dir.py index d204111a9..d9aa12782 100644 --- a/mne_bids_pipeline/steps/init/_01_init_derivatives_dir.py +++ b/mne_bids_pipeline/steps/init/_01_init_derivatives_dir.py @@ -9,7 +9,7 @@ from mne_bids.config import BIDS_VERSION from mne_bids.utils import _write_json -from mne_bids_pipeline._config_utils import _bids_kwargs, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import _bids_kwargs, get_subjects_sessions from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._run import _prep_out_files, failsafe_run @@ -76,8 +76,8 @@ def main(*, config): init_dataset(cfg=get_config(config=config), exec_params=config.exec_params) # Don't bother with parallelization here as I/O operations are generally # not well parallelized (and this should be very fast anyway) - for subject in get_subjects(config): - for session in get_sessions(config): + for subject, sessions in get_subjects_sessions(config).items(): + for session in sessions: init_subject_dirs( cfg=get_config( config=config, diff --git a/mne_bids_pipeline/steps/init/_02_find_empty_room.py b/mne_bids_pipeline/steps/init/_02_find_empty_room.py index 42826657f..d58f7df49 100644 --- a/mne_bids_pipeline/steps/init/_02_find_empty_room.py +++ b/mne_bids_pipeline/steps/init/_02_find_empty_room.py @@ -9,8 +9,7 @@ _pl, get_datatype, get_mf_reference_run, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import _empty_room_match_path from mne_bids_pipeline._io import _write_json @@ -127,8 +126,8 @@ def main(*, config) -> None: # This will be I/O bound if the sidecar is not complete, so let's not run # in parallel. logs = list() - for subject in get_subjects(config): - for session in get_sessions(config): + for subject, sessions in get_subjects_sessions(config).items(): + for session in sessions: run = get_mf_reference_run(config=config) logs.append( find_empty_room( diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index fb299be99..df96e4745 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -11,8 +11,7 @@ get_mf_cal_fname, get_mf_ctc_fname, get_runs_tasks, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import ( _bads_path, @@ -349,8 +348,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_02_head_pos.py b/mne_bids_pipeline/steps/preprocessing/_02_head_pos.py index 25c220686..aefce0efe 100644 --- a/mne_bids_pipeline/steps/preprocessing/_02_head_pos.py +++ b/mne_bids_pipeline/steps/preprocessing/_02_head_pos.py @@ -4,7 +4,7 @@ import mne -from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions from mne_bids_pipeline._import_data import ( _get_run_rest_noise_path, _import_data_kwargs, @@ -173,8 +173,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py index 3a68b39da..f3839e088 100644 --- a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py +++ b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py @@ -27,8 +27,7 @@ get_mf_cal_fname, get_mf_ctc_fname, get_runs_tasks, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import ( _get_mf_reference_run_path, @@ -613,8 +612,8 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) # Second: maxwell_filter @@ -637,8 +636,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py b/mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py index 899f953d3..958ccc279 100644 --- a/mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py +++ b/mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py @@ -23,7 +23,7 @@ from mne.io.pick import _picks_to_idx from mne.preprocessing import EOGRegression -from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions from mne_bids_pipeline._import_data import ( _get_run_rest_noise_path, _import_data_kwargs, @@ -328,8 +328,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_05_regress_artifact.py b/mne_bids_pipeline/steps/preprocessing/_05_regress_artifact.py index 89091532c..4b1c642a4 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05_regress_artifact.py +++ b/mne_bids_pipeline/steps/preprocessing/_05_regress_artifact.py @@ -6,7 +6,7 @@ from mne.io.pick import _picks_to_idx from mne.preprocessing import EOGRegression -from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions from mne_bids_pipeline._import_data import ( _get_run_rest_noise_path, _import_data_kwargs, @@ -159,8 +159,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py index b44d0c8b8..319f1c92c 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py +++ b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py @@ -21,8 +21,7 @@ _bids_kwargs, get_eeg_reference, get_runs, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import annotations_to_events, make_epochs from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -377,7 +376,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py b/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py index 3c065ce43..a1c9bde18 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py +++ b/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py @@ -20,8 +20,7 @@ _bids_kwargs, get_eeg_reference, get_runs, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -394,7 +393,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py b/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py index 45c4cb7cf..c4e7170f0 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py +++ b/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py @@ -17,8 +17,7 @@ _pl, _proj_path, get_runs, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -279,7 +278,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py b/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py index 8fe346b3a..c90b0fa33 100644 --- a/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py +++ b/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py @@ -17,8 +17,7 @@ _bids_kwargs, get_eeg_reference, get_runs, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import annotations_to_events, make_epochs from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -355,7 +354,7 @@ def main(*, config) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py b/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py index f8e5cd47a..2ea764ae7 100644 --- a/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py +++ b/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py @@ -12,7 +12,7 @@ from mne.preprocessing import read_ica from mne_bids import BIDSPath -from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions from mne_bids_pipeline._import_data import _get_run_rest_noise_path, _import_data_kwargs from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -273,8 +273,8 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) # Raw parallel, run_func = parallel_func( @@ -292,8 +292,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py b/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py index 09562d0b8..7713b2844 100644 --- a/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py +++ b/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py @@ -11,8 +11,7 @@ from mne_bids_pipeline._config_utils import ( _proj_path, get_runs_tasks, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._import_data import _get_run_rest_noise_path, _import_data_kwargs from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -183,8 +182,8 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) # Raw parallel, run_func = parallel_func( @@ -202,8 +201,8 @@ def main(*, config: SimpleNamespace) -> None: run=run, task=task, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for run, task in get_runs_tasks( config=config, subject=subject, diff --git a/mne_bids_pipeline/steps/preprocessing/_09_ptp_reject.py b/mne_bids_pipeline/steps/preprocessing/_09_ptp_reject.py index 5885aa1f3..9de09c50a 100644 --- a/mne_bids_pipeline/steps/preprocessing/_09_ptp_reject.py +++ b/mne_bids_pipeline/steps/preprocessing/_09_ptp_reject.py @@ -15,7 +15,7 @@ import numpy as np from mne_bids import BIDSPath -from mne_bids_pipeline._config_utils import _bids_kwargs, get_sessions, get_subjects +from mne_bids_pipeline._config_utils import _bids_kwargs, get_subjects_sessions from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func from mne_bids_pipeline._reject import _get_reject @@ -275,7 +275,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/sensor/_01_make_evoked.py b/mne_bids_pipeline/steps/sensor/_01_make_evoked.py index b5b6f436f..60c4b0553 100644 --- a/mne_bids_pipeline/steps/sensor/_01_make_evoked.py +++ b/mne_bids_pipeline/steps/sensor/_01_make_evoked.py @@ -11,8 +11,7 @@ _restrict_analyze_channels, get_all_contrasts, get_eeg_reference, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -198,7 +197,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/sensor/_02_decoding_full_epochs.py b/mne_bids_pipeline/steps/sensor/_02_decoding_full_epochs.py index 7d34d11cf..488292f2a 100644 --- a/mne_bids_pipeline/steps/sensor/_02_decoding_full_epochs.py +++ b/mne_bids_pipeline/steps/sensor/_02_decoding_full_epochs.py @@ -26,8 +26,7 @@ _restrict_analyze_channels, get_decoding_contrasts, get_eeg_reference, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._decoding import LogReg, _decoding_preproc_steps from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -280,8 +279,8 @@ def main(*, config: SimpleNamespace) -> None: condition2=cond_2, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for (cond_1, cond_2) in get_decoding_contrasts(config) ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/sensor/_03_decoding_time_by_time.py b/mne_bids_pipeline/steps/sensor/_03_decoding_time_by_time.py index 6a0d9c581..bd226cc93 100644 --- a/mne_bids_pipeline/steps/sensor/_03_decoding_time_by_time.py +++ b/mne_bids_pipeline/steps/sensor/_03_decoding_time_by_time.py @@ -34,8 +34,7 @@ _restrict_analyze_channels, get_decoding_contrasts, get_eeg_reference, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._decoding import LogReg, _decoding_preproc_steps from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -361,8 +360,8 @@ def main(*, config: SimpleNamespace) -> None: condition2=cond_2, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for cond_1, cond_2 in get_decoding_contrasts(config) ] save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/sensor/_04_time_frequency.py b/mne_bids_pipeline/steps/sensor/_04_time_frequency.py index 747ecb0de..659f5afff 100644 --- a/mne_bids_pipeline/steps/sensor/_04_time_frequency.py +++ b/mne_bids_pipeline/steps/sensor/_04_time_frequency.py @@ -14,8 +14,7 @@ _bids_kwargs, _restrict_analyze_channels, get_eeg_reference, - get_sessions, - get_subjects, + get_subjects_sessions, sanitize_cond_name, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -200,7 +199,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index 557bb3ca7..c365544b8 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -18,8 +18,7 @@ _restrict_analyze_channels, get_decoding_contrasts, get_eeg_reference, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._decoding import ( LogReg, @@ -554,8 +553,8 @@ def main(*, config: SimpleNamespace) -> None: session=session, contrast=contrast, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions for contrast in get_decoding_contrasts(config) ) save_logs(logs=logs, config=config) diff --git a/mne_bids_pipeline/steps/sensor/_06_make_cov.py b/mne_bids_pipeline/steps/sensor/_06_make_cov.py index df75a417b..8a5acc914 100644 --- a/mne_bids_pipeline/steps/sensor/_06_make_cov.py +++ b/mne_bids_pipeline/steps/sensor/_06_make_cov.py @@ -14,8 +14,7 @@ _restrict_analyze_channels, get_eeg_reference, get_noise_cov_bids_path, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -324,7 +323,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/source/_04_make_forward.py b/mne_bids_pipeline/steps/source/_04_make_forward.py index 45167d8a7..d82c4587f 100644 --- a/mne_bids_pipeline/steps/source/_04_make_forward.py +++ b/mne_bids_pipeline/steps/source/_04_make_forward.py @@ -17,8 +17,7 @@ get_fs_subject, get_fs_subjects_dir, get_runs, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -285,7 +284,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/source/_05_make_inverse.py b/mne_bids_pipeline/steps/source/_05_make_inverse.py index 3a88ab194..cf1f2c146 100644 --- a/mne_bids_pipeline/steps/source/_05_make_inverse.py +++ b/mne_bids_pipeline/steps/source/_05_make_inverse.py @@ -18,8 +18,7 @@ get_fs_subject, get_fs_subjects_dir, get_noise_cov_bids_path, - get_sessions, - get_subjects, + get_subjects_sessions, sanitize_cond_name, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -206,7 +205,7 @@ def main(*, config: SimpleNamespace) -> None: subject=subject, session=session, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, sessions in get_subjects_sessions(config).items() + for session in sessions ) save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/steps/source/_99_group_average.py b/mne_bids_pipeline/steps/source/_99_group_average.py index 77b0dae15..afddef24a 100644 --- a/mne_bids_pipeline/steps/source/_99_group_average.py +++ b/mne_bids_pipeline/steps/source/_99_group_average.py @@ -15,6 +15,7 @@ get_fs_subjects_dir, get_sessions, get_subjects, + get_subjects_sessions, sanitize_cond_name, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger @@ -221,8 +222,8 @@ def main(*, config: SimpleNamespace) -> None: mne.datasets.fetch_fsaverage(subjects_dir=get_fs_subjects_dir(config)) cfg = get_config(config=config) exec_params = config.exec_params - subjects = get_subjects(config) - sessions = get_sessions(config) + all_sessions = get_sessions(config) + subjects_sessions = get_subjects_sessions(config) logs = list() with get_parallel_backend(exec_params): @@ -235,7 +236,7 @@ def main(*, config: SimpleNamespace) -> None: fs_subject=get_fs_subject(config=cfg, subject=subject, session=session), session=session, ) - for subject in subjects + for subject, sessions in subjects_sessions.items() for session in sessions ) logs += [ @@ -245,6 +246,6 @@ def main(*, config: SimpleNamespace) -> None: session=session, subject="average", ) - for session in sessions + for session in all_sessions ] save_logs(config=config, logs=logs) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 952be5f13..8dc44f757 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -4,6 +4,7 @@ import shutil import sys from collections.abc import Collection +from contextlib import nullcontext from pathlib import Path from typing import TypedDict @@ -204,3 +205,61 @@ def test_run(dataset, monkeypatch, dataset_test, capsys, tmp_path): with capsys.disabled(): print() main() + + +@pytest.mark.parametrize("allow_missing_sessions", (False, True)) +def test_missing_sessions(tmp_path, monkeypatch, capsys, allow_missing_sessions): + """Test the `allow_missing_sessions` config variable.""" + dataset = "fake" + bids_root = tmp_path / dataset + files = ( + "dataset_description.json", + *(f"participants.{x}" for x in ("json", "tsv")), + *(f"sub-1/sub-1_sessions.{x}" for x in ("json", "tsv")), + *( + f"sub-1/ses-a/eeg/sub-1_ses-a_task-foo_{x}.tsv" + for x in ("channels", "events") + ), + *( + f"sub-1/ses-a/eeg/sub-1_ses-a_task-foo_eeg.{x}" + for x in ("eeg", "json", "vhdr", "vmrk") + ), + ) + for _file in files: + path = bids_root / _file + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + # fake a config file (can't use static file because `bids_root` is in `tmp_path`) + config = f""" +bids_root = "{bids_root}" +deriv_root = "{tmp_path / "derivatives" / "mne-bids-pipeline" / dataset}" +interactive = False +subjects = ["1"] +sessions = ["a", "b"] +ch_types = ["eeg"] +conditions = ["zzz"] +allow_missing_sessions = {allow_missing_sessions} +""" + config_path = tmp_path / "fake_config_missing_session.py" + with open(config_path, "w") as fid: + fid.write(config) + # set up the context handler + context = ( + nullcontext() + if allow_missing_sessions + else pytest.raises(RuntimeError, match=r"Subject 1 is missing session \('b',\)") + ) + # run + command = [ + "mne_bids_pipeline", + str(config_path), + "--steps=init/_01_init_derivatives_dir", + ] + if "--pdb" in sys.argv: + command.append("--n_jobs=1") + monkeypatch.setenv("_MNE_BIDS_STUDY_TESTING", "true") + monkeypatch.setattr(sys, "argv", command) + with capsys.disabled(): + print() + with context: + main() From cce897d3ac3324a6ab5fbd5f7333e1df3c61f919 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:08:17 +0200 Subject: [PATCH 3/4] [pre-commit.ci] pre-commit autoupdate (#1002) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d814d507f..7a08c1f72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ files: ^(.*\.(py|yaml))$ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.8 + rev: v0.6.9 hooks: - id: ruff args: ["--fix"] From 094926ec7e55609b1dc7755cea2c5dab3078b67c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 20:56:55 -0400 Subject: [PATCH 4/4] [pre-commit.ci] pre-commit autoupdate (#1006) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Larson --- .github/workflows/run-tests.yml | 12 ++++++++---- .pre-commit-config.yaml | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7979df854..13f3f51c0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,7 +3,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }} cancel-in-progress: true -on: [push, pull_request] +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] jobs: check-doc: @@ -18,7 +22,7 @@ jobs: with: python-version: "3.12" - run: pip install --upgrade pip - - run: pip install -ve .[tests] codespell tomli + - run: pip install -ve .[tests] codespell tomli --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr" - run: make codespell-error - run: pytest mne_bids_pipeline -m "not dataset_test" - uses: codecov/codecov-action@v4 @@ -31,7 +35,7 @@ jobs: runs-on: ${{ matrix.os }} defaults: run: - shell: bash -el {0} + shell: bash -e {0} strategy: matrix: include: @@ -50,7 +54,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install -ve .[tests] + - run: pip install -ve .[tests] --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr" - uses: actions/cache@v4 with: key: ds001971 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a08c1f72..14bcee504 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ files: ^(.*\.(py|yaml))$ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 + rev: v0.7.0 hooks: - id: ruff args: ["--fix"]