Merge remote-tracking branch 'upstream/main' into mypy

* upstream/main:
  [pre-commit.ci] pre-commit autoupdate (mne-tools#1006)
  [pre-commit.ci] pre-commit autoupdate (mne-tools#1002)
  allow missing sessions (mne-tools#1000)
  [pre-commit.ci] pre-commit autoupdate (mne-tools#1001)

.github/workflows/run-tests.yml

@@ -3,7 +3,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
   cancel-in-progress: true
 
-on: [push, pull_request]
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
 
 jobs:
   check-doc:
@@ -16,9 +20,9 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - run: pip install --upgrade pip
-    - run: pip install -ve .[tests] codespell tomli
+    - run: pip install -ve .[tests] codespell tomli --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr"
     - run: make codespell-error
     - run: pytest mne_bids_pipeline -m "not dataset_test"
     - uses: codecov/codecov-action@v4
@@ -31,7 +35,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     defaults:
       run:
-        shell: bash -el {0}
+        shell: bash -e {0}
     strategy:
       matrix:
         include:
@@ -49,8 +53,8 @@ jobs:
           pyvista: false
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"  # no "multidict" wheels on 3.12 yet
-      - run: pip install -ve .[tests]
+          python-version: "3.12"
+      - run: pip install -ve .[tests] --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr"
       - uses: actions/cache@v4
         with:
           key: ds001971

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 files: ^(.*\.(py|yaml))$
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.7
+    rev: v0.7.0
     hooks:
       - id: ruff
         args: ["--fix"]

docs/source/v1.10.md.inc

@@ -3,6 +3,7 @@
 ### :new: New features & enhancements
 
 - It is now possible to use separate MRIs for each session within a subject, as in longitudinal studies. This is achieved by creating separate "subject" folders for each subject-session combination, with the naming convention `sub-XXX_ses-YYY`, in the freesurfer `SUBJECTS_DIR`. (#987 by @drammock)
+- New config option [`allow_missing_sessions`][mne_bids_pipeline._config.allow_missing_sessions] allows to continue when not all sessions are present for all subjects. (#1000 by @drammock)
 
 [//]: # (### :warning: Behavior changes)
 

mne_bids_pipeline/_config.py

@@ -79,6 +79,12 @@
 BIDS dataset.
 """
 
+allow_missing_sessions: bool = False
+"""
+Whether to continue processing the dataset if some combinations of `subjects` and
+`sessions` are missing.
+"""
+
 task: str = ""
 """
 The task to process.

mne_bids_pipeline/_config_utils.py

@@ -122,6 +122,14 @@ def get_subjects(config: SimpleNamespace) -> list[str]:
 
 
 def get_sessions(config: SimpleNamespace) -> tuple[None] | tuple[str, ...]:
+    sessions = _get_sessions(config)
+    if not sessions:
+        return (None,)
+    else:
+        return sessions
+
+
+def _get_sessions(config: SimpleNamespace) -> tuple[str, ...]:
     sessions = copy.deepcopy(config.sessions)
     _all_sessions = _get_entity_vals_cached(
         root=config.bids_root,
@@ -131,10 +139,30 @@ def get_sessions(config: SimpleNamespace) -> tuple[None] | tuple[str, ...]:
     if sessions == "all":
         sessions = _all_sessions
 
-    if not sessions:
-        return (None,)
-    else:
-        return tuple(str(x) for x in sessions)
+    return tuple(str(x) for x in sessions)
+
+
+def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str] | list[None]]:
+    subj_sessions: dict[str, list[str] | list[None]] = dict()
+    cfg_sessions = _get_sessions(config)
+    for subject in get_subjects(config):
+        # Only traverse through the current subject's directory
+        valid_sessions_subj = _get_entity_vals_cached(
+            config.bids_root / f"sub-{subject}",
+            entity_key="session",
+            ignore_datatypes=_get_ignore_datatypes(config),
+        )
+        missing_sessions = set(cfg_sessions) - set(valid_sessions_subj)
+        if missing_sessions and not config.allow_missing_sessions:
+            raise RuntimeError(
+                f"Subject {subject} is missing session{_pl(missing_sessions)} "
+                f"{tuple(sorted(missing_sessions))}, and "
+                "`config.allow_missing_sessions` is False"
+            )
+        subj_sessions[subject] = sorted(set(cfg_sessions) & set(valid_sessions_subj))
+        if subj_sessions[subject] == []:
+            subj_sessions[subject] = [None]
+    return subj_sessions
 
 
 def get_runs_all_subjects(

mne_bids_pipeline/_docs.py

@@ -96,6 +96,7 @@
     "ch_types",
     "task_is_rest",
     "data_type",
+    "allow_missing_sessions",
 )
 # Eventually we could parse AST to get these, but this is simple enough
 _EXTRA_FUNCS = {
@@ -107,6 +108,10 @@
 
 class _ParseConfigSteps:
     def __init__(self, force_empty: tuple[str, ...] | None = None) -> None:
+        """Build a mapping from config options to tuples of steps that use each option.
+
+        The mapping is stored in `self.steps`.
+        """
         self._force_empty = _FORCE_EMPTY if force_empty is None else force_empty
         steps: dict[str, Any] = defaultdict(list)
 
@@ -123,6 +128,7 @@ def _add_step_option(step: str, option: str) -> None:
             _config_utils.get_fs_subjects_dir,
             _config_utils.get_mf_cal_fname,
             _config_utils.get_mf_ctc_fname,
+            _config_utils.get_subjects_sessions,
         ):
             this_list: list[str] = []
             assert isinstance(func_extra, FunctionType)
@@ -161,6 +167,19 @@ def _add_step_option(step: str, option: str) -> None:
                             if keyword.value.attr in ("exec_params",):
                                 continue
                             _add_step_option(step, keyword.value.attr)
+                        for arg in call.args:
+                            if not isinstance(arg, ast.Name):
+                                continue
+                            if arg.id != "config":
+                                continue
+                            assert isinstance(call.func, ast.Name)
+                            key = call.func.id
+                            # e.g., get_subjects_sessions(config)
+                            if key in _MANUAL_KWS:
+                                for option in _MANUAL_KWS[key]:
+                                    _add_step_option(step, option)
+                                break
+
                     # Also look for root-level conditionals like use_maxwell_filter
                     # or spatial_filter
                     for cond in ast.iter_child_nodes(func):

mne_bids_pipeline/steps/init/_01_init_derivatives_dir.py

@@ -8,7 +8,7 @@
 from mne_bids.config import BIDS_VERSION
 from mne_bids.utils import _write_json
 
-from mne_bids_pipeline._config_utils import _bids_kwargs, get_sessions, get_subjects
+from mne_bids_pipeline._config_utils import _bids_kwargs, get_subjects_sessions
 from mne_bids_pipeline._logging import gen_log_kwargs, logger
 from mne_bids_pipeline._run import _prep_out_files, failsafe_run
 from mne_bids_pipeline.typing import OutFilesT
@@ -76,8 +76,8 @@ def main(*, config):
     init_dataset(cfg=get_config(config=config), exec_params=config.exec_params)
     # Don't bother with parallelization here as I/O operations are generally
     # not well parallelized (and this should be very fast anyway)
-    for subject in get_subjects(config):
-        for session in get_sessions(config):
+    for subject, sessions in get_subjects_sessions(config).items():
+        for session in sessions:
             init_subject_dirs(
                 cfg=get_config(
                     config=config,

mne_bids_pipeline/steps/init/_02_find_empty_room.py

@@ -9,8 +9,7 @@
     _pl,
     get_datatype,
     get_mf_reference_run,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._import_data import _empty_room_match_path
 from mne_bids_pipeline._io import _write_json
@@ -127,8 +126,8 @@ def main(*, config) -> None:
     # This will be I/O bound if the sidecar is not complete, so let's not run
     # in parallel.
     logs = list()
-    for subject in get_subjects(config):
-        for session in get_sessions(config):
+    for subject, sessions in get_subjects_sessions(config).items():
+        for session in sessions:
             run = get_mf_reference_run(config=config)
             logs.append(
                 find_empty_room(

mne_bids_pipeline/steps/preprocessing/_01_data_quality.py

@@ -12,8 +12,7 @@
     get_mf_cal_fname,
     get_mf_ctc_fname,
     get_runs_tasks,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._import_data import (
     _bads_path,
@@ -354,8 +353,8 @@ def main(*, config: SimpleNamespace) -> None:
                 run=run,
                 task=task,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
             for run, task in get_runs_tasks(
                 config=config,
                 subject=subject,

mne_bids_pipeline/steps/preprocessing/_02_head_pos.py

@@ -4,7 +4,7 @@
 
 import mne
 
-from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects
+from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions
 from mne_bids_pipeline._import_data import (
     _get_run_rest_noise_path,
     _import_data_kwargs,
@@ -173,8 +173,8 @@ def main(*, config: SimpleNamespace) -> None:
                 run=run,
                 task=task,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
             for run, task in get_runs_tasks(
                 config=config,
                 subject=subject,

mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py

@@ -27,8 +27,7 @@
     get_mf_cal_fname,
     get_mf_ctc_fname,
     get_runs_tasks,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._import_data import (
     _get_mf_reference_run_path,
@@ -622,8 +621,8 @@ def main(*, config: SimpleNamespace) -> None:
                     subject=subject,
                     session=session,
                 )
-                for subject in get_subjects(config)
-                for session in get_sessions(config)
+                for subject, sessions in get_subjects_sessions(config).items()
+                for session in sessions
             )
 
         # Second: maxwell_filter
@@ -646,8 +645,8 @@ def main(*, config: SimpleNamespace) -> None:
                     run=run,
                     task=task,
                 )
-                for subject in get_subjects(config)
-                for session in get_sessions(config)
+                for subject, sessions in get_subjects_sessions(config).items()
+                for session in sessions
                 for run, task in get_runs_tasks(
                     config=config,
                     subject=subject,

mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py

@@ -23,7 +23,7 @@
 from mne.io.pick import _picks_to_idx
 from mne.preprocessing import EOGRegression
 
-from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects
+from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions
 from mne_bids_pipeline._import_data import (
     _get_run_rest_noise_path,
     _import_data_kwargs,
@@ -329,8 +329,8 @@ def main(*, config: SimpleNamespace) -> None:
                 run=run,
                 task=task,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
             for run, task in get_runs_tasks(
                 config=config,
                 subject=subject,

mne_bids_pipeline/steps/preprocessing/_05_regress_artifact.py

@@ -6,7 +6,7 @@
 from mne.io.pick import _picks_to_idx
 from mne.preprocessing import EOGRegression
 
-from mne_bids_pipeline._config_utils import get_runs_tasks, get_sessions, get_subjects
+from mne_bids_pipeline._config_utils import get_runs_tasks, get_subjects_sessions
 from mne_bids_pipeline._import_data import (
     _get_run_rest_noise_path,
     _import_data_kwargs,
@@ -159,8 +159,8 @@ def main(*, config: SimpleNamespace) -> None:
                 run=run,
                 task=task,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
             for run, task in get_runs_tasks(
                 config=config,
                 subject=subject,

mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py

@@ -21,8 +21,7 @@
     _bids_kwargs,
     get_eeg_reference,
     get_runs,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._import_data import annotations_to_events, make_epochs
 from mne_bids_pipeline._logging import gen_log_kwargs, logger
@@ -379,7 +378,7 @@ def main(*, config: SimpleNamespace) -> None:
                 subject=subject,
                 session=session,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
         )
     save_logs(config=config, logs=logs)

mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py

@@ -20,8 +20,7 @@
     _bids_kwargs,
     get_eeg_reference,
     get_runs,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._logging import gen_log_kwargs, logger
 from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func
@@ -394,7 +393,7 @@ def main(*, config: SimpleNamespace) -> None:
                 subject=subject,
                 session=session,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
         )
     save_logs(config=config, logs=logs)

mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py

@@ -17,8 +17,7 @@
     _pl,
     _proj_path,
     get_runs,
-    get_sessions,
-    get_subjects,
+    get_subjects_sessions,
 )
 from mne_bids_pipeline._logging import gen_log_kwargs, logger
 from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func
@@ -281,7 +280,7 @@ def main(*, config: SimpleNamespace) -> None:
                 subject=subject,
                 session=session,
             )
-            for subject in get_subjects(config)
-            for session in get_sessions(config)
+            for subject, sessions in get_subjects_sessions(config).items()
+            for session in sessions
         )
     save_logs(config=config, logs=logs)