From d0b8d7042a511c6ef99125cafe7ce5760decc63b Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 7 Jan 2025 16:36:31 -0600 Subject: [PATCH 01/12] fix ignoring sessions that have no data --- mne_bids_pipeline/_config_utils.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 8130475df..7faed86ef 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -142,14 +142,27 @@ def _get_sessions(config: SimpleNamespace) -> tuple[str, ...]: return tuple(str(x) for x in sessions) -def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str] | list[None]]: - subj_sessions: dict[str, list[str] | list[None]] = dict() +def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str]]: + subj_sessions: dict[str, list[str]] = dict() cfg_sessions = _get_sessions(config) + # find which tasks to ignore when deciding if a subj has data for a session + if config.task is None: + ignore_tasks = None + else: + all_tasks = _get_entity_vals_cached( + root=config.bids_root, + entity_key="task", + ignore_datatypes=_get_ignore_datatypes(config), + ) + ignore_tasks = tuple(set(all_tasks) - set([config.task])) for subject in get_subjects(config): - # Only traverse through the current subject's directory + # Check current subject's directory for available sessions valid_sessions_subj = _get_entity_vals_cached( config.bids_root / f"sub-{subject}", entity_key="session", + ignore_tasks=ignore_tasks, + ignore_acquisitions=("calibration", "crosstalk"), + ignore_suffixes=("scans", "coordsystem"), ignore_datatypes=_get_ignore_datatypes(config), ) missing_sessions = set(cfg_sessions) - set(valid_sessions_subj) @@ -159,9 +172,9 @@ def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str] | list f"{tuple(sorted(missing_sessions))}, and " "`config.allow_missing_sessions` is False" ) - subj_sessions[subject] = sorted(set(cfg_sessions) & set(valid_sessions_subj)) - if subj_sessions[subject] == []: - subj_sessions[subject] = [None] + keep_sessions = sorted(set(cfg_sessions) & set(valid_sessions_subj)) + if len(keep_sessions): + subj_sessions[subject] = keep_sessions return subj_sessions From 2421304fedd10fd6213aaa32e940446d59fc632b Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 7 Jan 2025 17:08:20 -0600 Subject: [PATCH 02/12] fix: default task is "" not None --- mne_bids_pipeline/_config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 7faed86ef..cd5914486 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -146,7 +146,7 @@ def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str]]: subj_sessions: dict[str, list[str]] = dict() cfg_sessions = _get_sessions(config) # find which tasks to ignore when deciding if a subj has data for a session - if config.task is None: + if config.task == "": ignore_tasks = None else: all_tasks = _get_entity_vals_cached( From edbaf61531b96c89ee8f2cf2df9fb6a1a7d7c686 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 7 Jan 2025 17:26:07 -0600 Subject: [PATCH 03/12] add the temporary pin --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d8643b495..d952a656c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ "openpyxl", "autoreject", "mne[hdf5] >=1.7", - "mne-bids[full]", + "mne-bids[full] @ git+https://github.com/mne-tools/mne-bids@41ce36a4dc938e85a2f6a32ed3eeb88c732bfbae", "filelock", ] dynamic = ["version"] From 7a71e495606f76773530460efbb98539a4184424 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 8 Jan 2025 10:39:29 -0600 Subject: [PATCH 04/12] allow direct references (temporary, for CI) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d952a656c..5c5be3e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,9 @@ exclude = [ "ignore_words.txt", ] +[tool.hatch.metadata] +allow-direct-references = true + [tool.codespell] skip = "docs/site/*,*.html,steps/freesurfer/contrib/*" ignore-words = "ignore_words.txt" From 92fad9b37bf8ae510ff4d1def4ab95cc5f404a91 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Thu, 9 Jan 2025 15:08:04 -0600 Subject: [PATCH 05/12] fix for datasets that don't have named sessions --- mne_bids_pipeline/_config_utils.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index cd5914486..cae8ac131 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -142,9 +142,11 @@ def _get_sessions(config: SimpleNamespace) -> tuple[str, ...]: return tuple(str(x) for x in sessions) -def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str]]: - subj_sessions: dict[str, list[str]] = dict() - cfg_sessions = _get_sessions(config) +def get_subjects_sessions( + config: SimpleNamespace, +) -> dict[str, tuple[None] | tuple[str, ...]]: + subj_sessions: dict[str, tuple[None] | tuple[str, ...]] = dict() + cfg_sessions = get_sessions(config) # find which tasks to ignore when deciding if a subj has data for a session if config.task == "": ignore_tasks = None @@ -165,14 +167,20 @@ def get_subjects_sessions(config: SimpleNamespace) -> dict[str, list[str]]: ignore_suffixes=("scans", "coordsystem"), ignore_datatypes=_get_ignore_datatypes(config), ) - missing_sessions = set(cfg_sessions) - set(valid_sessions_subj) - if missing_sessions and not config.allow_missing_sessions: - raise RuntimeError( - f"Subject {subject} is missing session{_pl(missing_sessions)} " - f"{tuple(sorted(missing_sessions))}, and " - "`config.allow_missing_sessions` is False" + # handle datasets that don't have (named) sessions + if cfg_sessions == (None,): + assert valid_sessions_subj == (), valid_sessions_subj + keep_sessions = cfg_sessions + else: + missing_sessions = tuple( + sorted(set(cfg_sessions) - set(valid_sessions_subj)) # type: ignore ) - keep_sessions = sorted(set(cfg_sessions) & set(valid_sessions_subj)) + if missing_sessions and not config.allow_missing_sessions: + raise RuntimeError( + f"Subject {subject} is missing session{_pl(missing_sessions)} " + f"{missing_sessions}, and `config.allow_missing_sessions` is False" + ) + keep_sessions = tuple(sorted(set(cfg_sessions) & set(valid_sessions_subj))) # type: ignore if len(keep_sessions): subj_sessions[subject] = keep_sessions return subj_sessions From adc5e4ce783cbe940e6a4150e4b85e25bc4a71e9 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 10 Jan 2025 09:47:36 -0600 Subject: [PATCH 06/12] cleaner --- mne_bids_pipeline/_config_utils.py | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index cae8ac131..7755ea7bc 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -145,42 +145,42 @@ def _get_sessions(config: SimpleNamespace) -> tuple[str, ...]: def get_subjects_sessions( config: SimpleNamespace, ) -> dict[str, tuple[None] | tuple[str, ...]]: - subj_sessions: dict[str, tuple[None] | tuple[str, ...]] = dict() - cfg_sessions = get_sessions(config) + subjects = get_subjects(config) + cfg_sessions = _get_sessions(config) + # easy case first: datasets that don't have (named) sessions + if not cfg_sessions: + return {subj: (None,) for subj in subjects} + # find which tasks to ignore when deciding if a subj has data for a session + ignore_datatypes = _get_ignore_datatypes(config) if config.task == "": ignore_tasks = None else: all_tasks = _get_entity_vals_cached( root=config.bids_root, entity_key="task", - ignore_datatypes=_get_ignore_datatypes(config), + ignore_datatypes=ignore_datatypes, ) ignore_tasks = tuple(set(all_tasks) - set([config.task])) - for subject in get_subjects(config): - # Check current subject's directory for available sessions + + # loop over subjs and check for available sessions + subj_sessions: dict[str, tuple[None] | tuple[str, ...]] = dict() + for subject in subjects: valid_sessions_subj = _get_entity_vals_cached( config.bids_root / f"sub-{subject}", entity_key="session", ignore_tasks=ignore_tasks, ignore_acquisitions=("calibration", "crosstalk"), ignore_suffixes=("scans", "coordsystem"), - ignore_datatypes=_get_ignore_datatypes(config), + ignore_datatypes=ignore_datatypes, ) - # handle datasets that don't have (named) sessions - if cfg_sessions == (None,): - assert valid_sessions_subj == (), valid_sessions_subj - keep_sessions = cfg_sessions - else: - missing_sessions = tuple( - sorted(set(cfg_sessions) - set(valid_sessions_subj)) # type: ignore + missing_sessions = sorted(set(cfg_sessions) - set(valid_sessions_subj)) + if missing_sessions and not config.allow_missing_sessions: + raise RuntimeError( + f"Subject {subject} is missing session{_pl(missing_sessions)} " + f"{missing_sessions}, and `config.allow_missing_sessions` is False" ) - if missing_sessions and not config.allow_missing_sessions: - raise RuntimeError( - f"Subject {subject} is missing session{_pl(missing_sessions)} " - f"{missing_sessions}, and `config.allow_missing_sessions` is False" - ) - keep_sessions = tuple(sorted(set(cfg_sessions) & set(valid_sessions_subj))) # type: ignore + keep_sessions = tuple(sorted(set(cfg_sessions) & set(valid_sessions_subj))) if len(keep_sessions): subj_sessions[subject] = keep_sessions return subj_sessions From 1b9546599d6fbd22b30bde70ec6a4f6f4c0d1b14 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 10 Jan 2025 16:00:39 -0600 Subject: [PATCH 07/12] fix warning expectation --- mne_bids_pipeline/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 84e3e7d65..108b26ea4 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -258,7 +258,7 @@ def test_missing_sessions( context = ( nullcontext() if allow_missing_sessions - else pytest.raises(RuntimeError, match=r"Subject 1 is missing session \('b',\)") + else pytest.raises(RuntimeError, match=r"Subject 1 is missing session \['b'\]") ) # run command = [ From 559c4acb5fce8dea84dabd4fab9cb775f66d62b7 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 24 Jan 2025 12:06:26 -0600 Subject: [PATCH 08/12] use inspect instead of pinning --- mne_bids_pipeline/_config_import.py | 12 ++++++++++++ mne_bids_pipeline/_config_utils.py | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/_config_import.py b/mne_bids_pipeline/_config_import.py index 2b34f7894..a480c1e25 100644 --- a/mne_bids_pipeline/_config_import.py +++ b/mne_bids_pipeline/_config_import.py @@ -6,12 +6,14 @@ import pathlib from dataclasses import field from functools import partial +from inspect import signature from types import SimpleNamespace from typing import Any import matplotlib import mne import numpy as np +from mne_bids import get_entity_vals from pydantic import BaseModel, ConfigDict, ValidationError from ._logging import gen_log_kwargs, logger @@ -324,6 +326,16 @@ def _check_config(config: SimpleNamespace, config_path: PathLike | None) -> None "Please set process_empty_room = True" ) + if ( + config.allow_missing_sessions + and "ignore_suffixes" not in signature(get_entity_vals).parameters + ): + raise ConfigError( + "You've requested to `allow_missing_sessions`, but this functionality " + "requires a newer version of `mne_bids` than you have available. Please " + "update MNE-BIDS (or if on the latest version, install the dev version)." + ) + bl = config.baseline if bl is not None: if (bl[0] is not None and bl[0] < config.epochs_tmin) or ( diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index bb784bf20..1c4a9a235 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -4,6 +4,7 @@ import functools import pathlib from collections.abc import Iterable, Sized +from inspect import signature from types import ModuleType, SimpleNamespace from typing import Any, Literal, TypeVar @@ -165,14 +166,19 @@ def get_subjects_sessions( # loop over subjs and check for available sessions subj_sessions: dict[str, tuple[None] | tuple[str, ...]] = dict() + kwargs = ( + dict(ignore_suffixes=("scans", "coordsystem")) + if "ignore_suffixes" in signature(mne_bids.get_entity_vals).parameters + else dict() + ) for subject in subjects: valid_sessions_subj = _get_entity_vals_cached( config.bids_root / f"sub-{subject}", entity_key="session", ignore_tasks=ignore_tasks, ignore_acquisitions=("calibration", "crosstalk"), - ignore_suffixes=("scans", "coordsystem"), ignore_datatypes=ignore_datatypes, + **kwargs, ) missing_sessions = sorted(set(cfg_sessions) - set(valid_sessions_subj)) if missing_sessions and not config.allow_missing_sessions: From 5987073c68e3805c9fff20a6c42d13be03fa292d Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 24 Jan 2025 12:06:52 -0600 Subject: [PATCH 09/12] Revert "add the temporary pin" This reverts commit edbaf61531b96c89ee8f2cf2df9fb6a1a7d7c686. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d2e8b7ee..05992b042 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ "openpyxl", "autoreject", "mne[hdf5] >=1.7", - "mne-bids[full] @ git+https://github.com/mne-tools/mne-bids@41ce36a4dc938e85a2f6a32ed3eeb88c732bfbae", + "mne-bids[full]", "filelock", ] dynamic = ["version"] From ef35ec17470cf6717e8cace7fba00278346b7c39 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 24 Jan 2025 12:07:09 -0600 Subject: [PATCH 10/12] Revert "allow direct references (temporary, for CI)" This reverts commit 7a71e495606f76773530460efbb98539a4184424. --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 05992b042..be1b411ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,9 +103,6 @@ exclude = [ "ignore_words.txt", ] -[tool.hatch.metadata] -allow-direct-references = true - [tool.codespell] skip = "docs/site/*,*.html,steps/freesurfer/contrib/*" ignore-words = "ignore_words.txt" From dd0dc6fa80b36e04908e60bbf62933a0a2dafda1 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 24 Jan 2025 14:50:00 -0600 Subject: [PATCH 11/12] use mne-bids main in circleCI --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 049b271ef..b824df64b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: command: | pip install --upgrade --progress-bar off pip # TODO: Restore https://api.github.com/repos/mne-tools/mne-bids/zipball/main pending https://github.com/mne-tools/mne-bids/pull/1349/files#r1885104885 - pip install --upgrade --progress-bar off "autoreject @ https://api.github.com/repos/autoreject/autoreject/zipball/master" "mne[hdf5] @ git+https://github.com/mne-tools/mne-python@main" "mne-bids[full]" numba + pip install --upgrade --progress-bar off "autoreject @ https://api.github.com/repos/autoreject/autoreject/zipball/master" "mne[hdf5] @ git+https://github.com/mne-tools/mne-python@main" "mne-bids[full] @ git+https://github.com/mne-tools/mne-bids@main" numba pip install -ve .[tests] pip install "PyQt6!=6.6.1" "PyQt6-Qt6!=6.6.1,!=6.6.2,!=6.6.3,!=6.7.0" - run: From f071d24006174b5f2a0b990c3ff6d2c2dd0dd3f6 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 24 Jan 2025 14:59:13 -0600 Subject: [PATCH 12/12] mne_bids main in GHA workflows too --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 703c980ef..d4570e352 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -22,7 +22,7 @@ jobs: with: python-version: "3.12" - run: pip install --upgrade pip - - run: pip install -ve .[tests] codespell tomli --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr" + - run: pip install -ve .[tests] "mne-bids[full] @ git+https://github.com/mne-tools/mne-bids@main" codespell tomli --only-binary="numpy,scipy,pandas,matplotlib,pyarrow,numexpr" - run: make codespell-error - run: pytest mne_bids_pipeline -m "not dataset_test" - uses: codecov/codecov-action@v5