Fix multi-subject CrossSubject GroupKFold result attribution

docs/source/whats_new.rst

@@ -35,6 +35,7 @@ Requirements
 
 Bugs
 ~~~~
+- Fix cross-subject ``GroupKFold`` folds that hold out multiple subjects: the parallel evaluator now keeps cache work plans per held-out subject, scores each ``(subject, session)`` independently, and saves the fitted fold model under every subject that still needs the result instead of attributing the whole fold to its first metadata row (:gh:`1212` by `lindicaphxag-tech`_).
 - Use ``gmean`` in TRCA and TRCSP for compatibility with pyRiemann 0.12 and 0.13, and pass the TRCSP mean metric by keyword (by `Bruno Aristimunha`_).
 - Fix the ``-e``/``--evaluations`` flag of ``python -m moabb.run``, which used ``type=list`` and so split its value into single characters: ``-e WithinSession`` reached :func:`moabb.benchmark` as ``['W', 'i', 't', ...]`` and raised ``KeyError: 'W'``. It now takes one or more evaluation names, space separated (by `Iain`_)
 - Fix the two install pages asking for optional extras MOABB does not have: the pip install page gave ``pip install moabb[deepleaning,carbonemission,docs]``, which is missing the ``r`` of ``deeplearning``, and pip only warns about an unrecognised extra, so following that page left ``braindecode`` uninstalled. The from-sources page asked for ``external``, removed in 1.2.0 (by `Iain`_).
@@ -1063,3 +1064,5 @@ API changes
 .. _Iain: https://github.com/NotAFlightRisk
 .. _Anna Sokolova: https://github.com/ZyntZ
 .. _Arthur031221: https://github.com/Arthur031221
+
+.. _lindicaphxag-tech: https://github.com/lindicaphxag-tech

moabb/evaluations/base.py

@@ -139,6 +139,7 @@ def _evaluate_fold(
     cv_ind,
     split_metadata=None,
     calib_idx=None,
+    score_subjects=None,
 ):
     """Evaluate a single CV fold. Pure function, no shared mutable state.
 
@@ -166,6 +167,9 @@ def _evaluate_fold(
         Cross-validation fold index.
     split_metadata : dict | None
         Extra metadata from the splitter.
+    score_subjects : sequence or None
+        Held-out subjects for which this pipeline still needs result rows. If
+        None, all subjects present in the test fold are scored.
 
     Returns
     -------
@@ -257,35 +261,58 @@ def _evaluate_fold(
     if tracker is not None:
         tracker.stop()
 
-    # Optionally save model
+    test_metadata = metadata.iloc[test_idx]
+    test_subjects = test_metadata["subject"].to_numpy()
+    if score_subjects is None:
+        score_subjects = list(pd.unique(test_subjects))
+    else:
+        score_subjects = list(score_subjects)
+
+    # Optionally save the fitted fold model under every held-out subject that
+    # still needs this pipeline. GroupKFold may hold out several subjects even
+    # though the estimator is fitted only once for the fold.
     hdf5_path = config["hdf5_path"]
     eval_type = config["eval_type"]
     if hdf5_path is not None and config["save_model"]:
-        model_save_path = _create_save_path(
-            hdf5_path=hdf5_path,
-            code=dataset.code,
-            subject=subject,
-            session="" if score_per_session else session,
-            name=pipeline_name,
-            grid=is_search,
-            eval_type=eval_type,
-        )
-        _save_model_cv(model=cvclf, save_path=model_save_path, cv_index=str(cv_ind))
+        save_subjects = score_subjects if score_per_session else [subject]
+        for save_subject in save_subjects:
+            model_save_path = _create_save_path(
+                hdf5_path=hdf5_path,
+                code=dataset.code,
+                subject=save_subject,
+                session="" if score_per_session else session,
+                name=pipeline_name,
+                grid=is_search,
+                eval_type=eval_type,
+            )
+            _save_model_cv(model=cvclf, save_path=model_save_path, cv_index=str(cv_ind))
 
     scorer = None if trialwise else _create_scorer(cvclf, scoring)
 
-    # Build score groups: per-session or full test set
+    # Score every held-out subject/session independently. Grouping on session
+    # alone mixes subjects that share labels such as "0" or "1".
     if score_per_session:
-        test_sessions = metadata.iloc[test_idx]["session"].values
-        score_groups = [
-            (test_idx[test_sessions == s], y_test[test_sessions == s], s)
-            for s in np.unique(test_sessions)
-        ]
+        test_sessions = test_metadata["session"].to_numpy()
+        score_groups = []
+        for group_subject in score_subjects:
+            subject_mask = test_subjects == group_subject
+            for group_session in pd.unique(test_sessions[subject_mask]):
+                mask = subject_mask & (test_sessions == group_session)
+                positions = np.flatnonzero(mask)
+                if positions.size:
+                    score_groups.append(
+                        (
+                            test_idx[positions],
+                            y_test[positions],
+                            group_subject,
+                            group_session,
+                        )
+                    )
     else:
-        score_groups = [(test_idx, y_test, session)]
+        score_groups = [(test_idx, y_test, subject, session)]
 
     results = []
-    for group_idx, group_y, group_session in score_groups:
+    for group_idx, group_y, group_subject, group_session in score_groups:
         is_error = False
         try:
             if trialwise:
@@ -301,7 +328,7 @@ def _evaluate_fold(
         res = {
             "time": duration,
             "dataset": dataset,
-            "subject": subject,
+            "subject": group_subject,
             "session": group_session,
             "n_samples": len(train_idx),
             "n_samples_test": len(group_y),
@@ -782,14 +809,23 @@ def _build_task_list(
 
         for cv_ind, train_idx, calib_idx, test_idx, split_meta in fold_preview:
             test_meta = metadata.iloc[test_idx]
-            subject = test_meta["subject"].iloc[0]
-
-            if subject not in work_plan:
-                continue
-            run_pipes = work_plan[subject]
+            held_out_subjects = list(pd.unique(test_meta["subject"]))
             session = test_meta["session"].iloc[0]
 
-            for name, clf in run_pipes.items():
+            # A cross-subject GroupKFold test fold can contain several subjects.
+            # Keep one fit task per pipeline, but retain exactly which held-out
+            # subjects still need result rows for that pipeline.
+            for name in pipelines:
+                score_subjects = [
+                    held_out_subject
+                    for held_out_subject in held_out_subjects
+                    if name in work_plan.get(held_out_subject, {})
+                ]
+                if not score_subjects:
+                    continue
+
+                subject = score_subjects[0]
+                clf = work_plan[subject][name]
                 task_config = dict(config)
                 if param_grid is not None and name in param_grid:
                     task_param_grid = {name: deepcopy(param_grid[name])}
@@ -809,6 +845,7 @@ def _build_task_list(
                         "cv_ind": cv_ind,
                         "split_metadata": split_meta,
                         "calib_idx": calib_idx,
+                        "score_subjects": score_subjects,
                     }
                 )
         return tasks

moabb/tests/test_evaluations.py

@@ -24,7 +24,7 @@
 from moabb.datasets.compound_dataset import compound
 from moabb.datasets.fake import FakeDataset
 from moabb.evaluations import evaluations as ev
-from moabb.evaluations.base import BaseEvaluation, optuna_available
+from moabb.evaluations.base import BaseEvaluation, _evaluate_fold, optuna_available
 from moabb.evaluations.splitters import LearningCurveSplitter
 from moabb.evaluations.utils import _create_save_path as create_save_path
 from moabb.evaluations.utils import _pipeline_requires_epochs
@@ -1060,6 +1060,114 @@ def test_cross_subject_evaluation_materializes_callable_test_fold(
     assert len(train) + len(test) == len(metadata)
 
 
+def test_cross_subject_group_kfold_scores_subject_sessions_separately(tmp_path):
+    """A multi-subject test fold must not merge equal session labels."""
+    X = np.arange(16, dtype=float).reshape(8, 2)
+    y = np.array([0, 1, 0, 1, 0, 1, 1, 0])
+    metadata = pd.DataFrame(
+        {
+            "subject": [1, 1, 2, 2, 3, 3, 4, 4],
+            "session": ["0", "1", "0", "1", "0", "1", "0", "1"],
+        }
+    )
+    dataset_multi = FakeDataset(
+        ["left_hand", "right_hand"], n_subjects=4, n_sessions=2, seed=42
+    )
+    config = {
+        "scoring": "accuracy",
+        "error_score": "raise",
+        "random_state": 42,
+        "param_grid": None,
+        "additional_columns": [],
+        "score_per_session": True,
+        "mne_labels": False,
+        "codecarbon_config": None,
+        "trialwise": False,
+        "n_jobs_grid": 1,
+        "optuna": False,
+        "time_out": 60,
+        "hdf5_path": str(tmp_path),
+        "eval_type": "CrossSubject",
+        "save_model": True,
+    }
+
+    results = _evaluate_fold(
+        X,
+        y,
+        metadata,
+        config,
+        dataset_multi,
+        "dummy",
+        Dummy(strategy="most_frequent"),
+        train_idx=np.array([0, 1, 2, 3]),
+        test_idx=np.array([4, 5, 6, 7]),
+        subject=3,
+        session="0",
+        cv_ind=0,
+        score_subjects=[3, 4],
+    )
+
+    assert {(r["subject"], r["session"]) for r in results} == {
+        (3, "0"),
+        (3, "1"),
+        (4, "0"),
+        (4, "1"),
+    }
+    assert all(r["n_samples_test"] == 1 for r in results)
+    for held_out_subject in (3, 4):
+        model_path = (
+            tmp_path
+            / "Models_CrossSubject"
+            / dataset_multi.code
+            / str(held_out_subject)
+            / "dummy"
+            / "fitted_model_0.pkl"
+        )
+        assert model_path.is_file()
+
+
+def test_cross_subject_task_builder_uses_all_held_out_subject_work_plans(tmp_path):
+    """A cached first subject must not suppress another subject in the same fold."""
+    dataset_multi = FakeDataset(
+        ["left_hand", "right_hand"], n_subjects=4, n_sessions=2, seed=42
+    )
+    evaluation = ev.CrossSubjectEvaluation(
+        paradigm=FakeImageryParadigm(),
+        datasets=[dataset_multi],
+        hdf5_path=tmp_path,
+        n_splits=2,
+    )
+    metadata = pd.DataFrame(
+        {"subject": np.repeat([1, 2, 3, 4], 2), "session": ["0", "1"] * 4}
+    )
+    y = np.array([0, 1] * 4)
+    splitter = evaluation._create_splitter()
+    folds = list(splitter.split(y, metadata))
+    held_out = list(pd.unique(metadata.iloc[folds[0][1]]["subject"]))
+    assert len(held_out) == 2
+
+    # Only the second subject in this fold still needs the pipeline. The old
+    # builder inspected held_out[0] only and skipped the whole fold.
+    target_subject = held_out[1]
+    pipeline = Dummy(strategy="most_frequent")
+    work_plan = {target_subject: {"dummy": pipeline}}
+    tasks = evaluation._build_task_list(
+        dataset_multi,
+        None,
+        y,
+        metadata,
+        evaluation._create_splitter(),
+        work_plan,
+        {"dummy": pipeline},
+        None,
+    )
+
+    assert len(tasks) == 1
+    assert tasks[0]["subject"] == target_subject
+    assert tasks[0]["score_subjects"] == [target_subject]
+    assert target_subject in set(metadata.iloc[tasks[0]["test_idx"]]["subject"])
+
+
 class Test_CrossSubj(TestWithinSess):
     def setup_method(self):
         self.eval = ev.CrossSubjectEvaluation(