Merge develop into unify-split-logic

Author: bruAristimunha

docs/source/whats_new.rst

@@ -39,12 +39,14 @@ API changes
 - When CodeCarbon is installed, MOABB HDF5 results have an additional column `codecarbon_task_name`. If CodeCarbon is configured to save to file, its own tabular results have a column `task_name`. These columns are unique UUID4s. Related rows can be joined to see detailed costs and benefits of predictive performance and computing profiling metrics (:gh:`866` by `Ethan Davis`_).
 - Isolated model fitting, duration tracking, and CodeCarbon compute profiling tracking. New and consistent ordering of duration and CodeCarbon tracking across all evaluations: (Higher priority, closest to model fitting) required duration tracking, (lower priority, second closest to model fitting) optional CodeCarbon tracking (:gh:`866` by `Ethan Davis`_).
 - Replaced unreliable wall clock duration tracking (Python's `time.time()`) in favor of performance counter duration tracking (Python's `time.perf_counter()`) (:gh:`866` by `Ethan Davis`_).
+- Enable choice of online or offline CodeCarbon through the parameterization of `codecarbon_config` when instantiating a :class:`moabb.evaluations.base.BaseEvaluation` child class (:gh:`956` by `Ethan Davis`_)
 - Renamed stimulus channel from ``stim`` to ``STI`` in BNCI motor imagery and error-related potential datasets for clarity and BIDS compliance (by `Bruno Aristimunha`_).
 - Added four new BNCI P300/ERP dataset classes: :class:`moabb.datasets.BNCI2015_009` (AMUSE), :class:`moabb.datasets.BNCI2015_010` (RSVP), :class:`moabb.datasets.BNCI2015_012` (PASS2D), and :class:`moabb.datasets.BNCI2015_013` (ErrP) (by `Bruno Aristimunha`_).
 
 Requirements
 ~~~~~~~~~~~~
-- Requires CodeCarbon environment variables or a configuration file to be defined in the home directory or the current working directory (:gh:`866` by `Ethan Davis`_).
+- Allows CodeCarbon environment variables or a configuration file to be defined in the home directory or the current working directory (:gh:`866` by `Ethan Davis`_).
+- Added ``filelock`` as a core dependency to fix missing import errors in utils (:gh:`959` by `Mateusz Naklicki`_).
 
 Bugs
 ~~~~
@@ -56,6 +58,7 @@ Bugs
 - Fixing option to pickle model (:gh:`870` by `Ethan Davis`_)
 - Normalize Zenodo download paths and add a custom user-agent to improve download robustness (:gh:`946` by `Bruno Aristimunha`_)
 - Use the BNCI mirror host to avoid download timeouts (:gh:`946` by `Bruno Aristimunha`_)
+- Prevent Python mutable default argument when defining CodeCarbon configurations (:gh:`956` by `Ethan Davis`_)
 - Fix copytree FileExistsError in BrainInvaders2013a download by adding dirs_exist_ok=True (by `Bruno Aristimunha`_)
 - Ensure optional additional scoring columns in evaluation results (:gh:`957` by `Ethan Davis`_)
 
@@ -709,3 +712,4 @@ API changes
 .. _Romani Michele: https://github.com/BRomans
 .. _Lionel Kusch: https://github.com/lionelkusch
 .. _Victor Martinez-Cagigal: https://github.com/vicmarcag
+.. _Mateusz Naklicki: https://github.com/luluu9

moabb/evaluations/base.py

@@ -9,6 +9,7 @@
 from moabb.analysis import Results
 from moabb.datasets.base import BaseDataset
 from moabb.evaluations.utils import (
+    Emissions,
     _convert_sklearn_params_to_optuna,
     _create_scorer,
     _DictScorer,
@@ -113,7 +114,7 @@ def __init__(
         optuna=False,
         time_out=60 * 15,
         verbose=None,
-        codecarbon_config=dict(save_to_file=False, log_level="error"),
+        codecarbon_config=None,
     ):
         self.random_state = random_state
         self.n_jobs = n_jobs
@@ -128,7 +129,7 @@ def __init__(
         self.optuna = optuna
         self.time_out = time_out
         self.verbose = verbose
-        self.codecarbon_config = codecarbon_config
+        self.emissions = Emissions(codecarbon_config=codecarbon_config)
 
         self.additional_columns = additional_columns
         if additional_columns is None:

moabb/evaluations/evaluations.py

@@ -47,7 +47,7 @@ def _pipeline_requires_epochs(pipeline):
 
 
 try:
-    from codecarbon import EmissionsTracker
+    from codecarbon import EmissionsTracker  # noqa
 
     _carbonfootprint = True
 except ImportError:
@@ -265,7 +265,8 @@ def _evaluate(
                     emissions = np.nan
                     task_name = ""
                     if _carbonfootprint:
-                        tracker = EmissionsTracker(**self.codecarbon_config)
+                        # Initialise CodeCarbon per cross-validation
+                        tracker = self.emissions.create_tracker()
                         tracker.start()
 
                     # Create scorer once before CV loop
@@ -366,11 +367,11 @@ def _evaluate(
                             X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1]
                         )
                         res = {
-                            "time": duration / self.n_splits,
+                            "time": duration / self.n_splits,  # 5 fold CV
                             "dataset": dataset,
                             "subject": subject,
                             "session": session,
-                            "n_samples": len(y_cv),
+                            "n_samples": len(y_cv),  # not training sample
                             "n_channels": nchan,
                             "pipeline": name,
                         }
@@ -384,6 +385,61 @@ def _evaluate(
 
                         yield res
 
+    def get_data_size_subsets(self, y):
+        if self.data_size is None:
+            raise ValueError(
+                "Cannot create data subsets without valid policy for data_size."
+            )
+        if self.data_size["policy"] == "ratio":
+            vals = np.array(self.data_size["value"])
+            if np.any(vals < 0) or np.any(vals > 1):
+                raise ValueError("Data subset ratios must be in range [0, 1]")
+            upto = np.ceil(vals * len(y)).astype(int)
+            indices = [np.array(range(i)) for i in upto]
+        elif self.data_size["policy"] == "per_class":
+            classwise_indices = dict()
+            n_smallest_class = np.inf
+            for cl in np.unique(y):
+                cl_i = np.where(cl == y)[0]
+                classwise_indices[cl] = cl_i
+                n_smallest_class = (
+                    len(cl_i) if len(cl_i) < n_smallest_class else n_smallest_class
+                )
+            indices = []
+            for ds in self.data_size["value"]:
+                if ds > n_smallest_class:
+                    raise ValueError(
+                        f"Smallest class has {n_smallest_class} samples. "
+                        f"Desired samples per class {ds} is too large."
+                    )
+                indices.append(
+                    np.concatenate(
+                        [classwise_indices[cl][:ds] for cl in classwise_indices]
+                    )
+                )
+        else:
+            raise ValueError(f"Unknown policy {self.data_size['policy']}")
+        return indices
+
+    def score_explicit(self, res, clf, X_train, y_train, X_test, y_test):
+        """Fit model and update result dict with scores and duration."""
+        if not self.mne_labels:
+            # convert labels if array, keep them if epochs and mne_labels is set
+            le = LabelEncoder()
+            y_train = le.fit_transform(y_train)
+            y_test = le.transform(y_test)
+        t_start = perf_counter()
+        try:
+            model = clf.fit(X_train, y_train)
+            _ensure_fitted(model)
+            scorer = _create_scorer(model, self.paradigm.scoring)
+            _score_and_update(res, scorer, model, X_test, y_test)
+        except ValueError as e:
+            if self.error_score == "raise":
+                raise e
+            res["score"] = self.error_score
+        res["time"] = perf_counter() - t_start
+
     def evaluate(
         self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None
     ):
@@ -497,7 +553,7 @@ def evaluate(
 
                 if _carbonfootprint:
                     # Initialise CodeCarbon per cross-validation
-                    tracker = EmissionsTracker(**self.codecarbon_config)
+                    tracker = self.emissions.create_tracker()
                     tracker.start()
 
                 # Create scorer once before CV loop
@@ -684,7 +740,7 @@ def evaluate(
 
         if _carbonfootprint:
             # Initialise CodeCarbon per cross-validation
-            tracker = EmissionsTracker(**self.codecarbon_config)
+            tracker = self.emissions.create_tracker()
             tracker.start()
 
         # Progressbar at subject level

moabb/evaluations/utils.py

@@ -25,6 +25,14 @@
     optuna_available = False
 
 
+try:
+    from codecarbon import EmissionsTracker, OfflineEmissionsTracker
+
+    _carbonfootprint = True
+except ImportError:
+    _carbonfootprint = False
+
+
 def _ensure_fitted(estimator):
     """Ensure an estimator is properly marked as fitted for sklearn 1.8+.
 
@@ -462,3 +470,34 @@ def _score_and_update(res, scorer, model, X, y_true):
     """
     score = scorer(model, X, y_true)
     return _update_result_with_scores(res, score)
+
+
+class Emissions:
+    def __init__(self, codecarbon_config=None):
+        self.codecarbon_config = codecarbon_config
+        if codecarbon_config is None:
+            # Default CodeCarbon configurations
+            self.codecarbon_config = dict(save_to_file=False, log_level="error")
+            self.codecarbon_offline = False
+        else:
+            # Offline mode parameters are a superset of online mode parameters
+            # Hardcode check avoids object reflection for security and compatibility
+            # For more information see CodeCarbon documentation
+            # https://mlco2.github.io/codecarbon/parameters.html#specific-parameters-for-offline-mode
+            offline_params = [
+                "country_iso_code",
+                "region",
+                "cloud_provider",
+                "cloud_region",
+                "country_2letter_iso_code",
+            ]
+            self.codecarbon_offline = any(
+                key in self.codecarbon_config for key in offline_params
+            )
+
+    def create_tracker(self):
+        if self.codecarbon_offline:
+            tracker = OfflineEmissionsTracker(**self.codecarbon_config)
+        else:
+            tracker = EmissionsTracker(**self.codecarbon_config)
+        return tracker

moabb/tests/test_emissions.py

@@ -0,0 +1,149 @@
+"""Tests for the Emissions class in evaluations/utils.py."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from moabb.evaluations.utils import Emissions
+
+
+try:
+    from codecarbon import EmissionsTracker, OfflineEmissionsTracker  # noqa
+
+    CODECARBON_AVAILABLE = True
+except ImportError:
+    CODECARBON_AVAILABLE = False
+
+
+def test_default_config():
+    """Test Emissions initialization with default config (None)."""
+    emissions = Emissions()
+
+    # Check default configuration is set
+    assert emissions.codecarbon_config == dict(save_to_file=False, log_level="error")
+    # Check codecarbon_offline is set to False for default config
+    assert emissions.codecarbon_offline is False
+
+
+def test_custom_config_online_mode():
+    """Test Emissions initialization with custom config (online mode)."""
+    custom_config = {
+        "save_to_file": True,
+        "log_level": "info",
+        "project_name": "test_project",
+    }
+    emissions = Emissions(codecarbon_config=custom_config)
+
+    # Check custom configuration is set
+    assert emissions.codecarbon_config == custom_config
+    # Check codecarbon_offline is False when no offline params are present
+    assert emissions.codecarbon_offline is False
+
+
+@pytest.mark.parametrize(
+    "offline_param,param_value",
+    [
+        ("country_iso_code", "USA"),
+        ("region", "us-west-1"),
+        ("cloud_provider", "aws"),
+        ("cloud_region", "us-west-1"),
+        ("country_2letter_iso_code", "US"),
+    ],
+)
+def test_custom_config_offline_mode(offline_param, param_value):
+    """Test Emissions initialization with various offline parameters."""
+    custom_config = {
+        "save_to_file": False,
+        "log_level": "error",
+        offline_param: param_value,
+    }
+    emissions = Emissions(codecarbon_config=custom_config)
+
+    # Check custom configuration is set
+    assert emissions.codecarbon_config == custom_config
+    # Check codecarbon_offline is True when offline param is present
+    assert emissions.codecarbon_offline is True
+
+
+def test_custom_config_offline_mode_multiple_params():
+    """Test Emissions initialization with multiple offline parameters."""
+    custom_config = {
+        "save_to_file": True,
+        "log_level": "info",
+        "country_iso_code": "USA",
+        "region": "california",
+        "cloud_provider": "aws",
+    }
+    emissions = Emissions(codecarbon_config=custom_config)
+
+    assert emissions.codecarbon_config == custom_config
+    assert emissions.codecarbon_offline is True
+
+
+@pytest.mark.skipif(not CODECARBON_AVAILABLE, reason="codecarbon not installed")
+@patch("moabb.evaluations.utils.EmissionsTracker")
+def test_create_tracker_default_config(mock_emissions_tracker):
+    """Test create_tracker with default config uses EmissionsTracker."""
+    mock_tracker = MagicMock()
+    mock_emissions_tracker.return_value = mock_tracker
+
+    emissions = Emissions()
+    tracker = emissions.create_tracker()
+
+    # Verify EmissionsTracker was called with correct config
+    mock_emissions_tracker.assert_called_once_with(save_to_file=False, log_level="error")
+    assert tracker == mock_tracker
+
+
+@pytest.mark.skipif(not CODECARBON_AVAILABLE, reason="codecarbon not installed")
+@patch("moabb.evaluations.utils.EmissionsTracker")
+def test_create_tracker_online_mode(mock_emissions_tracker):
+    """Test create_tracker with online config uses EmissionsTracker."""
+    mock_tracker = MagicMock()
+    mock_emissions_tracker.return_value = mock_tracker
+
+    custom_config = {"save_to_file": True, "log_level": "info"}
+    emissions = Emissions(codecarbon_config=custom_config)
+    tracker = emissions.create_tracker()
+
+    # Verify EmissionsTracker was called with custom config
+    mock_emissions_tracker.assert_called_once_with(save_to_file=True, log_level="info")
+    assert tracker == mock_tracker
+
+
+@pytest.mark.skipif(not CODECARBON_AVAILABLE, reason="codecarbon not installed")
+@patch("moabb.evaluations.utils.OfflineEmissionsTracker")
+def test_create_tracker_offline_mode(mock_offline_emissions_tracker):
+    """Test create_tracker with offline config uses OfflineEmissionsTracker."""
+    mock_tracker = MagicMock()
+    mock_offline_emissions_tracker.return_value = mock_tracker
+
+    custom_config = {
+        "save_to_file": False,
+        "log_level": "error",
+        "country_iso_code": "USA",
+    }
+    emissions = Emissions(codecarbon_config=custom_config)
+    tracker = emissions.create_tracker()
+
+    # Verify OfflineEmissionsTracker was called with custom config
+    mock_offline_emissions_tracker.assert_called_once_with(
+        save_to_file=False, log_level="error", country_iso_code="USA"
+    )
+    assert tracker == mock_tracker
+
+
+@pytest.mark.parametrize(
+    "config,expected_offline",
+    [
+        (None, False),
+        ({}, False),
+        ({"save_to_file": False}, False),
+        ({"country_iso_code": "USA"}, True),
+    ],
+)
+def test_codecarbon_offline_attribute_always_exists(config, expected_offline):
+    """Test that codecarbon_offline attribute always exists regardless of config."""
+    emissions = Emissions(codecarbon_config=config)
+    assert hasattr(emissions, "codecarbon_offline")
+    assert emissions.codecarbon_offline == expected_offline

pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
     "memory-profiler>=0.61.0",
     "edflib-python>=1.0.6",
     "edfio>=0.4.2",
+    "filelock>=3.18.0",
     "pytest>=8.3.5",
     "mne-bids>=0.16",
     "scikit-learn>=1.6",