Add BIDS-first configuration schema and update related scripts

snesmaeili · snesmaeili · commit 70350aea3a9b · 2025-09-09T15:14:05.000-04:00
- Introduced `bidsfirst_config_schema.json` to define the configuration structure for BIDS-first processing of DSI-24 data.
- Updated `dsi_bids.py`, `event_mapping.py`, and `preprocess_single.py` to reference the new configuration schema.
- Added example configurations for Go/No-Go and Five-Point tasks in `configs/examples/`.
- Enhanced CI workflow to validate BIDS-first example configs.
- Updated `README.md` to include references to the new BIDS-first schema and examples.
- Introduced `export_repo_to_xml.py` for improved repository serialization and documentation.
- Revised `pyproject.toml` to manage project dependencies and development tools more effectively.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -26,7 +26,7 @@ jobs:
           pre-commit run --all-files
       - name: Run tests
         run: pytest -q
-      - name: Validate configs
+      - name: Validate pipeline configs (modular)
         run: |
           python - <<'PY'
           import json, sys, yaml
@@ -41,6 +41,9 @@ jobs:
               continue
             if 'drafts' in p.parts:
               continue
+            if 'examples' in p.parts:
+              # examples are BIDS-first configs validated in a separate step
+              continue
             try:
               if p.suffix.lower() == '.json':
                 cfg = json.loads(p.read_text())
@@ -59,3 +62,37 @@ jobs:
                 print(f" - {loc}: {err.message}")
           sys.exit(0 if ok else 1)
           PY
+
+      - name: Validate BIDS-first example configs
+        run: |
+          python - <<'PY'
+          import json, sys
+          from pathlib import Path
+          from jsonschema import Draft7Validator
+          schema_path = Path('bidsfirst_config_schema.json')
+          if not schema_path.exists():
+            print('[skip] bidsfirst_config_schema.json not found')
+            raise SystemExit(0)
+          schema = json.loads(schema_path.read_text())
+          validator = Draft7Validator(schema)
+          ok = True
+          ex_dir = Path('configs/examples')
+          if not ex_dir.exists():
+            print('[skip] configs/examples not present')
+            raise SystemExit(0)
+          for p in ex_dir.rglob('*.json'):
+            try:
+              cfg = json.loads(p.read_text())
+            except Exception as e:
+              print(f"[invalid parse] {p}: {e}")
+              ok = False
+              continue
+            errors = sorted(validator.iter_errors(cfg), key=lambda e: e.path)
+            if errors:
+              ok = False
+              print(f"[schema errors] {p}")
+              for err in errors:
+                loc = "/".join(str(x) for x in err.path) or '<root>'
+                print(f" - {loc}: {err.message}")
+          sys.exit(0 if ok else 1)
+          PY
diff --git a/README.md b/README.md
@@ -79,6 +79,11 @@ This repository is designed for both single-subject and multi-subject pipelines,
     # optional: dev tools
     pip install -r requirements-dev.txt       # adds pytest, black, flake8, etc.
     ```
+   Or install as a package for a cleaner setup:
+   ```bash
+   pip install -e .            # users
+   pip install -e .[dev]       # contributors (adds tests/docs tools)
+   ```
 3. **Supported Python**:
    - We test on Python 3.10–3.12 (recommended with MNE 1.5+).
 ## Usage Examples
@@ -266,6 +271,9 @@ python preprocess_batch.py \
   --task gonogo --n-jobs 1
 ```
 
-See `configs/examples/` for Go/No‑Go and Five‑Point examples, and `config_schema.json` for the schema driving both BIDSify and preprocessing.
+See `configs/examples/` for Go/No‑Go and Five‑Point examples.
+Schemas:
+- BIDS‑first examples: `bidsfirst_config_schema.json`
+- Modular pipeline: `scr/config_schema.json`
 
 
diff --git a/bidsfirst_config_schema.json b/bidsfirst_config_schema.json
diff --git a/configs/examples/fivepoint.json b/configs/examples/fivepoint.json
@@ -0,0 +1,51 @@
+{
+  "project": {"name": "neurotheque-examples", "version": "0.1.0"},
+  "paths": {
+    "source_root": "./data/input",
+    "bids_root": "./derivatives/bids",
+    "derivatives_root": null,
+    "overwrite": false
+  },
+  "bids": {
+    "manufacturer": "Wearable Sensing",
+    "manufacturersModelName": "DSI-24",
+    "line_freq": 60,
+    "montage": "standard_1020",
+    "eeg_reference": "average",
+    "dataset_description": {"Name": "DSI-24 Example", "BIDSVersion": "1.8.0"}
+  },
+  "filename_parsing": {
+    "patterns": [
+      "(?P<subject>\\d+)[_-](?P<session>\\d+).*?(?P<task>5pt).*?(?P<run>\\d+)",
+      "sub[-_](?P<subject>\\d+).*?ses[-_](?P<session>\\d+).*?task[-_](?P<task>[A-Za-z0-9]+).*?run[-_](?P<run>\\d+)"
+    ],
+    "fallbacks": {"subject": "01", "session": "001", "task": "5pt", "run": "01"}
+  },
+  "events": {
+    "trigger_channel": "Trigger",
+    "default_task": "5pt",
+    "tasks": {
+      "5pt": {
+        "event_map": {"stim": [1], "response": [2]},
+        "onset_shift_sec": 0.0,
+        "debounce_ms": 5.0,
+        "epoching": {"tmin": -0.2, "tmax": 0.8, "baseline": [null, 0.0], "preload": true, "reject_by_annotation": true}
+      }
+    }
+  },
+  "preprocessing": {
+    "resample_sfreq": null,
+    "filter": {"l_freq": 1.0, "h_freq": 40.0},
+    "notch": {"freqs": [60, 120]},
+    "bad_channels": [],
+    "reref": "average",
+    "ica": {"method": "fastica", "n_components": "auto", "ecg": true, "eog": true, "reject_by_annotation": true},
+    "autoreject": {"enabled": false, "method": "local"},
+    "metrics": {"psd": true, "epochs_rejection": true},
+    "save_intermediates": true
+  },
+  "report": {"enabled": true, "title": "Neurothèque Preprocessing Report", "sections": ["psd", "ica", "epochs"], "include_logs": true},
+  "logging": {"level": "INFO", "json_log": true, "log_to_console": true},
+  "batch": {"mode": "single", "n_jobs": 1}
+}
+
diff --git a/configs/examples/gonogo.json b/configs/examples/gonogo.json
@@ -0,0 +1,52 @@
+{
+  "project": {"name": "neurotheque-examples", "version": "0.1.0"},
+  "paths": {
+    "source_root": "./data/input",
+    "bids_root": "./derivatives/bids",
+    "derivatives_root": null,
+    "overwrite": false
+  },
+  "bids": {
+    "manufacturer": "Wearable Sensing",
+    "manufacturersModelName": "DSI-24",
+    "line_freq": 60,
+    "montage": "standard_1020",
+    "eeg_reference": "average",
+    "dataset_description": {"Name": "DSI-24 Example", "BIDSVersion": "1.8.0"}
+  },
+  "filename_parsing": {
+    "patterns": [
+      "(?P<subject>\\d+)[_-](?P<session>\\d+).*?(?P<task>gonogo).*?(?P<run>\\d+)",
+      "sub[-_](?P<subject>\\d+).*?ses[-_](?P<session>\\d+).*?task[-_](?P<task>[A-Za-z0-9]+).*?run[-_](?P<run>\\d+)"
+    ],
+    "fallbacks": {"subject": "01", "session": "001", "task": "gonogo", "run": "01"}
+  },
+  "events": {
+    "trigger_channel": "Trigger",
+    "default_task": "gonogo",
+    "tasks": {
+      "gonogo": {
+        "event_map": {"go": [1], "nogo": [2], "response": [8]},
+        "aliases": {"G": 1, "N": 2},
+        "onset_shift_sec": 0.0,
+        "debounce_ms": 5.0,
+        "epoching": {"tmin": -0.2, "tmax": 0.8, "baseline": [null, 0.0], "preload": true, "reject_by_annotation": true}
+      }
+    }
+  },
+  "preprocessing": {
+    "resample_sfreq": null,
+    "filter": {"l_freq": 1.0, "h_freq": 40.0},
+    "notch": {"freqs": [60, 120]},
+    "bad_channels": [],
+    "reref": "average",
+    "ica": {"method": "fastica", "n_components": "auto", "ecg": true, "eog": true, "reject_by_annotation": true},
+    "autoreject": {"enabled": false, "method": "local"},
+    "metrics": {"psd": true, "epochs_rejection": true},
+    "save_intermediates": true
+  },
+  "report": {"enabled": true, "title": "Neurothèque Preprocessing Report", "sections": ["psd", "ica", "epochs"], "include_logs": true},
+  "logging": {"level": "INFO", "json_log": true, "log_to_console": true},
+  "batch": {"mode": "single", "n_jobs": 1}
+}
+
diff --git a/dsi_bids.py b/dsi_bids.py
@@ -409,7 +409,7 @@ def _iter_edfs(root: Path) -> Iterable[Path]:
 
 def main(argv: Optional[List[str]] = None) -> int:
     p = argparse.ArgumentParser(description="BIDSify a folder of DSI‑24 EDF files (JSON‑driven).")
-    p.add_argument("--config", required=True, help="Path to JSON config (see config_schema.json).")
+    p.add_argument("--config", required=True, help="Path to JSON config (see bidsfirst_config_schema.json).")
     p.add_argument("--source", required=True, help="Folder containing .edf files (recursed).")
     p.add_argument("--bids-root", required=True, help="Output BIDS root.")
     args = p.parse_args(argv)
@@ -452,4 +452,3 @@ def main(argv: Optional[List[str]] = None) -> int:
 
 if __name__ == "__main__":
     raise SystemExit(main())
-
diff --git a/event_mapping.py b/event_mapping.py
@@ -5,7 +5,7 @@
   - mapped events (with debounced / onset-shifted samples)
   - event_id {label -> code} suitable for MNE/MNE-BIDS
 
-Expected task JSON structure (per config_schema.json):
+Expected task JSON structure (per bidsfirst_config_schema.json):
 {
   "event_map": { "GO": [1], "NOGO": [2], "START": [7] },
   "aliases?": { "Start": 7 },
@@ -123,4 +123,3 @@ def map_events_from_config(
         ev_keep = ev_keep[np.argsort(ev_keep[:, 0])]
 
     return ev_keep, dict(event_id)
-
diff --git a/export_repo_to_xml.py b/export_repo_to_xml.py
@@ -0,0 +1,15 @@
+"""Compat wrapper for tools/dev/export_repo_to_xml.py.
+
+Allows calling from repo root:
+  python export_repo_to_xml.py --out repo.xml [--args]
+"""
+
+from __future__ import annotations
+
+import runpy
+from pathlib import Path
+
+
+if __name__ == "__main__":
+    script = Path(__file__).parent / "tools" / "dev" / "export_repo_to_xml.py"
+    runpy.run_path(str(script), run_name="__main__")
diff --git a/notebooks/preprocessing/gonogo_intermediate_preprocessing.ipynb b/notebooks/preprocessing/gonogo_intermediate_preprocessing.ipynb
@@ -1326,7 +1326,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "eeg_analysis",
+   "display_name": "neuropipe",
    "language": "python",
    "name": "python3"
   },
@@ -1340,7 +1340,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.11.11"
   }
  },
  "nbformat": 4,
diff --git a/preprocess_single.py b/preprocess_single.py
@@ -293,7 +293,7 @@ def preprocess_one(
 
 def main(argv: Optional[list[str]] = None) -> int:
     ap = argparse.ArgumentParser(description="Preprocess a single BIDS run (JSON-driven).")
-    ap.add_argument("--config", required=True, help="JSON config (see config_schema.json)")
+    ap.add_argument("--config", required=True, help="JSON config (see bidsfirst_config_schema.json)")
     ap.add_argument("--bids-root", required=True)
     ap.add_argument("--sub", required=True)
     ap.add_argument("--ses")
@@ -310,4 +310,3 @@ def main(argv: Optional[list[str]] = None) -> int:
 
 if __name__ == "__main__":
     raise SystemExit(main())
-
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,3 +22,36 @@ warn_return_any = true
 warn_unreachable = true
 no_implicit_optional = true
 strict_equality = true
+[project]
+name = "neurotheque-resources"
+version = "0.1.0"
+requires-python = ">=3.10"
+description = "EEG preprocessing and analysis pipeline (MNE, AutoReject, BIDS-first tools)"
+readme = "README.md"
+dependencies = [
+  "mne>=1.5.0",
+  "mne-bids>=0.13",
+  "numpy>=1.20.0",
+  "scipy>=1.6.0",
+  "matplotlib>=3.3.0",
+  "pandas>=1.2.0",
+  "autoreject>=0.3.0",
+  "mne-icalabel>=0.2.0",
+  "PyYAML>=6.0",
+  "jsonschema>=4.0.0",
+]
+
+[project.optional-dependencies]
+analysis = ["statsmodels>=0.12.0", "fooof>=1.0.0"]
+dev = [
+  "pytest>=7",
+  "pytest-cov>=4",
+  "black",
+  "isort",
+  "ruff",
+  "pre-commit",
+  "jupyter",
+  "sphinx",
+  "sphinx-rtd-theme",
+  "sphinx-autodoc-typehints",
+]
diff --git a/reports/fivepoint/fivepoint/analysis/01/001/task-5pt/run-01/band_power_summary.csv b/reports/fivepoint/fivepoint/analysis/01/001/task-5pt/run-01/band_power_summary.csv
@@ -1,6 +1,6 @@
 phase,metric,value
-stim,alpha_ERD,-0.15373997786841578
-stim,FM_theta,-0.02234160802606728
-resp,mu_ERD_pre,-0.03226107057625681
-resp,beta_ERD_pre,-0.0268033372889781
-resp,PMBR,-0.1223481318774348
+stim,alpha_ERD,-0.14683753560691049
+stim,FM_theta,-0.01965976606848644
+resp,mu_ERD_pre,-0.03227442705991137
+resp,beta_ERD_pre,-0.026809832885732555
+resp,PMBR,-0.12235814040894098
diff --git a/scr/config_schema.json b/scr/config_schema.json
@@ -3,6 +3,9 @@
   "title": "NeuroTheque Pipeline Configuration",
   "type": "object",
   "properties": {
+    "default_subject": {"type": "string", "description": "Default subject ID (e.g., '01')"},
+    "default_session": {"type": "string", "description": "Default session ID (e.g., '001')"},
+    "default_run": {"type": "string", "description": "Default run ID (e.g., '01')"},
     "directory": {
       "type": "object",
       "properties": {
@@ -59,7 +62,25 @@
             "properties": {
               "name": {
                 "type": "string",
-                "description": "Name of the step to execute"
+                "description": "Name of the step to execute",
+                "enum": [
+                  "LoadData",
+                  "FilterStep",
+                  "ICAStep",
+                  "ICAExtractionStep",
+                  "ICALabelingStep",
+                  "AutoRejectStep",
+                  "SaveCheckpoint",
+                  "SaveData",
+                  "PrepChannelsStep",
+                  "SplitTasksStep",
+                  "AutoSave",
+                  "EpochingStep",
+                  "TriggerParsingStep",
+                  "SyntheticRawStep",
+                  "GoNoGoTriggerStep",
+                  "GoNoGoEpochingStep"
+                ]
               },
               "params": {
                 "type": "object",
diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py
diff --git a/tools/dev/export_repo_to_xml.py b/tools/dev/export_repo_to_xml.py
diff --git a/tools/dev/generate_config.py b/tools/dev/generate_config.py
