[FIX] pass dataset level to stats actions when using python CLI (#1258)

* refactor to improve testing python CLI

* fix links

* add dataset level analysis using docker

* pin dependencies for containers and pass options to ignore nidm in circle ci

* use local files to build apptainer image

* fci

* fix path

* pass options through CLI

* typo

* run default model in CI

* add README to apptainer def

* fix default

* add submodules

* glob

* fci

* python CLI fixes

* use absolute path

* fix test

* fix tests

* show options in debug mode

* fix saveOptions

* fix

* do not use nidm for run and subject level for ds000001

* mount options

Author: Remi-Gau

.circleci/config.yml

@@ -75,7 +75,7 @@ jobs:
                 -   data/ds000001
                 -   data/ds000001-fmriprep
 
-    test:
+    default_model:
         machine:
             image: ubuntu-2204:2022.10.2
 
@@ -87,13 +87,53 @@ jobs:
         -   run: mkdir -p ${HOME}/outputs/ds000001/derivatives
 
         -   run:
-                name: print version
+                name: default model subject level
                 command: |
                     user_name=cpplab
                     repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
                     docker run -ti --rm \
                       -v /tmp/workspace/data/ds000001:/bids_dataset \
-                        ${user_name}/${repo_name} --version
+                      -v ${HOME}/outputs:/outputs \
+                        ${user_name}/${repo_name} \
+                          /bids_dataset \
+                          /outputs/ds000001 \
+                          subject \
+                          --action default_model \
+                          --task balloonanalogrisktask \
+                          --space MNI152NLin2009cAsym \
+                          --verbosity 3
+
+                    cat ${HOME}/outputs/ds000001/derivatives/models/*.json
+
+        -   run:
+                name: default model dataset level
+                command: |
+                    user_name=cpplab
+                    repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
+                    docker run -ti --rm \
+                      -v /tmp/workspace/data/ds000001:/bids_dataset \
+                      -v ${HOME}/outputs:/outputs \
+                        ${user_name}/${repo_name} \
+                          /bids_dataset \
+                          /outputs/ds000001 \
+                          dataset \
+                          --action default_model \
+                          --task balloonanalogrisktask \
+                          --space MNI152NLin2009cAsym \
+                          --verbosity 3
+
+                    cat ${HOME}/outputs/ds000001/derivatives/models/*.json
+
+    stats:
+        machine:
+            image: ubuntu-2204:2022.10.2
+
+        steps:
+        -   attach_workspace:
+                at: /tmp/workspace
+        -   run: docker load -i /tmp/workspace/docker/image.tar
+
+        -   run: mkdir -p ${HOME}/outputs/ds000001/derivatives
 
         -   run:
                 name: smooth
@@ -112,21 +152,21 @@ jobs:
                           --participant_label 01 02 \
                           --space MNI152NLin2009cAsym \
                           --fwhm 8 \
-                          --verbosity 2
-                no_output_timeout: 6h
+                          --verbosity 3
 
-    # needed to access the model
+        # needed to access the model
         -   checkout
 
         -   run:
-                name: stats
+                name: stats subject level
                 command: |
                     user_name=cpplab
                     repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
                     docker run -ti --rm  \
                       -v /tmp/workspace/data/ds000001:/bids_dataset \
                       -v ${HOME}/outputs:/outputs \
                       -v ~/project/demos/openneuro/models:/models \
+                      -v ~/project/demos/openneuro/options:/options \
                         ${user_name}/${repo_name} \
                           /bids_dataset \
                           /outputs/ds000001 \
@@ -137,11 +177,33 @@ jobs:
                           --ignore slicetiming \
                           --space MNI152NLin2009cAsym \
                           --skip_validation \
-                          --fwhm 8 \
+                          --fwhm 0 \
                           --participant_label 01 02 \
-                          --verbosity 2
-                no_output_timeout: 6h
+                          --verbosity 3 \
+                          --options /options/ds000001.json
 
+        -   run:
+                name: stats group level
+                command: |
+                    user_name=cpplab
+                    repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
+                    docker run -ti --rm  \
+                      -v /tmp/workspace/data/ds000001:/bids_dataset \
+                      -v ${HOME}/outputs:/outputs \
+                      -v ~/project/demos/openneuro/models:/models \
+                      -v ~/project/demos/openneuro/options:/options \
+                        ${user_name}/${repo_name} \
+                          /bids_dataset \
+                          /outputs/ds000001 \
+                          dataset \
+                          --action stats \
+                          --preproc_dir /outputs/ds000001/derivatives/bidspm-preproc \
+                          --model_file /models/model-balloonanalogrisktaskDefault_smdl.json \
+                          --space MNI152NLin2009cAsym \
+                          --skip_validation \
+                          --fwhm 0 \
+                          --verbosity 3 \
+                          --options /options/ds000001.json
 
     deploy:
 
@@ -187,14 +249,18 @@ workflows:
         jobs:
         -   build
         -   get_data
-        -   test:
+        -   default_model:
+                requires:
+                -   build
+                -   get_data
+        -   stats:
                 requires:
                 -   build
                 -   get_data
         -   deploy:
                 context:
                 -   DOCKER_HUB
                 requires:
-                -   test
+                -   build
 
 # VS Code Extension Version: 1.5.1

.github/workflows/apptainer_build.yml

@@ -17,6 +17,9 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         -   uses: actions/checkout@v4
+            with:
+                submodules: recursive
+                fetch-depth: 0
 
         -   uses: eWaterCycle/setup-apptainer@v2
             with:

.github/workflows/tests.yml

@@ -65,7 +65,7 @@ jobs:
                     matlab: R2023b
                     mode: slow
                 -   test_type: unit
-                    os: macos-latest
+                    os: macos-13
                     matlab: R2023b
                     mode: fast
             fail-fast: false

.gitignore

@@ -29,6 +29,7 @@ skipped_roi_*.tsv
 CHANGES
 README
 tests/data/tsv_files/moae_results_table.tsv
+htmlcov
 
 # Project specific
 onsets*_events.mat
@@ -48,6 +49,8 @@ src/bidspm/_version.py
 **/__pycache__
 **/build
 .coverage
+.pytest_cache
+.tox
 
 .mypy_cache
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM bids/base_validator
+FROM bids/base_validator:1.13.1
 
 ARG DEBIAN_FRONTEND="noninteractive"
 
@@ -48,6 +48,7 @@ WORKDIR /home/neuro
 COPY . /home/neuro/bidspm
 WORKDIR /home/neuro/bidspm
 RUN pip install --no-cache-dir --upgrade pip && \
+    pip3 --no-cache-dir install -r requirements.txt && \
     pip3 --no-cache-dir install . && \
     octave --no-gui --eval "addpath('/opt/spm12/'); savepath ();" && \
     octave --no-gui --eval "addpath(pwd); savepath(); bidspm(); path"

Makefile

@@ -108,6 +108,7 @@ coverage: ## use coverage
 	coverage erase
 	coverage run --source src -m pytest
 	coverage report -m
+	coverage html
 
 ################################################################################
 #   DOCKER

README.md

@@ -40,7 +40,7 @@ To start using bidspm, you just need to initialize it for the current MATLAB / O
 bidspm()
 ```
 
-Please see our [documentation](https://bidspm.readthedocs.io/en/latest/installation.html) for more info.
+Please see our [documentation](https://bidspm.readthedocs.io/en/latest/installation/index.html) for more info.
 
 ## Usage
 

bidspm.def

@@ -1,5 +1,14 @@
 BootStrap: docker
-From: bids/base_validator
+From: bids/base_validator:1.13.1
+
+%files
+    pyproject.toml /opt/bidspm/pyproject.toml
+    requirements.txt /opt/bidspm/requirements.txt
+    README.md  /opt/bidspm/README.md
+    bidspm.m /opt/bidspm/bidspm.m
+    src /opt/bidspm/src
+    lib /opt/bidspm/lib
+    .git /opt/bidspm/.git
 
 %post
     apt-get -qq update
@@ -38,10 +47,9 @@ From: bids/base_validator
     make -C /opt/spm12/src PLATFORM=octave install
     ln -s /opt/spm12/bin/spm12-octave /usr/local/bin/spm12
 
-    mkdir /opt/bidspm
-    git clone --recurse-submodules https://github.com/cpp-lln-lab/bidspm.git /opt/bidspm
-    pip install --no-cache-dir --upgrade pip && \
-    pip3 --no-cache-dir install /opt/bidspm && \
+    pip install --upgrade pip
+    pip install -r /opt/bidspm/requirements.txt
+    pip install /opt/bidspm
     octave --no-gui --eval "addpath('/opt/spm12/'); savepath ('/usr/share/octave/site/m/startup/octaverc');" && \
     octave --no-gui --eval "addpath('/opt/bidspm/'); savepath('/usr/share/octave/site/m/startup/octaverc'); bidspm(); path"
 

bidspm.m

@@ -179,15 +179,6 @@ function initBidspm(dev)
                        pathSep, ...
                        genpath(fullfile(rootDir(), 'src', 'workflows', 'stats')));
 
-    % add library that do not have an set up script
-    libList = {'spmup'};
-
-    for i = 1:numel(libList)
-      BIDSPM_PATHS = cat(2, BIDSPM_PATHS, ...
-                         pathSep, ...
-                         genpath(fullfile(rootDir(), 'lib', libList{i})));
-    end
-
     libList = {'mancoreg', ...
                'bids-matlab', ...
                'slice_display', ...
@@ -270,7 +261,7 @@ function updateMacstoolbox()
   end
 
   if exist(target_dir, 'dir') == 7
-    msg = sprintf('updating MACS toolbox: ');
+    msg = sprintf('updating MACS toolbox\n');
     fprintf(1, msg);
     [status, cmdout] = system(sprintf('git -C %s pull', target_dir));
     if status ~= 0

demos/openneuro/Makefile

@@ -18,11 +18,12 @@ data_ds000001:
 	mkdir -p inputs
 	cd inputs && datalad install ///openneuro/ds000001
 	cd inputs && datalad install ///openneuro-derivatives/ds000001-fmriprep
-	cd inputs/ds000001 && datalad get sub-0[1-5] -J 3
-	cd inputs/ds000001-fmriprep && datalad get sub-0[1-5]/func/*tsv -J 12
-	cd inputs/ds000001-fmriprep && datalad get sub-0[1-5]/func/*json -J 12
-	cd inputs/ds000001-fmriprep && datalad get sub-0[1-5]/func/*MNI*desc-*bold.nii.gz -J 12
-	cd inputs/ds000001-fmriprep && datalad get sub-0[1-5]/anat/*MNI*desc-preproc*.nii.gz -J 12
+	cd inputs/ds000001 && datalad get sub-0[1-2] -J 3
+	cd inputs/ds000001-fmriprep && datalad get sub-0[1-2]/func/*tsv -J 12
+	cd inputs/ds000001-fmriprep && datalad get sub-0[1-2]/func/*json -J 12
+	cd inputs/ds000001-fmriprep && datalad get sub-0[1-2]/anat/*MNI*desc-preproc*.nii.gz -J 12
+	cd inputs/ds000001-fmriprep && datalad get sub-0[1-2]/func/*MNI*desc-preproc*.nii.gz -J 12
+    cd inputs/ds000001-fmriprep && datalad get sub-0[1-2]/func/*MNI*desc-*bold.nii.gz -J 12
 
 data_ds000114:
 	mkdir -p inputs