Merge pull request #79 from mgxd/fix/validation

FIX: Add BIDS validation

Author: mgxd

Dockerfile

@@ -126,6 +126,13 @@ RUN useradd -m -s /bin/bash -G users nibabies
 WORKDIR /home/nibabies
 ENV HOME="/home/nibabies"
 
+# Install node + npm packages
+RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - && \
+    apt-get install -y --no-install-recommends \
+                    nodejs && \
+    npm install -g svgo [email protected] && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ~/.npm ~/.empty
+
 # Installing and setting up miniconda
 RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
     bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \
@@ -143,11 +150,10 @@ RUN conda install -y python=3.7.1 \
                      numpy=1.20 \
                      scipy=1.6 \
                      scikit-learn=0.19 \
-                     matplotlib=2.2.2 \
-                     pandas=0.23.4 \
-                     libxml2=2.9.8 \
-                     libxslt=1.1.32 \
-                     graphviz=2.40.1 \
+                     matplotlib=2.2 \
+                     libxml2=2.9 \
+                     libxslt=1.1 \
+                     graphviz=2.40 \
                      traits=4.6.0 \
                      pip=19.1 \
                      zlib; sync && \

nibabies/utils/bids.py

@@ -5,6 +5,7 @@
 import os
 import json
 from pathlib import Path
+import sys
 
 
 def write_bidsignore(deriv_dir):
@@ -186,3 +187,90 @@ def group_bolds_ref(*, layout, subject):
         assert len(combinations) == len(files), "Nonequal number of combinations and file buckets"
 
     return combinations, files
+
+
+def validate_input_dir(exec_env, bids_dir, participant_label):
+    # Ignore issues and warnings that should not influence NiBabies
+    import tempfile
+    import subprocess
+    validator_config_dict = {
+        "ignore": [
+            "EVENTS_COLUMN_ONSET",
+            "EVENTS_COLUMN_DURATION",
+            "TSV_EQUAL_ROWS",
+            "TSV_EMPTY_CELL",
+            "TSV_IMPROPER_NA",
+            "VOLUME_COUNT_MISMATCH",
+            "BVAL_MULTIPLE_ROWS",
+            "BVEC_NUMBER_ROWS",
+            "DWI_MISSING_BVAL",
+            "INCONSISTENT_SUBJECTS",
+            "INCONSISTENT_PARAMETERS",
+            "BVEC_ROW_LENGTH",
+            "B_FILE",
+            "PARTICIPANT_ID_COLUMN",
+            "PARTICIPANT_ID_MISMATCH",
+            "TASK_NAME_MUST_DEFINE",
+            "PHENOTYPE_SUBJECTS_MISSING",
+            "STIMULUS_FILE_MISSING",
+            "DWI_MISSING_BVEC",
+            "EVENTS_TSV_MISSING",
+            "TSV_IMPROPER_NA",
+            "ACQTIME_FMT",
+            "Participants age 89 or higher",
+            "DATASET_DESCRIPTION_JSON_MISSING",
+            "FILENAME_COLUMN",
+            "WRONG_NEW_LINE",
+            "MISSING_TSV_COLUMN_CHANNELS",
+            "MISSING_TSV_COLUMN_IEEG_CHANNELS",
+            "MISSING_TSV_COLUMN_IEEG_ELECTRODES",
+            "UNUSED_STIMULUS",
+            "CHANNELS_COLUMN_SFREQ",
+            "CHANNELS_COLUMN_LOWCUT",
+            "CHANNELS_COLUMN_HIGHCUT",
+            "CHANNELS_COLUMN_NOTCH",
+            "CUSTOM_COLUMN_WITHOUT_DESCRIPTION",
+            "ACQTIME_FMT",
+            "SUSPICIOUSLY_LONG_EVENT_DESIGN",
+            "SUSPICIOUSLY_SHORT_EVENT_DESIGN",
+            "MALFORMED_BVEC",
+            "MALFORMED_BVAL",
+            "MISSING_TSV_COLUMN_EEG_ELECTRODES",
+            "MISSING_SESSION"
+        ],
+        "error": ["NO_T1W"],
+        "ignoredFiles": ['/dataset_description.json', '/participants.tsv']
+    }
+    # Limit validation only to data from requested participants
+    if participant_label:
+        all_subs = set([s.name[4:] for s in bids_dir.glob('sub-*')])
+        selected_subs = set([s[4:] if s.startswith('sub-') else s
+                             for s in participant_label])
+        bad_labels = selected_subs.difference(all_subs)
+        if bad_labels:
+            error_msg = 'Data for requested participant(s) label(s) not found. Could ' \
+                        'not find data for participant(s): %s. Please verify the requested ' \
+                        'participant labels.'
+            if exec_env == 'docker':
+                error_msg += ' This error can be caused by the input data not being ' \
+                             'accessible inside the docker container. Please make sure all ' \
+                             'volumes are mounted properly (see https://docs.docker.com/' \
+                             'engine/reference/commandline/run/#mount-volume--v---read-only)'
+            if exec_env == 'singularity':
+                error_msg += ' This error can be caused by the input data not being ' \
+                             'accessible inside the singularity container. Please make sure ' \
+                             'all paths are mapped properly (see https://www.sylabs.io/' \
+                             'guides/3.0/user-guide/bind_paths_and_mounts.html)'
+            raise RuntimeError(error_msg % ','.join(bad_labels))
+
+        ignored_subs = all_subs.difference(selected_subs)
+        if ignored_subs:
+            for sub in ignored_subs:
+                validator_config_dict["ignoredFiles"].append("/sub-%s/**" % sub)
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp:
+        temp.write(json.dumps(validator_config_dict))
+        temp.flush()
+        try:
+            subprocess.check_call(['bids-validator', str(bids_dir), '-c', temp.name])
+        except FileNotFoundError:
+            print("bids-validator does not appear to be installed", file=sys.stderr)