Merge pull request #507 from dbic/enh-bids-meta

RF/ENH: populate a singular scans.json at the top level of the bids dataset

Author: yarikoptic

heudiconv/bids.py

@@ -18,7 +18,7 @@
     load_json,
     save_json,
     create_file_if_missing,
-    json_dumps_pretty,
+    json_dumps,
     set_readonly,
     is_readonly,
     get_datetime,
@@ -120,6 +120,9 @@ def populate_bids_templates(path, defaults={}):
     create_file_if_missing(op.join(path, 'README'),
         "TODO: Provide description for the dataset -- basic details about the "
         "study, possibly pointing to pre-registration (if public or embargoed)")
+    create_file_if_missing(op.join(path, 'scans.json'),
+        json_dumps(SCANS_FILE_FIELDS, sort_keys=False)
+    )
 
     populate_aggregated_jsons(path)
 
@@ -404,11 +407,6 @@ def add_rows_to_scans_keys_file(fn, newrows):
         os.unlink(fn)
     else:
         fnames2info = newrows
-        # Populate _scans.json (an optional file to describe column names in
-        # _scans.tsv). This auto generation will make BIDS-validator happy.
-        scans_json = '.'.join(fn.split('.')[:-1] + ['json'])
-        if not op.lexists(scans_json):
-            save_json(scans_json, SCANS_FILE_FIELDS, sort_keys=False)
 
     header = SCANS_FILE_FIELDS
     # prepare all the data rows

heudiconv/tests/test_main.py

@@ -4,13 +4,16 @@
 from heudiconv.main import workflow
 from heudiconv import __version__
 from heudiconv.utils import (create_file_if_missing,
+                             load_json,
                              set_readonly,
                              is_readonly)
 from heudiconv.bids import (populate_bids_templates,
                             add_participant_record,
                             get_formatted_scans_key_row,
                             add_rows_to_scans_keys_file,
-                            find_subj_ses)
+                            find_subj_ses,
+                            SCANS_FILE_FIELDS,
+                            )
 from heudiconv.external.dlad import MIN_VERSION, add_to_datalad
 
 from .utils import TESTS_DATA_PATH
@@ -81,6 +84,9 @@ def test_populate_bids_templates(tmpdir):
     assert "something" not in description_file.read()
     assert "TODO" in description_file.read()
 
+    # Explicit str() is needed for py 3.5. TODO: remove when dropping 3.5
+    assert load_json(str(tmpdir / "scans.json")) == SCANS_FILE_FIELDS
+
 
 def test_add_participant_record(tmpdir):
     tf = tmpdir.join('participants.tsv')
@@ -127,6 +133,7 @@ def test_prepare_for_datalad(tmpdir):
         '.gitattributes',
         '.datalad/config', '.datalad/.gitattributes',
         'dataset_description.json',
+        'scans.json',
         'CHANGES', 'README'}
     assert set(ds.repo.get_indexed_files()) == target_files
     # and all are under git
@@ -217,7 +224,9 @@ def _check_rows(fn, rows):
         assert dates == sorted(dates)
 
     _check_rows(fn, rows)
-    assert op.exists(opj(tmpdir.strpath, 'file.json'))
+    # we no longer produce a sidecar .json file there and only generate
+    # it while populating templates for BIDS
+    assert not op.exists(opj(tmpdir.strpath, 'file.json'))
     # add a new one
     extra_rows = {
         'a_new_file.nii.gz': ['2016adsfasd23', '', 'fasadfasdf'],

heudiconv/utils.py

@@ -147,21 +147,6 @@ def write_config(outfile, info):
         fp.writelines(PrettyPrinter().pformat(info))
 
 
-def _canonical_dumps(json_obj, **kwargs):
-    """ Dump `json_obj` to string, allowing for Python newline bug
-
-    Runs ``json.dumps(json_obj, \*\*kwargs), then removes trailing whitespaces
-    added when doing indent in some Python versions. See
-    https://bugs.python.org/issue16333. Bug seems to be fixed in 3.4, for now
-    fixing manually not only for aestetics but also to guarantee the same
-    result across versions of Python.
-    """
-    out = json.dumps(json_obj, **kwargs)
-    if 'indent' in kwargs:
-        out = out.replace(' \n', '\n')
-    return out
-
-
 def load_json(filename):
     """Load data from a json file
 
@@ -220,19 +205,25 @@ def save_json(filename, data, indent=2, sort_keys=True, pretty=False):
                 % (str(exc), filename)
             )
     if not pretty:
-        j = _canonical_dumps(data, **dumps_kw)
+        j = json_dumps(data, **dumps_kw)
     assert j is not None  # one way or another it should have been set to a str
     with open(filename, 'w') as fp:
         fp.write(j)
 
 
+def json_dumps(json_obj, indent=2, sort_keys=True):
+    """Unified (default indent and sort_keys) invocation of json.dumps
+    """
+    return json.dumps(json_obj, indent=indent, sort_keys=sort_keys)
+
+
 def json_dumps_pretty(j, indent=2, sort_keys=True):
     """Given a json structure, pretty print it by colliding numeric arrays
     into a line.
 
     If resultant structure differs from original -- throws exception
     """
-    js = _canonical_dumps(j, indent=indent, sort_keys=sort_keys)
+    js = json_dumps(j, indent=indent, sort_keys=sort_keys)
     # trim away \n and spaces between entries of numbers
     js_ = re.sub(
         '[\n ]+("?[-+.0-9e]+"?,?) *\n(?= *"?[-+.0-9e]+"?)', r' \1',