Merge pull request #218 from dartmouth-pbs/bf-dlmetadata

BF+ENH: set dist-restrictions metadata correctly

Author: yarikoptic

heudiconv/external/dlad.py

@@ -1,4 +1,6 @@
+import inspect
 import os
+
 import os.path as op
 import logging
 from glob import glob
@@ -61,17 +63,30 @@ def add_to_datalad(topdir, studydir, msg, bids):
         assert ds.is_installed()
         superds = ds
 
-    create_file_if_missing(
-        op.join(studydir, '.gitattributes'),
-        """\
+    # TODO: we need a helper (in DataLad ideally) to ease adding such
+    # specifications
+    gitattributes_path = op.join(studydir, '.gitattributes')
+    # We will just make sure that all our desired rules are present in it
+    desired_attrs = """\
 * annex.largefiles=(largerthan=100kb)
 *.json annex.largefiles=nothing
 *.txt annex.largefiles=nothing
 *.tsv annex.largefiles=nothing
 *.nii.gz annex.largefiles=anything
 *.tgz annex.largefiles=anything
 *_scans.tsv annex.largefiles=anything
-""")
+"""
+    if op.exists(gitattributes_path):
+        with open(gitattributes_path, 'rb') as f:
+            known_attrs = [line.decode('utf-8').rstrip() for line in f.readlines()]
+    else:
+        known_attrs = []
+    for attr in desired_attrs.split('\n'):
+        if attr not in known_attrs:
+            known_attrs.append(attr)
+    with open(gitattributes_path, 'wb') as f:
+        f.write('\n'.join(known_attrs).encode('utf-8'))
+
     # so for mortals it just looks like a regular directory!
     if not ds.config.get('annex.thin'):
         ds.config.add('annex.thin', 'true', where='local')
@@ -155,7 +170,12 @@ def mark_sensitive(ds, path_glob):
     paths = glob(op.join(ds.path, path_glob))
     if not paths:
         return
-    ds.repo.set_metadata(
+    lgr.debug("Marking %d files with distribution-restrictions field",
+              len(paths))
+    # set_metadata can be a bloody generator
+    res = ds.repo.set_metadata(
         paths,
-        init=[('distribution-restrictions', 'sensitive')],
-        recursive=True)
+        init=dict([('distribution-restrictions', 'sensitive')]),
+        recursive=True)
+    if inspect.isgenerator(res):
+        res = list(res)

heudiconv/external/tests/__init__.py

@@ -0,0 +1,23 @@
+from ..dlad import mark_sensitive
+from datalad.api import Dataset
+from ...utils import create_tree
+
+
+def test_mark_sensitive(tmpdir):
+    ds = Dataset(str(tmpdir)).create(force=True)
+    create_tree(
+        str(tmpdir),
+        {
+            'f1': 'd1',
+            'f2': 'd2',
+            'g1': 'd3',
+            'g2': 'd1',
+         }
+    )
+    ds.add('.')
+    mark_sensitive(ds, 'f*')
+    all_meta = dict(ds.repo.get_metadata('.'))
+    target_rec = {'distribution-restrictions': ['sensitive']}
+    # g2 since the same content
+    assert not all_meta.pop('g1', None)  # nothing or empty record
+    assert all_meta == {'f1': target_rec, 'f2': target_rec, 'g2': target_rec}

heudiconv/external/tests/test_dlad.py

@@ -389,3 +389,63 @@ def clear_temp_dicoms(item_dicoms):
 def file_md5sum(filename):
     with open(filename, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
+
+
+# Borrowed from DataLad (MIT license), with "archives" functionality commented
+# out
+class File(object):
+    """Helper for a file entry in the create_tree/@with_tree
+
+    It allows to define additional settings for entries
+    """
+    def __init__(self, name, executable=False):
+        """
+
+        Parameters
+        ----------
+        name : str
+          Name of the file
+        executable: bool, optional
+          Make it executable
+        """
+        self.name = name
+        self.executable = executable
+
+    def __str__(self):
+        return self.name
+
+
+def create_tree(path, tree, archives_leading_dir=True):
+    """Given a list of tuples (name, load) or a dict create such a tree
+
+    if load is a tuple or a dict itself -- that would create either a subtree
+    or an archive with that content and place it into the tree if name ends
+    with .tar.gz
+    """
+    lgr.log(5, "Creating a tree under %s", path)
+    if not op.exists(path):
+        os.makedirs(path)
+
+    if isinstance(tree, dict):
+        tree = tree.items()
+
+    for file_, load in tree:
+        if isinstance(file_, File):
+            executable = file_.executable
+            name = file_.name
+        else:
+            executable = False
+            name = file_
+        full_name = op.join(path, name)
+        if isinstance(load, (tuple, list, dict)):
+            # if name.endswith('.tar.gz') or name.endswith('.tar') or name.endswith('.zip'):
+            #     create_tree_archive(path, name, load, archives_leading_dir=archives_leading_dir)
+            # else:
+            create_tree(full_name, load, archives_leading_dir=archives_leading_dir)
+        else:
+            with open(full_name, 'w') as f:
+                if sys.version_info[0] == 2 and not isinstance(load, str):
+                    load = load.encode('utf-8')
+                f.write(load)
+        if executable:
+            os.chmod(full_name, os.stat(full_name).st_mode | stat.S_IEXEC)