Merge pull request #283 from daeh/master

added decoding of bytes literal strings returned by subprocess during anonymization

Author: yarikoptic

.gitignore

@@ -94,6 +94,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \
            'traits>=4.6.0' \
            'scipy' \
            'numpy' \
+           'pandas' \
            'nomkl' \
     && sync && conda clean -tipsy && sync \
     && bash -c "source activate base \
@@ -146,6 +147,7 @@ RUN echo '{ \
     \n          "traits>=4.6.0", \
     \n          "scipy", \
     \n          "numpy", \
+    \n          "pandas", \
     \n          "nomkl" \
     \n        ], \
     \n        "pip_install": [ \

Dockerfile

@@ -100,7 +100,7 @@ def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid,
         anon_outdir = outdir
 
     # Generate heudiconv info folder
-    idir = op.join(outdir, '.heudiconv', sid)
+    idir = op.join(outdir, '.heudiconv', anon_sid)
     if bids and ses:
         idir = op.join(idir, 'ses-%s' % str(ses))
     if anon_outdir == outdir:

heudiconv/convert.py

@@ -0,0 +1,19 @@
+#! /usr/bin/env python 
+
+import sys
+import re
+import ctypes
+
+
+def bids_id_(sid):
+    parsed_id = re.compile(r"^(?:sub-|)(.+)$").search(sid).group(1)
+    return str(ctypes.c_size_t(hash(parsed_id)).value)
+
+
+def main():
+    sid = sys.argv[1]
+    return bids_id_(sid)
+
+
+if __name__ == '__main__':
+    main()

heudiconv/tests/anonymize_script.py

@@ -19,16 +19,17 @@
 
 @pytest.mark.parametrize('subject', ['sub-sid000143'])
 @pytest.mark.parametrize('heuristic', ['reproin.py'])
+@pytest.mark.parametrize('anon_cmd', [None, 'anonymize_script.py'])
 @pytest.mark.skipif(not have_datalad, reason="no datalad")
-def test_conversion(tmpdir, subject, heuristic):
+def test_conversion(tmpdir, subject, heuristic, anon_cmd):
     tmpdir.chdir()
     try:
         datadir = fetch_data(tmpdir.strpath, subject)
     except IncompleteResultsError as exc:
         pytest.skip("Failed to fetch test data: %s" % str(exc))
     outdir = tmpdir.mkdir('out').strpath
 
-    args = gen_heudiconv_args(datadir, outdir, subject, heuristic)
+    args = gen_heudiconv_args(datadir, outdir, subject, heuristic, anon_cmd)
     runner(args) # run conversion
 
     # verify functionals were converted

heudiconv/tests/test_regression.py

@@ -5,14 +5,16 @@
 TESTS_DATA_PATH = op.join(op.dirname(__file__), 'data')
 
 
-def gen_heudiconv_args(datadir, outdir, subject, heuristic_file, xargs=None):
+def gen_heudiconv_args(datadir, outdir, subject, heuristic_file, anon_cmd=None, xargs=None):
     heuristic = op.realpath(op.join(HEURISTICS_PATH, heuristic_file))
     args = ["-d", op.join(datadir, 'sourcedata/{subject}/*/*/*.tgz'),
             "-c", "dcm2niix",
             "-o", outdir,
             "-s", subject,
             "-f", heuristic,
             "--bids",]
+    if anon_cmd:
+        args += ["--anon-cmd", op.join(op.dirname(__file__), anon_cmd), "-a", outdir]
     if xargs:
         args += xargs
 

heudiconv/tests/utils.py

@@ -103,9 +103,19 @@ def dec(obj):
 
 
 def anonymize_sid(sid, anon_sid_cmd):
+    import sys
     from subprocess import check_output
+    
     cmd = [anon_sid_cmd, sid]
-    return check_output(cmd).strip()
+    shell_return = check_output(cmd)
+
+    ### Handle subprocess returning a bytes literal string to a python3 interpreter
+    if all([sys.version_info[0] > 2, isinstance(shell_return, bytes), isinstance(sid, str)]):
+        anon_sid = shell_return.decode()
+    else:
+        anon_sid = shell_return
+    
+    return anon_sid.strip()
 
 
 def create_file_if_missing(filename, content):
@@ -448,4 +458,4 @@ def create_tree(path, tree, archives_leading_dir=True):
                     load = load.encode('utf-8')
                 f.write(load)
         if executable:
-            os.chmod(full_name, os.stat(full_name).st_mode | stat.S_IEXEC)
+            os.chmod(full_name, os.stat(full_name).st_mode | stat.S_IEXEC)

heudiconv/utils.py

@@ -10,7 +10,7 @@ docker run --rm $image generate docker -b neurodebian:stretch -p apt \
     --dcm2niix version=v1.0.20180622 method=source \
     --install git gcc pigz liblzma-dev libc-dev git-annex-standalone netbase \
     --copy . /src/heudiconv \
-    --miniconda use_env=base conda_install="python=3.6 traits>=4.6.0 scipy numpy nomkl" \
+    --miniconda use_env=base conda_install="python=3.6 traits>=4.6.0 scipy numpy nomkl pandas" \
       pip_install="/src/heudiconv[all]" \
       pip_opts="--editable" \
     --entrypoint "heudiconv" \