Merge pull request #30 from AdebayoBraimah/dev5

Dev5

Author: AdebayoBraimah

CHANGES.rst

@@ -1,11 +1,17 @@
 CHANGES
 =========
 
+v0.1.1
+---------
+
+* BUG FIX: Fixed issue for incorrect references to tmp directories. This caused image conversion exceptions to be thrown, and thus no NIFTI BIDS files would be returned.
+* BUG FIX: Fixed issue for cases in which hidden indexing files (._) would be included in the file search.
+
 v0.1.0
 ---------
 
 * BUG FIX: Fixed bug in setup, which prevented proper installation and usage of the ``study_proc`` executable.
-* UPDATE: The documentation is now `available <https://convert-source.readthedocs.io/en/master/>`_.
+* UPDATE: The documentation is now `available <https://convert-source.readthedocs.io/en/0.1.0/>`_.
 
 v0.1.rc1
 ---------

README.md

@@ -1,4 +1,4 @@
-[![CircleCI](https://circleci.com/gh/AdebayoBraimah/convert_source.svg?style=svg)](https://app.circleci.com/pipelines/github/AdebayoBraimah/convert_source) [![Documentation Status](https://readthedocs.org/projects/convert-source/badge/?version=latest)](https://convert-source.readthedocs.io/en/master/)
+[![CircleCI](https://circleci.com/gh/AdebayoBraimah/convert_source.svg?style=svg)](https://app.circleci.com/pipelines/github/AdebayoBraimah/convert_source) [![Documentation Status](https://readthedocs.org/projects/convert-source/badge/?version=latest)](https://convert-source.readthedocs.io/en/latest/)
 
 # convert_source
 Convert source `DICOM`, `PAR REC` or `NIFTI` image data to BIDS directory layout.

convert_source/batch_convert.py

@@ -111,17 +111,21 @@ def batch_proc(study_img_dir: str,
             * Corresponding list of bval files.
             * Corresponding list of bvec files.
     '''
+    study_img_dir: str = os.path.abspath(study_img_dir)
+    
     # Check dependencies
     dcm2niix_cmd: Command = Command("dcm2niix")
     dcm2niix_cmd.check_dependency(path_envs=path_envs)
 
     # Write logs
     misc_dir: str = os.path.join(out_dir,'.misc')
     if os.path.exists(misc_dir):
+        out_dir: str = os.path.abspath(out_dir)
         misc_dir: str = os.path.abspath(misc_dir)
     else:
         os.makedirs(misc_dir)
         misc_dir: str = os.path.abspath(misc_dir)
+        out_dir: str = os.path.abspath(out_dir)
 
     now = datetime.now()
     dt_string: str = str(now.strftime("%m_%d_%Y_%H_%M"))
@@ -786,7 +790,7 @@ def source_to_bids(sub_data: SubDataInfo,
     # Using TmpDir and TmpFile context managers
     with TmpDir(tmp_dir=sub_tmp,use_cwd=False) as tmp:
         with TmpDir.TmpFile(tmp_dir=tmp.tmp_dir) as f:
-            _ = tmp.mk_tmp_dir()
+            tmp.mk_tmp_dir()
             [_path, basename, _ext] = f.file_parts()
             try:
                 img_data = convert_image_data(file=data,
@@ -904,7 +908,7 @@ def source_to_bids(sub_data: SubDataInfo,
                                            append_dwi_info=append_dwi_info,
                                            zero_pad=zero_pad,
                                            cprss_lvl=cprss_lvl)
-                    _ = tmp.rm_tmp_dir()
+                    tmp.rm_tmp_dir()
                     return (imgs,
                             jsons,
                             bvals,
@@ -929,7 +933,7 @@ def source_to_bids(sub_data: SubDataInfo,
                                            append_dwi_info=append_dwi_info,
                                            zero_pad=zero_pad,
                                            cprss_lvl=cprss_lvl)
-                    _ = tmp.rm_tmp_dir()
+                    tmp.rm_tmp_dir()
                     return (imgs,
                             jsons,
                             bvals,
@@ -991,13 +995,13 @@ def source_to_bids(sub_data: SubDataInfo,
                     else:
                         bvecs.append("")
                 # Clean-up
-                _ = tmp.rm_tmp_dir()
+                tmp.rm_tmp_dir()
                 return (imgs,
                         jsons,
                         bvals,
                         bvecs)
             except ConversionError:
-                _ = tmp.rm_tmp_dir()
+                tmp.rm_tmp_dir()
                 return [""],[""],[""],[""]
 
 def nifti_to_bids(sub_data: SubDataInfo,
@@ -1075,7 +1079,7 @@ def nifti_to_bids(sub_data: SubDataInfo,
 
     # Use TmpDir and NiiFile class context managers
     with TmpDir(tmp_dir=sub_tmp, use_cwd=False) as tmp:
-        _ = tmp.mk_tmp_dir()
+        tmp.mk_tmp_dir()
         with NiiFile(data) as n:
             [path, basename, ext] = n.file_parts()
             img_files: List[str] = glob.glob(os.path.join(path,basename + "*" + ext))
@@ -1270,7 +1274,7 @@ def nifti_to_bids(sub_data: SubDataInfo,
                     bvals.append("")
                     bvecs.append("")
         # Clean-up
-        _ = tmp.rm_tmp_dir()
+        tmp.rm_tmp_dir()
 
         return (imgs,
                 jsons,
@@ -1401,7 +1405,7 @@ def bids_ignore(out_dir: str) -> str:
 
     # Write to file using File class context manager
     with File(new_file) as f:
-        f.write_txt(".misc \n")
+        f.write_txt(".misc/* \n")
         f.write_txt("unknown/* \n")
 
     return new_file
@@ -1426,9 +1430,68 @@ def log_file(log: str) -> LogFile:
     log: LogFile = LogFile(log_file=log)
 
     now = datetime.now()
-    dt_string = now.strftime("%A %B %d, %Y %H:%M%:%S")
+    dt_string = now.strftime("%A %B %d, %Y %H:%M:%S")
 
     log.info(dt_string)
     log.info(f"convert_source v{__version__}")
 
     return log
+
+# This function was added for the Mac OS X case in which hidden
+# temporary indexing files are present throughout a given directory.
+#
+# This function was designed to handle that use case, however:
+#
+# * Implementation methods are currently unclear
+# * This approach is VERY SLOW as each parent directory is recursively searched.
+#
+# Moreover, this should be included in a later release should this continue to be an
+# issue moving forward.
+#
+# Adebayo Braimah - 12 March 2021
+#
+# def dir_clean_up(directory: str) -> str:
+#     '''Removes temporary indexing files (commonly found on
+#     Mac OS X).
+#
+#     These files are generally problematic as they cause several of
+#     ``convert_source``'s core functions to behave unpredictably.
+#
+#     Any identified tempoary indexing files are removed.
+#
+#     NOTE: The implementation here does work, but is VERY SLOW as the number of 
+#         files and directories is assumed to be large.
+#    
+#     Usage example:
+#         >>> directory = dir_clean_up(directory)
+#         >>>
+#        
+#     Arguments:
+#         directory: Input parent directory to recursively search (for hidden files in).
+#
+#     Returns:
+#         Absolute path to directory as a string.
+#     '''
+#     from shutil import rmtree
+#
+#     if os.path.exists(directory):
+#         directory = os.path.abspath(directory)
+#     else:
+#         raise FileNotFoundError("Input directory does not appear to exist.")
+#
+#     # This works - but is slow | O(n) space and time
+#     for root,dirnames,filenames in os.walk(directory):
+#         if len(dirnames) > 0:
+#             for dirname in dirnames:
+#                 if '._' in dirname:
+#                     hd_name: str = os.path.join(root,dirname)
+#                     rmtree(hd_name)
+#                     print(hd_name)
+#         if len(filenames) > 0:
+#             for file in filenames:
+#                 if '._' in file:
+#                     hf_name: str = os.path.join(root,file)
+#                     # hf_list.append(hf_name)
+#                     os.remove(hf_name)
+#                     print(hf_name)
+#     return directory

convert_source/cs_utils/fileio.py

@@ -185,7 +185,8 @@ def file_parts(self,
         file = self.abs_path()
 
         if platform.system().lower() == "windows":
-            [path, _filename] = os.path.splitdrive(file)
+            # [path, _filename] = os.path.splitdrive(file)
+           [path, _filename] = os.path.split(file) 
         else:
             [path, _filename] = os.path.split(file)
 
@@ -359,7 +360,7 @@ def __init__(self,
                 self.tmp_file: str = tmp_file
             else:
                 _n: int = 10000 # maximum N for random number generator
-                self.tmp_file: str = "tmp_file_" + str(random.randint(0,_n))
+                self.tmp_file: str = "tmp_file_" + str(random.randint(0,_n)) + ".txt"
 
             if ext:
                 self.tmp_file: str = self.tmp_file + f".{ext}"

convert_source/cs_utils/img_dir.py

@@ -82,6 +82,9 @@ def img_dir_list(directory: str,
     for root,dirnames,filenames in os.walk(directory):
         if len(filenames) > 0:
             for file in filenames:
+                if '._' in file:
+                    # Skip hidden files if they exist.
+                    continue
                 if '.dcm' in file.lower() or '.PAR' in file.upper() or '.nii' in file.lower():
                     file_name = os.path.join(root,file)
                     if '.dcm' in file.lower():

convert_source/cs_utils/utils.py

@@ -847,7 +847,7 @@ def convert_image_data(file: str,
     # Create TmpDir object
     with TmpDir(tmp_dir=out_dir,use_cwd=False) as tmp_dir:
         # Create temporary output directory
-        _ = tmp_dir.mk_tmp_dir()
+        tmp_dir.mk_tmp_dir()
 
         # Output directory
         out_dir: str = os.path.abspath(out_dir)
@@ -866,7 +866,7 @@ def convert_image_data(file: str,
         [imgs, jsons, bvals, bvecs] = img_data.copy_img_data(target_dir=out_dir)
 
         # Clean-up
-        _ = tmp_dir.rm_tmp_dir(rm_parent=False)
+        tmp_dir.rm_tmp_dir(rm_parent=False)
 
     # Image file check
     if len(imgs) == 0:

convert_source/imgio/pario.py

@@ -178,7 +178,7 @@ def get_echo_time(par_file: str,
     df: pd.DataFrame = df.dropna(axis=0)
 
     with TmpDir(tmp_dir=tmp_dir,use_cwd=False) as tmp:
-        _ = tmp.mk_tmp_dir()
+        tmp.mk_tmp_dir()
         with TmpDir.TmpFile(tmp_dir=tmp.tmp_dir,ext="txt") as f:
             df.to_csv(f.file,sep=",",header=False,index=False)
             mat = np.loadtxt(f.file,delimiter=",")
@@ -222,7 +222,7 @@ def get_flip_angle(par_file: str,
     df: pd.DataFrame = df.dropna(axis=0)
 
     with TmpDir(tmp_dir=tmp_dir,use_cwd=False) as tmp:
-        _ = tmp.mk_tmp_dir()
+        tmp.mk_tmp_dir()
         with TmpDir.TmpFile(tmp_file="file.tmp.txt",tmp_dir=tmp.tmp_dir) as f:
             df.to_csv(f.file,sep=",",header=False,index=False)
             mat = np.loadtxt(f.file,delimiter=",")

convert_source/tests/01_fileio_test.py

@@ -30,6 +30,7 @@ def test_file_class():
         assert f.file == 'test.file.txt'
 
 def test_command_class():
+    """NOTE: This test will FAIL on Windows operating systems."""
     x: str = 'ls'
     c = Command(x)
     c.cmd_list.append(os.getcwd())
@@ -44,7 +45,7 @@ def test_command_class():
 def test_tmpdir_class():
     x: str = 'tmpdir'
     with TmpDir(x,True) as tmp:
-        _ = tmp.mk_tmp_dir()
+        tmp.mk_tmp_dir()
         assert os.path.exists(tmp.tmp_dir) == True
         with TmpDir.TmpFile(tmp.tmp_dir) as f:
             f.touch()

convert_source/tests/04_convert_image_test.py

@@ -89,6 +89,9 @@ class PlatformInferError(Exception):
     assert os.path.exists(file_name) == False
 
 def test_dependency():
+    """NOTE: This test will FAIL if ``dcm2niix`` is already in the
+    system path.
+    """
     dcm2niix_cmd: Command = Command("dcm2niix")
 
     with pytest.raises(DependencyError):
@@ -130,6 +133,7 @@ def test_data_conversion():
         [img, json, bval, bvec] = convert_image_data(data,'test_img',os.getcwd())
 
 def test_cleanup():
+    """NOTE: This test currently FAILS on Windows operating systems."""
     del_method: bool = False
     rm_method: bool = False
     rm_item_method: bool = False

convert_source/tests/07_dcmio_test.py

@@ -127,6 +127,7 @@ def test_get_mb():
     assert get_mb(data) == 1 
 
 def test_cleanup():
+    """NOTE: This test currently FAILS on Windows operating systems."""
     del_method: bool = False
     rm_method: bool = False
     rm_item_method: bool = False