Merge pull request #1190 from JuliaSprenger/enh/get_io

[IO] Extend utilities to find suitable IO for a dataset

Author: apdavison

neo/io/__init__.py

@@ -10,6 +10,7 @@
 Functions:
 
 .. autofunction:: neo.io.get_io
+.. autofunction:: neo.io.list_candidate_ios
 
 
 Classes:
@@ -262,7 +263,8 @@
 
 """
 
-import os.path
+import pathlib
+from collections import Counter
 
 # try to import the neuroshare library.
 # if it is present, use the neuroshareapiio to load neuroshare files
@@ -350,7 +352,7 @@
     CedIO,
     EDFIO,
     ElanIO,
-    # ElphyIO,
+    ElphyIO,
     ExampleIO,
     IgorIO,
     IntanIO,
@@ -359,7 +361,8 @@
     MEArecIO,
     MaxwellIO,
     MicromedIO,
-    NixIO,  # place NixIO before other IOs that use HDF5 to make it the default for .h5 files
+    NixIO,
+    NixIOFr,
     NeoMatlabIO,
     NestIO,
     NeuralynxIO,
@@ -384,14 +387,86 @@
     WinWcpIO
 ]
 
+# for each supported extension list the ios supporting it
+io_by_extension = {}
+for current_io in iolist:  # do not use `io` as variable name here as this overwrites the module io
+    for extension in current_io.extensions:
+        extension = extension.lower()
+        # extension handling should not be case sensitive
+        io_by_extension.setdefault(extension, []).append(current_io)
 
-def get_io(filename, *args, **kwargs):
+
+def get_io(file_or_folder, *args, **kwargs):
     """
     Return a Neo IO instance, guessing the type based on the filename suffix.
     """
-    extension = os.path.splitext(filename)[1][1:]
-    for io in iolist:
-        if extension in io.extensions:
-            return io(filename, *args, **kwargs)
+    ios = list_candidate_ios(file_or_folder)
+    for io in ios:
+        try:
+            return io(file_or_folder, *args, **kwargs)
+        except:
+            continue
+
+    raise IOError(f"Could not identify IO for {file_or_folder}")
+
 
-    raise IOError("File extension %s not registered" % extension)
+def list_candidate_ios(file_or_folder, ignore_patterns=['*.ini', 'README.txt', 'README.md']):
+    """
+    Identify neo IO that can potentially load data in the file or folder
+
+    Parameters
+    ----------
+    file_or_folder (str, pathlib.Path)
+        Path to the file or folder to load
+    ignore_patterns (list)
+        List of patterns to ignore when scanning for known formats. See pathlib.PurePath.match().
+        Default: ['ini']
+
+    Returns
+    -------
+    list
+        List of neo io classes that are associated with the file extensions detected
+    """
+    file_or_folder = pathlib.Path(file_or_folder)
+
+    if file_or_folder.is_file():
+        suffix = file_or_folder.suffix[1:].lower()
+        if suffix not in io_by_extension:
+            raise ValueError(f'{suffix} is not a supported format of any IO.')
+        return io_by_extension[suffix]
+
+    elif file_or_folder.is_dir():
+        # scan files in folder to determine io type
+        filenames = [f for f in file_or_folder.glob('*') if f.is_file()]
+        # keep only relevant filenames
+        filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])]
+
+        # if no files are found in the folder, check subfolders
+        # this is necessary for nested-folder based formats like spikeglx
+        if not filenames:
+            filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()]
+            # keep only relevant filenames
+            filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])]
+
+    # if only file prefix was provided, e.g /mydatafolder/session1-
+    # to select all files sharing the `session1-` prefix
+    elif file_or_folder.parent.exists():
+        filenames = file_or_folder.parent.glob(file_or_folder.name + '*')
+
+    else:
+        raise ValueError(f'{file_or_folder} does not contain data files of a supported format')
+
+    # find the io that fits the best with the files contained in the folder
+    potential_ios = []
+    for filename in filenames:
+        for suffix in filename.suffixes:
+            suffix = suffix[1:].lower()
+            if suffix in io_by_extension:
+                potential_ios.extend(io_by_extension[suffix])
+
+    if not potential_ios:
+        raise ValueError(f'Could not determine IO to load {file_or_folder}')
+
+    # return ios ordered by number of files supported
+    counter = Counter(potential_ios).most_common()
+    return [io for io, count in counter]

neo/io/neuroshareapiio.py

@@ -72,7 +72,7 @@ class NeuroshareapiIO(BaseIO):
 
     name = "Neuroshare"
 
-    extensions = []
+    extensions = ['mcd']
 
     # This object operates on neuroshare files
     mode = "file"

neo/io/neurosharectypesio.py

@@ -108,7 +108,7 @@ class NeurosharectypesIO(BaseIO):
     write_params = None
 
     name = 'neuroshare'
-    extensions = []
+    extensions = ['mcd']
     mode = 'file'
 
     def __init__(self, filename='', dllname=''):

neo/rawio/__init__.py

@@ -235,7 +235,7 @@
 
 def get_rawio_class(filename_or_dirname):
     """
-    Return a neo.rawio class guess from file extention.
+    Return a neo.rawio class guess from file extension.
     """
     _, ext = os.path.splitext(filename_or_dirname)
     ext = ext[1:]

neo/rawio/alphaomegarawio.py

@@ -90,16 +90,18 @@ class AlphaOmegaRawIO(BaseRawIO):
     def __init__(self, dirname="", lsx_files=None, prune_channels=True):
         super().__init__(dirname=dirname)
         self.dirname = Path(dirname)
+
         self._lsx_files = lsx_files
         self._mpx_files = None
-        if self.dirname.is_dir():
-            self._explore_folder()
-        else:
-            self.logger.error(f"{self.dirname} is not a folder")
         self._prune_channels = prune_channels
         self._opened_files = {}
         self._ignore_unknown_datablocks = True  # internal debug property
 
+        if self.dirname.is_dir():
+            self._explore_folder()
+        else:
+            raise ValueError(f"{self.dirname} is not a folder")
+
     def _explore_folder(self):
         """
         If class was instantiated with lsx_files (list of .lsx files), load only

neo/rawio/axographrawio.py

@@ -218,7 +218,7 @@ class AxographRawIO(BaseRawIO):
     """
     name = 'AxographRawIO'
     description = 'This IO reads .axgd/.axgx files created with AxoGraph'
-    extensions = ['axgd', 'axgx']
+    extensions = ['axgd', 'axgx', '']
     rawmode = 'one-file'
 
     def __init__(self, filename, force_single_segment=False):

neo/rawio/biocamrawio.py

@@ -29,7 +29,7 @@ class BiocamRawIO(BaseRawIO):
         >>> float_chunk = r.rescale_signal_raw_to_float(raw_chunk, dtype='float64',
                                                         channel_indexes=[0, 3, 6])
     """
-    extensions = ['h5']
+    extensions = ['h5', 'brw']
     rawmode = 'one-file'
 
     def __init__(self, filename=''):

neo/rawio/blackrockrawio.py

@@ -122,7 +122,7 @@ class BlackrockRawIO(BaseRawIO):
     """
 
     extensions = ['ns' + str(_) for _ in range(1, 7)]
-    extensions.extend(['nev', ])  # 'sif', 'ccf' not yet supported
+    extensions.extend(['nev', 'sif', 'ccf'])  # 'sif', 'ccf' not yet supported
     rawmode = 'multi-file'
 
     def __init__(self, filename=None, nsx_override=None, nev_override=None,
@@ -154,6 +154,9 @@ def __init__(self, filename=None, nsx_override=None, nev_override=None,
         else:
             self._filenames['nev'] = self.filename
 
+        self._filenames['sif'] = self.filename
+        self._filenames['ccf'] = self.filename
+
         # check which files are available
         self._avail_files = dict.fromkeys(self.extensions, False)
         self._avail_nsx = []

neo/rawio/neuralynxrawio/neuralynxrawio.py

@@ -75,7 +75,7 @@ class NeuralynxRawIO(BaseRawIO):
 
             Display all information about signal channels, units, segment size....
     """
-    extensions = ['nse', 'ncs', 'nev', 'ntt']
+    extensions = ['nse', 'ncs', 'nev', 'ntt', 'nvt', 'nrd']  # nvt and nrd are not yet supported
     rawmode = 'one-dir'
 
     _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'),

neo/rawio/nixrawio.py

@@ -31,7 +31,7 @@
 
 class NIXRawIO(BaseRawIO):
 
-    extensions = ['nix']
+    extensions = ['nix', 'h5']
     rawmode = 'one-file'
 
     def __init__(self, filename=''):