Merge branch 'main' of https://github.com/datajoint/element-array-ephys into no-curation

Author: Thinh Nguyen

.gitignore

@@ -1,3 +1,6 @@
+# User data
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,5 +1,4 @@
 # DataJoint Element - Array Electrophysiology Element
-DataJoint Element for array electrophysiology.
 
 This repository features DataJoint pipeline design for extracellular array electrophysiology, 
 with ***Neuropixels*** probe and ***kilosort*** spike sorting method. 
@@ -13,12 +12,16 @@ ephys pipeline.
 
 See [Background](Background.md) for the background information and development timeline.
 
-## The Pipeline Architecture
+## Element architecture
 
 ![element-array-ephys diagram](images/attached_array_ephys_element.svg)
 
 As the diagram depicts, the array ephys element starts immediately downstream from ***Session***, 
-and also requires some notion of ***Location*** as a dependency for ***InsertionLocation***.
+and also requires some notion of ***Location*** as a dependency for ***InsertionLocation***. We 
+provide an [example workflow](https://github.com/datajoint/workflow-array-ephys/) with a 
+[pipeline script](https://github.com/datajoint/workflow-array-ephys/blob/main/workflow_array_ephys/pipeline.py)
+that models (a) combining this Element with the corresponding [Element-Session](https://github.com/datajoint/element-session)
+, and (b) declaring a ***SkullReference*** table to provide Location.
 
 ### The design of probe
 
@@ -45,14 +48,24 @@ This ephys element features automatic ingestion for spike sorting results from t
     + ***WaveformSet*** - A set of spike waveforms for units from a given CuratedClustering
 
 ## Installation
-```
-pip install element-array-ephys
-```
 
-If you already have an older version of ***element-array-ephys*** installed using `pip`, upgrade with
-```
-pip install --upgrade element-array-ephys
-```
++ Install `element-array-ephys`
+    ```
+    pip install element-array-ephys
+    ```
+
++ Upgrade `element-array-ephys` previously installed with `pip`
+    ```
+    pip install --upgrade element-array-ephys
+    ```
+
++ Install `element-interface`
+
+    + `element-interface` is a dependency of `element-array-ephys`, however it is not contained within `requirements.txt`.
+     
+    ```
+    pip install "element-interface @ git+https://github.com/datajoint/element-interface"
+    ```
 
 ## Usage
 
@@ -65,12 +78,12 @@ To activate the `element-array-ephys`, ones need to provide:
     + schema name for the ephys module
 
 2. Upstream tables
-    + Session table
-    + SkullReference table (Reference table for InsertionLocation, specifying the skull reference)
+    + Session table: A set of keys identifying a recording session (see [Element-Session](https://github.com/datajoint/element-session)).
+    + SkullReference table: A reference table for InsertionLocation, specifying the skull reference (see [example pipeline](https://github.com/datajoint/workflow-array-ephys/blob/main/workflow_array_ephys/pipeline.py)).
 
-3. Utility functions
-    + get_ephys_root_data_dir()
-    + get_session_directory()
+3. Utility functions. See [example definitions here](https://github.com/datajoint/workflow-array-ephys/blob/main/workflow_array_ephys/paths.py)
+    + get_ephys_root_data_dir(): Returns your root data directory.
+    + get_session_directory(): Returns the path of the session data relative to the root.
 
 For more detail, check the docstring of the `element-array-ephys`:
 

element_array_ephys/__init__.py

@@ -1,7 +1,4 @@
 import datajoint as dj
-import pathlib
-import uuid
-import hashlib
 import logging
 import os
 
@@ -12,74 +9,4 @@
 def get_logger(name):
     log = logging.getLogger(name)
     log.setLevel(os.getenv('LOGLEVEL', 'INFO'))
-    return log
-
-
-def find_full_path(root_directories, relative_path):
-    """
-    Given a relative path, search and return the full-path
-     from provided potential root directories (in the given order)
-        :param root_directories: potential root directories
-        :param relative_path: the relative path to find the valid root directory
-        :return: full-path (pathlib.Path object)
-    """
-    relative_path = _to_Path(relative_path)
-
-    if relative_path.exists():
-        return relative_path
-
-    # turn to list if only a single root directory is provided
-    if isinstance(root_directories, (str, pathlib.Path)):
-        root_directories = [_to_Path(root_directories)]
-
-    for root_dir in root_directories:
-        if (_to_Path(root_dir) / relative_path).exists():
-            return _to_Path(root_dir) / relative_path
-
-    raise FileNotFoundError('No valid full-path found (from {})'
-                            ' for {}'.format(root_directories, relative_path))
-
-
-def find_root_directory(root_directories, full_path):
-    """
-    Given multiple potential root directories and a full-path,
-    search and return one directory that is the parent of the given path
-        :param root_directories: potential root directories
-        :param full_path: the full path to search the root directory
-        :return: root_directory (pathlib.Path object)
-    """
-    full_path = _to_Path(full_path)
-
-    if not full_path.exists():
-        raise FileNotFoundError(f'{full_path} does not exist!')
-
-    # turn to list if only a single root directory is provided
-    if isinstance(root_directories, (str, pathlib.Path)):
-        root_directories = [_to_Path(root_directories)]
-
-    try:
-        return next(_to_Path(root_dir) for root_dir in root_directories
-                    if _to_Path(root_dir) in set(full_path.parents))
-
-    except StopIteration:
-        raise FileNotFoundError('No valid root directory found (from {})'
-                                ' for {}'.format(root_directories, full_path))
-
-
-def _to_Path(path):
-    """
-    Convert the input "path" into a pathlib.Path object
-    Handles one odd Windows/Linux incompatibility of the "\\"
-    """
-    return pathlib.Path(str(path).replace('\\', '/'))
-
-
-def dict_to_uuid(key):
-    """
-    Given a dictionary `key`, returns a hash string as UUID
-    """
-    hashed = hashlib.md5()
-    for k, v in sorted(key.items()):
-        hashed.update(str(k).encode())
-        hashed.update(str(v).encode())
-    return uuid.UUID(hex=hashed.hexdigest())
+    return log

element_array_ephys/ephys.py

@@ -6,8 +6,11 @@
 import importlib
 from decimal import Decimal
 
+from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
+
 from .readers import spikeglx, kilosort, openephys
-from . import probe, find_full_path, find_root_directory, dict_to_uuid, get_logger
+from . import probe, get_logger
+
 
 log = get_logger(__name__)
 
@@ -52,7 +55,6 @@ def activate(ephys_schema_name, probe_schema_name=None, *, create_schema=True,
     global _linking_module
     _linking_module = linking_module
 
-    # activate
     probe.activate(probe_schema_name, create_schema=create_schema,
                    create_tables=create_tables)
     schema.activate(ephys_schema_name, create_schema=create_schema,
@@ -63,9 +65,10 @@ def activate(ephys_schema_name, probe_schema_name=None, *, create_schema=True,
 
 def get_ephys_root_data_dir() -> list:
     """
-    All data paths, directories in DataJoint Elements are recommended to be stored as
-    relative paths, with respect to some user-configured "root" directory,
-     which varies from machine to machine (e.g. different mounted drive locations)
+    All data paths, directories in DataJoint Elements are recommended to be 
+    stored as relative paths, with respect to some user-configured "root" 
+    directory, which varies from machine to machine (e.g. different mounted 
+    drive locations)
 
     get_ephys_root_data_dir() -> list
         This user-provided function retrieves the possible root data directories
@@ -91,7 +94,7 @@ def get_session_directory(session_key: dict) -> str:
         Retrieve the session directory containing the
          recorded Neuropixels data for a given Session
         :param session_key: a dictionary of one Session `key`
-        :return: a string for full path to the session directory
+        :return: a string for relative or full path to the session directory
     """
     return _linking_module.get_session_directory(session_key)
 
@@ -224,21 +227,22 @@ class EphysFile(dj.Part):
         """
 
     def make(self, key):
-        sess_dir = find_full_path(get_ephys_root_data_dir(),
+        session_dir = find_full_path(get_ephys_root_data_dir(),
                                   get_session_directory(key))
         inserted_probe_serial_number = (ProbeInsertion * probe.Probe & key).fetch1('probe')
 
         # search session dir and determine acquisition software
         for ephys_pattern, ephys_acq_type in zip(['*.ap.meta', '*.oebin'],
                                                  ['SpikeGLX', 'Open Ephys']):
-            ephys_meta_filepaths = list(sess_dir.rglob(ephys_pattern))
+            ephys_meta_filepaths = list(session_dir.rglob(ephys_pattern))
             if ephys_meta_filepaths:
                 acq_software = ephys_acq_type
                 break
         else:
             raise FileNotFoundError(
                 f'Ephys recording data not found!'
-                f' Neither SpikeGLX nor Open Ephys recording files found')
+                f' Neither SpikeGLX nor Open Ephys recording files found'
+                f' in {session_dir}')
 
         supported_probe_types = probe.ProbeType.fetch('probe_type')
 
@@ -277,12 +281,13 @@ def make(self, key):
                 'recording_duration': (spikeglx_meta.recording_duration
                                        or spikeglx.retrieve_recording_duration(meta_filepath))})
 
-            root_dir = find_root_directory(get_ephys_root_data_dir(), meta_filepath)
+            root_dir = find_root_directory(get_ephys_root_data_dir(), 
+                                           meta_filepath)
             self.EphysFile.insert1({
                 **key,
                 'file_path': meta_filepath.relative_to(root_dir).as_posix()})
         elif acq_software == 'Open Ephys':
-            dataset = openephys.OpenEphys(sess_dir)
+            dataset = openephys.OpenEphys(session_dir)
             for serial_number, probe_data in dataset.probes.items():
                 if str(serial_number) == inserted_probe_serial_number:
                     break
@@ -313,8 +318,7 @@ def make(self, key):
                 'recording_datetime': probe_data.recording_info['recording_datetimes'][0],
                 'recording_duration': np.sum(probe_data.recording_info['recording_durations'])})
 
-            root_dir = find_root_directory(
-                get_ephys_root_data_dir(),
+            root_dir = find_root_directory(get_ephys_root_data_dir(),
                 probe_data.recording_info['recording_files'][0])
             self.EphysFile.insert([{**key,
                                     'file_path': fp.relative_to(root_dir).as_posix()}
@@ -661,16 +665,16 @@ class Curation(dj.Manual):
     curation_id: int
     ---
     curation_time: datetime             # time of generation of this set of curated clustering results 
-    curation_output_dir: varchar(255)   # output directory of the curated results, relative to clustering root data directory
+    curation_output_dir: varchar(255)   # output directory of the curated results, relative to root data directory
     quality_control: bool               # has this clustering result undergone quality control?
     manual_curation: bool               # has manual curation been performed on this clustering result?
     curation_note='': varchar(2000)  
     """
 
     def create1_from_clustering_task(self, key, curation_note=''):
         """
-        A convenient function to create a new corresponding "Curation"
-         for a particular "ClusteringTask"
+        A function to create a new corresponding "Curation" for a particular 
+        "ClusteringTask"
         """
         if key not in Clustering():
             raise ValueError(f'No corresponding entry in Clustering available'
@@ -684,8 +688,10 @@ def create1_from_clustering_task(self, key, curation_note=''):
         # Synthesize curation_id
         curation_id = dj.U().aggr(self & key, n='ifnull(max(curation_id)+1,1)').fetch1('n')
         self.insert1({**key, 'curation_id': curation_id,
-                      'curation_time': creation_time, 'curation_output_dir': output_dir,
-                      'quality_control': is_qc, 'manual_curation': is_curated,
+                      'curation_time': creation_time, 
+                      'curation_output_dir': output_dir,
+                      'quality_control': is_qc, 
+                      'manual_curation': is_curated,
                       'curation_note': curation_note})
 
 
@@ -835,9 +841,9 @@ def yield_unit_waveforms():
                 spikeglx_meta_filepath = get_spikeglx_meta_filepath(key)
                 neuropixels_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent)
             elif acq_software == 'Open Ephys':
-                sess_dir = find_full_path(get_ephys_root_data_dir(),
-                                          get_session_directory(key))
-                openephys_dataset = openephys.OpenEphys(sess_dir)
+                session_dir = find_full_path(get_ephys_root_data_dir(), 
+                                             get_session_directory(key))
+                openephys_dataset = openephys.OpenEphys(session_dir)
                 neuropixels_recording = openephys_dataset.probes[probe_serial_number]
 
             def yield_unit_waveforms():
@@ -884,12 +890,13 @@ def get_spikeglx_meta_filepath(ephys_recording_key):
     except FileNotFoundError:
         # if not found, search in session_dir again
         if not spikeglx_meta_filepath.exists():
-            sess_dir = find_full_path(get_ephys_root_data_dir(),
-                                      get_session_directory(ephys_recording_key))
+            session_dir = find_full_path(get_ephys_root_data_dir(), 
+                                         get_session_directory(
+                                             ephys_recording_key))
             inserted_probe_serial_number = (ProbeInsertion * probe.Probe
                                             & ephys_recording_key).fetch1('probe')
 
-            spikeglx_meta_filepaths = [fp for fp in sess_dir.rglob('*.ap.meta')]
+            spikeglx_meta_filepaths = [fp for fp in session_dir.rglob('*.ap.meta')]
             for meta_filepath in spikeglx_meta_filepaths:
                 spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath)
                 if str(spikeglx_meta.probe_SN) == inserted_probe_serial_number: