Merge branch 'main' of https://github.com/datajoint/elements-ephys into main

Author: dimitri-yatsenko

elements_ephys/__init__.py

@@ -1 +1,5 @@
-from .ephys import schema as ephys
+__author__ = "DataJoint NEURO"
+__date__ = "December 15, 2020"
+__version__ = "0.0.1"
+
+__all__ = ['__author__', '__version__', '__date__']

elements_ephys/ephys.py

@@ -14,68 +14,65 @@
 
 
 def activate(ephys_schema_name, probe_schema_name=None, create_schema=True, create_tables=True, add_objects=None):
-    upstream_tables = ("Session", "SkullReference")
     assert isinstance(add_objects, Mapping)
+
+    upstream_tables = ("Session", "SkullReference")
     try:
         raise RuntimeError("Table %s is required for module ephys" % next(
             name for name in upstream_tables
-            if not isinstance(add_objects.get(name, None), (dj.Manual, dj.Lookup, dj.Imported, dj.Computed))))
+            if not isinstance(add_objects.get(name, None), (dj.Manual, dj.Lookup, dj.Imported, dj.Computed, dj.user_tables.OrderedClass))))
     except StopIteration:
         pass  # all ok
 
     required_functions = ("get_neuropixels_data_directory", "get_paramset_idx", "get_kilosort_output_directory")
-    assert isinstance(add_objects, Mapping)
     try:
         raise RuntimeError("Function %s is required for module ephys" % next(
             name for name in required_functions
             if not inspect.isfunction(add_objects.get(name, None))))
     except StopIteration:
         pass  # all ok
 
-    if not probe.schema.is_activated:
+    if not probe.schema.database:
         probe.schema.activate(probe_schema_name or ephys_schema_name,
                               create_schema=create_schema, create_tables=create_tables)
+
     schema.activate(ephys_schema_name, create_schema=create_schema,
                     create_tables=create_tables, add_objects=add_objects)
 
 
 # REQUIREMENTS: The workflow module must define these functions ---------------
 
 
-def get_neuropixels_data_directory():
-    return None
+def get_neuropixels_data_directory(probe_insertion_key: dict) -> str:
+    """
+    Retrieve the recorded Neuropixels data directory for a given ProbeInsertion
+    :param probe_insertion_key: a dictionary of one ProbeInsertion `key`
+    :return: a string for full path to the resulting Neuropixels data directory
+    """
+    assert set(ProbeInsertion().primary_key) <= set(probe_insertion_key)
+    raise NotImplementedError('Workflow module should define function: "get_neuropixels_data_directory"')
 
 
 def get_kilosort_output_directory(clustering_task_key: dict) -> str:
     """
     Retrieve the Kilosort output directory for a given ClusteringTask
-    :param clustering_task_key: a dictionary of one EphysRecording
+    :param clustering_task_key: a dictionary of one ClusteringTask `key`
     :return: a string for full path to the resulting Kilosort output directory
     """
     assert set(EphysRecording().primary_key) <= set(clustering_task_key)
-    raise NotImplementedError('Workflow module should define')
+    raise NotImplementedError('Workflow module should define function: "get_kilosort_output_directory"')
 
 
 def get_paramset_idx(ephys_rec_key: dict) -> int:
     """
-    Retrieve attribute `paramset_idx` from the ClusteringParamSet record for the given EphysRecording key.
-    :param ephys_rec_key: a dictionary of one EphysRecording
+    Retrieve attribute `paramset_idx` from the ClusteringParamSet record for the given EphysRecording.
+    :param ephys_rec_key: a dictionary of one EphysRecording `key`
     :return: int specifying the `paramset_idx`
     """
     assert set(EphysRecording().primary_key) <= set(ephys_rec_key)
-    raise NotImplementedError('Workflow module should define')
+    raise NotImplementedError('Workflow module should define function: get_paramset_idx')
 
 
-def dict_to_uuid(key):
-    """
-    Given a dictionary `key`, returns a hash string
-    """
-    hashed = hashlib.md5()
-    for k, v in sorted(key.items()):
-        hashed.update(str(k).encode())
-        hashed.update(str(v).encode())
-    return uuid.UUID(hex=hashed.hexdigest())
-
 # ===================================== Probe Insertion =====================================
 
 
@@ -109,17 +106,13 @@ class InsertionLocation(dj.Manual):
 
 
 # ===================================== Ephys Recording =====================================
-# The abstract function _get_neuropixels_data_directory() should expect one argument in the form of a
-# dictionary with the keys from user-defined Subject and Session, as well as
-# "insertion_number" (as int) based on the "ProbeInsertion" table definition in this djephys
-
 
 @schema
 class EphysRecording(dj.Imported):
     definition = """
     -> ProbeInsertion      
     ---
-    -> ElectrodeConfig
+    -> probe.ElectrodeConfig
     sampling_rate: float # (Hz) 
     """
 
@@ -143,7 +136,7 @@ def make(self, key):
                 neuropixels_meta.probe_model))
 
         # ---- compute hash for the electrode config (hash of dict of all ElectrodeConfig.Electrode) ----
-        ec_hash = uuid.UUID(dict_to_uuid({k['electrode']: k for k in eg_members}))
+        ec_hash = dict_to_uuid({k['electrode']: k for k in eg_members})
 
         el_list = sorted([k['electrode'] for k in eg_members])
         el_jumps = [-1] + np.where(np.diff(el_list) > 1)[0].tolist() + [len(el_list) - 1]
@@ -185,7 +178,7 @@ class Electrode(dj.Part):
         """
 
     def make(self, key):
-        neuropixels_dir = EphysRecording._get_neuropixels_data_directory(key)
+        neuropixels_dir = get_neuropixels_data_directory(key)
         neuropixels_recording = neuropixels.Neuropixels(neuropixels_dir)
 
         lfp = neuropixels_recording.lfdata[:, :-1].T  # exclude the sync channel
@@ -201,7 +194,7 @@ def make(self, key):
         q_electrodes = probe.ProbeType.Electrode * probe.ElectrodeConfig.Electrode & key
         electrodes = []
         for recorded_site in np.arange(lfp.shape[0]):
-            shank, shank_col, shank_row, _ = neuropixels_recording.neuropixels_meta.shankmap['data'][recorded_site]
+            shank, shank_col, shank_row, _ = neuropixels_recording.npx_meta.shankmap['data'][recorded_site]
             electrodes.append((q_electrodes
                                & {'shank': shank,
                                   'shank_col': shank_col,
@@ -269,14 +262,15 @@ class ClusteringTask(dj.Imported):
     """
 
     def make(self, key):
-        key['paramset_idx'] = ClusteringTask._get_paramset_idx(key)
-        self.insert1(key)
+        key['paramset_idx'] = get_paramset_idx(key)
 
-        data_dir = pathlib.Path(ClusteringTask._get_ks_data_dir(key))
+        data_dir = pathlib.Path(get_kilosort_output_directory(key))
         if not data_dir.exists():
             # this is where we can trigger the clustering job
             print(f'Clustering results not yet found for {key} - Triggering kilosort not yet implemented!')
 
+        self.insert1(key)
+
 
 @schema
 class ClusterQualityLabel(dj.Lookup):
@@ -319,7 +313,7 @@ class Unit(dj.Part):
         """
 
     def make(self, key):
-        ks_dir = ClusteringTask._get_ks_data_dir(key)
+        ks_dir = get_kilosort_output_directory(key)
         ks = kilosort.Kilosort(ks_dir)
         # ---------- Clustering ----------
         creation_time, is_curated, is_qc = kilosort.extract_clustering_info(ks_dir)
@@ -462,7 +456,7 @@ def make(self, key):
 
 
 def get_neuropixels_chn2electrode_map(ephys_recording_key):
-    neuropixels_dir = EphysRecording._get_neuropixels_data_directory(ephys_recording_key)
+    neuropixels_dir = get_neuropixels_data_directory(ephys_recording_key)
     meta_filepath = next(pathlib.Path(neuropixels_dir).glob('*.ap.meta'))
     neuropixels_meta = neuropixels.NeuropixelsMeta(meta_filepath)
     e_config_key = (EphysRecording * probe.ElectrodeConfig & ephys_recording_key).fetch1('KEY')
@@ -475,3 +469,14 @@ def get_neuropixels_chn2electrode_map(ephys_recording_key):
                                                'shank_col': shank_col,
                                                'shank_row': shank_row}).fetch1('KEY')
     return chn2electrode_map
+
+
+def dict_to_uuid(key):
+    """
+    Given a dictionary `key`, returns a hash string
+    """
+    hashed = hashlib.md5()
+    for k, v in sorted(key.items()):
+        hashed.update(str(k).encode())
+        hashed.update(str(v).encode())
+    return uuid.UUID(hex=hashed.hexdigest())

elements_ephys/probe.py

@@ -10,6 +10,7 @@
 def activate(schema_name, create_schema=True, create_tables=True):
     schema.activate(schema_name, create_schema=create_schema, create_tables=create_tables)
 
+
 @schema
 class ProbeType(dj.Lookup):
     definition = """

elements_ephys/readers/kilosort.py

@@ -4,22 +4,11 @@
 import numpy as np
 import re
 import logging
+from .utils import convert_to_number
 
 log = logging.getLogger(__name__)
 
 
-def convert_to_number(value: str):
-    if isinstance(value, str):
-        try:
-            value = int(value)
-        except ValueError:
-            try:
-                value = float(value)
-            except ValueError:
-                pass
-    return value
-
-
 class Kilosort:
 
     ks_files = [
@@ -85,7 +74,7 @@ def _stat(self):
                 prm = {}
                 for line in open(f, 'r').readlines():
                     k, v = line.strip('\n').split('=')
-                    prm[k.strip()] = handle_string(v.strip())
+                    prm[k.strip()] = convert_to_number(v.strip())
                 log.debug('prm: {}'.format(prm))
                 self._data[base] = prm
 

elements_ephys/readers/neuropixels.py

@@ -1,18 +1,7 @@
 from datetime import datetime
 import numpy as np
 import pathlib
-
-
-def convert_to_number(value: str):
-    if isinstance(value, str):
-        try:
-            value = int(value)
-        except ValueError:
-            try:
-                value = float(value)
-            except ValueError:
-                pass
-    return value
+from .utils import convert_to_number
 
 
 class Neuropixels:

elements_ephys/readers/utils.py

@@ -0,0 +1,12 @@
+
+
+def convert_to_number(value: str):
+    if isinstance(value, str):
+        try:
+            value = int(value)
+        except ValueError:
+            try:
+                value = float(value)
+            except ValueError:
+                pass
+    return value

requirements.txt

@@ -1,2 +1 @@
-datajoint
-canonical-utils @ git+https://github.com/vathes/canonical-utils.git
+datajoint>=0.12.7

setup.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
 from setuptools import setup, find_packages
 from os import path
-import sys
 
 here = path.abspath(path.dirname(__file__))