Merge branch 'NeuralEnsemble:master' into cboulay/br_filespec_3_0

Author: cboulay

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,28 @@
+---
+name: Bug report
+about: Create a report to help us fix problems
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behaviour, preferably providing a simple code example (if the error happens in the middle of some complex code, please try to find a simpler, minimal example that demonstrates the error), and showing the full traceback.
+
+If the error occurs when reading a file that you can't share publicly, please let us know, and we'll get in touch to discuss sharing it privately.
+
+**Expected behaviour**
+If the bug is incorrect behaviour, rather than an unexpected Exception, please give a clear and concise description of what you expected to happen.
+
+**Environment:**
+ - OS: [e.g. macOS, Linux, Windows]
+ - Python version
+ - Neo version
+ - NumPy version
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/confusing-documentation.md

@@ -0,0 +1,17 @@
+---
+name: Confusing documentation
+about: Let us know if the documentation is confusing or incorrect
+title: ''
+labels: Documentation
+assignees: ''
+
+---
+
+**Which page is the problem on?**
+The URL of the documentation page where the problem is, and either copy-paste the confusing text (for a short section of text), or give the first few and last few words (for a long section).
+
+**What is the problem?**
+Is the documentation (a) confusing or (b) incorrect? In what way?
+
+**Suggestions for fixing the problem**
+If the documentation is confusing, can you suggest an improvement? If the documentation is incorrect, what should it say instead?

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea to improve Neo
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

neo/core/epoch.py

@@ -6,6 +6,7 @@
 '''
 
 from copy import deepcopy, copy
+from numbers import Number
 
 import numpy as np
 import quantities as pq
@@ -53,10 +54,12 @@ class Epoch(DataObject):
     *Required attributes/properties*:
         :times: (quantity array 1D, numpy array 1D or list) The start times
            of each time period.
-        :durations: (quantity array 1D, numpy array 1D, list, or quantity scalar)
+        :durations: (quantity array 1D, numpy array 1D, list, quantity scalar or float)
            The length(s) of each time period.
-           If a scalar, the same value is used for all time periods.
+           If a scalar/float, the same value is used for all time periods.
         :labels: (numpy.array 1D dtype='U' or list) Names or labels for the time periods.
+        :units: (quantity units or str) Required if the times is a list or NumPy
+                array, not if it is a :class:`Quantity`
 
     *Recommended attributes/properties*:
         :name: (str) A label for the dataset,
@@ -88,8 +91,10 @@ def __new__(cls, times=None, durations=None, labels=None, units=None, name=None,
             raise ValueError("Times array has more than 1 dimension")
         if isinstance(durations, (list, tuple)):
             durations = np.array(durations)
-        if durations is None:
+        elif durations is None:
             durations = np.array([]) * pq.s
+        elif isinstance(durations, Number):
+            durations = durations * np.ones(times.shape)
         elif durations.size != times.size:
             if durations.size == 1:
                 durations = durations * np.ones_like(times.magnitude)

neo/io/phyio.py

@@ -7,6 +7,9 @@ class PhyIO(PhyRawIO, BaseFromRaw):
     description = "Phy IO"
     mode = 'dir'
 
-    def __init__(self, dirname):
-        PhyRawIO.__init__(self, dirname=dirname)
+    def __init__(self, dirname, load_amplitudes=False, load_pcs=False):
+        PhyRawIO.__init__(self,
+                          dirname=dirname,
+                          load_amplitudes=load_amplitudes,
+                          load_pcs=load_pcs)
         BaseFromRaw.__init__(self, dirname)

neo/rawio/blackrockrawio.py

@@ -458,8 +458,7 @@ def _parse_header(self):
             # Find maximal and minimal time for each nev segment
             for k, (data, ev_ids) in self.nev_data.items():
                 for i in np.unique(ev_ids):
-                    mask = [ev_ids == i]
-                    curr_data = data[mask]
+                    curr_data = data[ev_ids == i]
                     if curr_data.size > 0:
                         if max(curr_data['timestamp']) >= max_nev_times.get(i, 0):
                             max_nev_times[i] = max(curr_data['timestamp'])

neo/rawio/phyrawio.py

@@ -16,6 +16,7 @@
 import re
 import csv
 import ast
+import warnings
 
 
 class PhyRawIO(BaseRawIO):
@@ -35,9 +36,11 @@ class PhyRawIO(BaseRawIO):
     extensions = []
     rawmode = 'one-dir'
 
-    def __init__(self, dirname=''):
+    def __init__(self, dirname='', load_amplitudes=False, load_pcs=False):
         BaseRawIO.__init__(self)
         self.dirname = dirname
+        self.load_pcs = load_pcs
+        self.load_amplitudes = load_amplitudes
 
     def _source_name(self):
         return self.dirname
@@ -53,16 +56,24 @@ def _parse_header(self):
         else:
             self._spike_clusters = self._spike_templates
 
-        # TODO: Add this when array_annotations are ready
-        # if (phy_folder / 'amplitudes.npy').is_file():
-        #     amplitudes = np.squeeze(np.load(phy_folder / 'amplitudes.npy'))
-        # else:
-        #     amplitudes = np.ones(len(spike_times))
-        #
-        # if (phy_folder / 'pc_features.npy').is_file():
-        #     pc_features = np.squeeze(np.load(phy_folder / 'pc_features.npy'))
-        # else:
-        #     pc_features = None
+        self._amplitudes = None
+        if self.load_amplitudes:
+            if (phy_folder / 'amplitudes.npy').is_file():
+                self._amplitudes = np.squeeze(np.load(phy_folder / 'amplitudes.npy'))
+            else:
+                warnings.warn('Amplitudes requested but "amplitudes.npy"'
+                              'not found in the data folder.')
+
+        self._pc_features = None
+        self._pc_feature_ind = None
+        if self.load_pcs:
+            if ((phy_folder / 'pc_features.npy').is_file()
+                    and (phy_folder / 'pc_feature_ind.npy').is_file()):
+                self._pc_features = np.squeeze(np.load(phy_folder / 'pc_features.npy'))
+                self._pc_feature_ind = np.squeeze(np.load(phy_folder / 'pc_feature_ind.npy'))
+            else:
+                warnings.warn('PCs requested but "pc_features.npy" and/or'
+                              '"pc_feature_ind.npy" not found in the data folder.')
 
         # SEE: https://stackoverflow.com/questions/4388626/
         #  python-safe-eval-string-to-bool-int-float-none-string
@@ -138,17 +149,41 @@ def _parse_header(self):
 
             # Loop over list of list of dict and annotate each st
             for annotation_list in annotation_lists:
-                clust_key, property_name = tuple(annotation_list[0].
-                                                 keys())
-                if property_name == 'KSLabel':
-                    annotation_name = 'quality'
-                else:
-                    annotation_name = property_name.lower()
-                for annotation_dict in annotation_list:
-                    if int(annotation_dict[clust_key]) == clust_id:
-                        spiketrain_an[annotation_name] = \
-                            annotation_dict[property_name]
-                        break
+                clust_key, *property_names = tuple(annotation_list[0].keys())
+                for property_name in property_names:
+                    if property_name == 'KSLabel':
+                        annotation_name = 'quality'
+                    else:
+                        annotation_name = property_name.lower()
+                    for annotation_dict in annotation_list:
+                        if int(annotation_dict[clust_key]) == clust_id:
+                            spiketrain_an[annotation_name] = \
+                                annotation_dict[property_name]
+                            break
+
+            cluster_mask = (self._spike_clusters == clust_id).flatten()
+
+            current_templates = self._spike_templates[cluster_mask].flatten()
+            unique_templates = np.unique(current_templates)
+            spiketrain_an['templates'] = unique_templates
+            spiketrain_an['__array_annotations__']['templates'] = current_templates
+
+            if self._amplitudes is not None:
+                spiketrain_an['__array_annotations__']['amplitudes'] = \
+                    self._amplitudes[cluster_mask]
+
+            if self._pc_features is not None:
+                current_pc_features = self._pc_features[cluster_mask]
+                _, num_pcs, num_pc_channels = current_pc_features.shape
+                for pc_idx in range(num_pcs):
+                    for channel_idx in range(num_pc_channels):
+                        key = 'channel{channel_idx}_pc{pc_idx}'.format(channel_idx=channel_idx,
+                                                                       pc_idx=pc_idx)
+                        spiketrain_an['__array_annotations__'][key] = \
+                                current_pc_features[:, pc_idx, channel_idx]
+
+            if self._pc_feature_ind is not None:
+                spiketrain_an['pc_feature_ind'] = self._pc_feature_ind[unique_templates]
 
     def _segment_t_start(self, block_index, seg_index):
         assert block_index == 0
@@ -221,7 +256,7 @@ def _rescale_epoch_duration(self, raw_duration, dtype):
     def _parse_tsv_or_csv_to_list_of_dict(filename):
         list_of_dict = list()
         letter_pattern = re.compile('[a-zA-Z]')
-        float_pattern = re.compile(r'\d*\.')
+        float_pattern = re.compile(r'-?\d*\.')
         with open(filename) as csvfile:
             if filename.suffix == '.csv':
                 reader = csv.DictReader(csvfile, delimiter=',')
@@ -233,15 +268,19 @@ def _parse_tsv_or_csv_to_list_of_dict(filename):
 
             for row in reader:
                 if line == 0:
-                    key1, key2 = tuple(row.keys())
+                    cluster_id_key, *annotation_keys = tuple(row.keys())
                 # Convert cluster ID to int
-                row[key1] = int(row[key1])
+                row[cluster_id_key] = int(row[cluster_id_key])
                 # Convert strings without letters
-                if letter_pattern.match(row[key2]) is None:
-                    if float_pattern.match(row[key2]) is None:
-                        row[key2] = int(row[key2])
-                    else:
-                        row[key2] = float(row[key2])
+                for key in annotation_keys:
+                    value = row[key]
+                    if not len(value):
+                        row[key] = None
+                    elif letter_pattern.match(value) is None:
+                        if float_pattern.match(value) is None:
+                            row[key] = int(value)
+                        else:
+                            row[key] = float(value)
 
                 list_of_dict.append(row)
                 line += 1

neo/test/rawiotest/test_phyrawio.py

@@ -31,26 +31,35 @@ def test_csv_tsv_parser_with_csv(self):
         csv_tempfile = Path(tempfile.gettempdir()).joinpath('test.csv')
         with open(csv_tempfile, 'w') as csv_file:
             csv_writer = csv.writer(csv_file, delimiter=',')
-            csv_writer.writerow(['cluster_id', 'some_annotation'])
-            csv_writer.writerow([1, 'Good'])
-            csv_writer.writerow([2, 10])
-            csv_writer.writerow([3, 1.23])
+            csv_writer.writerow(['cluster_id', 'some_annotation', 'some_other_annotation'])
+            csv_writer.writerow([1, 'Good', 'Bad'])
+            csv_writer.writerow([2, 10, -2])
+            csv_writer.writerow([3, 1.23, -0.38])
 
         # the parser in PhyRawIO runs csv.DictReader to parse the file
         # csv.DictReader for python version 3.6+ returns list of OrderedDict
         if (3, 6) <= sys.version_info < (3, 8):
             target = [OrderedDict({'cluster_id': 1,
-                                   'some_annotation': 'Good'}),
+                                   'some_annotation': 'Good',
+                                   'some_other_annotation': 'Bad'}),
                       OrderedDict({'cluster_id': 2,
-                                   'some_annotation': 10}),
+                                   'some_annotation': 10,
+                                   'some_other_annotation': -2}),
                       OrderedDict({'cluster_id': 3,
-                                   'some_annotation': 1.23})]
+                                   'some_annotation': 1.23,
+                                   'some_other_annotation': -0.38})]
 
         # csv.DictReader for python version 3.8+ returns list of dict
         elif sys.version_info >= (3, 8):
-            target = [{'cluster_id': 1, 'some_annotation': 'Good'},
-                      {'cluster_id': 2, 'some_annotation': 10},
-                      {'cluster_id': 3, 'some_annotation': 1.23}]
+            target = [{'cluster_id': 1,
+                       'some_annotation': 'Good',
+                       'some_other_annotation': 'Bad'},
+                      {'cluster_id': 2,
+                       'some_annotation': 10,
+                       'some_other_annotation': -2},
+                      {'cluster_id': 3,
+                       'some_annotation': 1.23,
+                       'some_other_annotation': -0.38}]
 
         list_of_dict = PhyRawIO._parse_tsv_or_csv_to_list_of_dict(csv_tempfile)