save electrodes files and channels in each collection on alignment resolved; allow single alignment resolution with manually resolved

Author: mayofaulkner

ibllib/qc/alignment_qc.py

@@ -3,12 +3,13 @@
 import numpy as np
 
 from ibllib.atlas import AllenAtlas
-from ibllib.atlas.regions import BrainRegions
 from ibllib.pipes import histology
 from ibllib.pipes.ephys_alignment import EphysAlignment
 from ibllib.qc import base
 from ibllib.oneibl.patcher import FTPPatcher
 from ibllib.qc.base import CRITERIA as CRITERIA_BASE
+from ibllib.io.spikeglx import _geometry_from_meta, read_meta_data
+from ibllib.ephys.neuropixel import trace_header
 
 _log = logging.getLogger('ibllib')
 CRITERIA = {"PASS": 0.8}
@@ -37,7 +38,7 @@ def __init__(self, probe_id, one=None, brain_atlas=None, channels=True, collecti
         # Get the brain atlas
         self.brain_atlas = brain_atlas or AllenAtlas(25)
         # Flag for uploading channels to alyx. For testing purposes
-        self.channels = channels
+        self.channels_flag = channels
 
         self.insertion = self.one.alyx.get(f'/insertions/{self.eid}', clobber=True)
         self.resolved = (self.insertion.get('json', {'temp': 0}).get('extended_qc').
@@ -66,14 +67,14 @@ def load_data(self, prev_alignments=None, xyz_picks=None, depths=None, cluster_c
         align_keys = [*self.alignments.keys()]
         self.align_keys_sorted = sorted(align_keys, reverse=True)
 
-        if len(self.alignments) < 2:
-            return
-
         if not np.any(xyz_picks):
             self.xyz_picks = np.array(self.insertion['json']['xyz_picks']) / 1e6
         else:
             self.xyz_picks = xyz_picks
 
+        if len(self.alignments) < 2:
+            return
+
         if not self.probe_collection:
             all_collections = self.one.list_collections(self.insertion['session'])
             if f'alf/{self.insertion["name"]}/pykilosort' in all_collections:
@@ -182,7 +183,11 @@ def resolve_manual(self, align_key, update=True, upload_alyx=True, upload_flatir
                           f"set 'force=True'")
             file_paths = []
         else:
-            results = self.compute_alignment_status()
+            if self.sim_matrix.shape[0] > 1:
+                results = self.compute_alignment_status()
+            else:
+                results = {'alignment_count': self.sim_matrix.shape[0]}
+
             results['alignment_resolved'] = True
             results['alignment_stored'] = align_key
             results['alignment_resolved_by'] = 'experimenter'
@@ -203,7 +208,7 @@ def compute_similarity_matrix(self):
         parent brain region
         """
 
-        r = BrainRegions()
+        r = self.brain_atlas.regions
 
         clusters = dict()
         for iK, key in enumerate(self.align_keys_sorted):
@@ -248,7 +253,7 @@ def compute_alignment_status(self):
         """
         # Set diagonals to zero so we don't use those to find max
         np.fill_diagonal(self.sim_matrix, 0)
-        # self.sim_matrix[self.sim_matrix == 1] = 0
+
         max_sim = np.max(self.sim_matrix)
 
         results = {'alignment_qc': max_sim,
@@ -260,14 +265,10 @@ def compute_alignment_status(self):
             results.update({'alignment_resolved': True})
             results.update({'alignment_resolved_by': 'qc'})
 
-            # outcome = 'PASS'
-
         else:
             results.update({'alignment_stored': self.align_keys_sorted[0]})
             results.update({'alignment_resolved': False})
 
-            # outcome = 'WARNING'
-
         return results
 
     def upload_channels(self, alignment_key, upload_alyx, upload_flatiron):
@@ -277,73 +278,89 @@ def upload_channels(self, alignment_key, upload_alyx, upload_flatiron):
 
         feature = np.array(self.alignments[alignment_key][0])
         track = np.array(self.alignments[alignment_key][1])
-        ephysalign = EphysAlignment(self.xyz_picks, self.depths,
+
+        try:
+            meta_dset = self.one.list_datasets(self.insertion['session'], '*ap.meta',
+                                               collection=f'raw_ephys_data/{self.insertion["name"]}')
+
+            meta_file = self.one.load_dataset(self.insertion['session'], meta_dset[0].split('/')[-1],
+                                              collection=f'raw_ephys_data/{self.insertion["name"]}',
+                                              download_only=True)
+            geometry = _geometry_from_meta(read_meta_data(meta_file))
+            chns = np.c_[geometry['x'], geometry['y']]
+        except Exception as err:
+            self.log.warning(f"Could not compute channel locations from meta file, errored with message: {err}. "
+                             f"Will use default Neuropixel 1 channels")
+            geometry = trace_header(version=1)
+            chns = np.c_[geometry['x'], geometry['y']]
+
+        ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1],
                                     track_prev=track,
                                     feature_prev=feature,
                                     brain_atlas=self.brain_atlas)
-
-        # Find the channels
         channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6)
-        channels_brainID = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
+        channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
 
-        # Find the clusters
-        r = BrainRegions()
-        clusters_mlapdv = channels_mlapdv[self.cluster_chns]
-        clusters_brainID = channels_brainID[self.cluster_chns]
-        clusters_brainAcro = r.get(ids=clusters_brainID).acronym
+        # Need to change channels stored on alyx as well as the stored key is not the same as the latest key
+        if upload_alyx:
+            if alignment_key != self.align_keys_sorted[0]:
+                histology.register_aligned_track(self.eid, channels_mlapdv / 1e6,
+                                                 chn_coords=chns, one=self.one,
+                                                 overwrite=True, channels=self.channels_flag)
+
+                ephys_traj = self.one.alyx.get(f'/trajectories?&probe_insertion={self.eid}'
+                                               '&provenance=Ephys aligned histology track',
+                                               clobber=True)
+                patch_dict = {'json': self.alignments}
+                self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'],
+                                   data=patch_dict)
 
-        # upload datasets to flatiron
         files_to_register = []
         if upload_flatiron:
             ftp_patcher = FTPPatcher(one=self.one)
-            insertion = self.one.alyx.get(f'/insertions/{self.eid}', clobber=True)
-            alf_path = self.one.eid2path(insertion['session']).joinpath(self.probe_collection)
+
+            alf_path = self.one.eid2path(self.insertion['session']).joinpath('alf', self.insertion["name"])
             alf_path.mkdir(exist_ok=True, parents=True)
 
-            # Make the channels.mlapdv dataset
-            f_name = alf_path.joinpath('channels.mlapdv.npy')
+            f_name = alf_path.joinpath('electrodeSites.mlapdv.npy')
             np.save(f_name, channels_mlapdv)
             files_to_register.append(f_name)
 
-            # Make the channels.brainLocationIds dataset
-            f_name = alf_path.joinpath('channels.brainLocationIds_ccf_2017.npy')
-            np.save(f_name, channels_brainID)
+            f_name = alf_path.joinpath('electrodeSites.brainLocationIds_ccf_2017.npy')
+            np.save(f_name, channels_atlas_id)
             files_to_register.append(f_name)
 
-            # Make the clusters.mlapdv dataset
-            f_name = alf_path.joinpath('clusters.mlapdv.npy')
-            np.save(f_name, clusters_mlapdv)
+            f_name = alf_path.joinpath('electrodeSites.localCoordinates.npy')
+            np.save(f_name, chns)
             files_to_register.append(f_name)
 
-            # Make the clusters.brainLocationIds dataset
-            f_name = alf_path.joinpath('clusters.brainLocationIds_ccf_2017.npy')
-            np.save(f_name, clusters_brainID)
-            files_to_register.append(f_name)
+            probe_collections = self.one.list_collections(self.insertion['session'], filename='channels*',
+                                                          collection=f'alf/{self.insertion["name"]}*')
 
-            # Make the clusters.brainLocationAcronym dataset
-            f_name = alf_path.joinpath('clusters.brainLocationAcronyms_ccf_2017.npy')
-            np.save(f_name, clusters_brainAcro)
-            files_to_register.append(f_name)
+            for collection in probe_collections:
+                chns = self.one.load_dataset(self.insertion['session'], 'channels.localCoordinates', collection=collection)
+                ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1],
+                                            track_prev=track,
+                                            feature_prev=feature,
+                                            brain_atlas=self.brain_atlas)
+                channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6)
+                channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
+
+                alf_path = self.one.eid2path(self.insertion['session']).joinpath(collection)
+                alf_path.mkdir(exist_ok=True, parents=True)
+
+                f_name = alf_path.joinpath('channels.mlapdv.npy')
+                np.save(f_name, channels_mlapdv)
+                files_to_register.append(f_name)
+
+                f_name = alf_path.joinpath('channels.brainLocationIds_ccf_2017.npy')
+                np.save(f_name, channels_atlas_id)
+                files_to_register.append(f_name)
 
             self.log.info("Writing datasets to FlatIron")
             ftp_patcher.create_dataset(path=files_to_register,
                                        created_by=self.one.alyx.user)
 
-        # Need to change channels stored on alyx as well as the stored key is not the same as the
-        # latest key
-        if upload_alyx:
-            if alignment_key != self.align_keys_sorted[0]:
-                histology.register_aligned_track(self.eid, channels_mlapdv / 1e6,
-                                                 chn_coords=self.chn_coords, one=self.one,
-                                                 overwrite=True, channels=self.channels)
-
-                ephys_traj = self.one.alyx.get(f'/trajectories?&probe_insertion={self.eid}'
-                                               '&provenance=Ephys aligned histology track',
-                                               clobber=True)
-                patch_dict = {'json': self.alignments}
-                self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'],
-                                   data=patch_dict)
-
         return files_to_register
 
     def update_experimenter_evaluation(self, prev_alignments=None, override=False):

ibllib/tests/qc/test_alignment_qc.py

@@ -359,16 +359,6 @@ def test_data_content(self):
         channels_id = np.load(alf_path.joinpath('channels.brainLocationIds_ccf_2017.npy'))
         assert(channels_mlapdv.shape[0] == channels_id.shape[0])
 
-        clusters_mlapdv = np.load(alf_path.joinpath('clusters.mlapdv.npy'))
-        assert(np.all(np.abs(clusters_mlapdv) > 0))
-        clusters_id = np.load(alf_path.joinpath('clusters.brainLocationIds_ccf_2017.npy'))
-        assert(clusters_mlapdv.shape[0] == clusters_id.shape[0])
-        assert(np.all(np.in1d(clusters_mlapdv, channels_mlapdv)))
-        assert (np.all(np.in1d(clusters_id, channels_id)))
-        clusters_acro = np.load(alf_path.joinpath('clusters.brainLocationAcronyms_ccf_2017.npy'),
-                                allow_pickle=True)
-        assert(clusters_acro.shape == clusters_id.shape)
-
     def test_upload_to_flatiron(self):
         for file in self.file_paths:
             file_registered = one.alyx.get(f'/datasets?&session={EPHYS_SESSION}'