Merge pull request #874 from int-brain-lab/2.40.1

Resolves #873

Author: k1o0

.github/workflows/ibllib_ci.yml

@@ -18,12 +18,12 @@ jobs:
       max-parallel: 2
       matrix:
         os: ["windows-latest", "ubuntu-latest"]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.10", "3.12"]
         exclude:
           - os: windows-latest
-            python-version: 3.8
+            python-version: 3.10
           - os: ubuntu-latest
-            python-version: 3.11
+            python-version: 3.12
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/python-publish.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.40.0'
+__version__ = '2.40.1'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/raw_data_loaders.py

@@ -200,6 +200,7 @@ def load_embedded_frame_data(session_path, label: str, raw=False):
     """
     Load the embedded frame count and GPIO for a given session.  If the file doesn't exist,
     or is empty, None values are returned.
+
     :param session_path: Absolute path of session folder
     :param label: The specific video to load, one of ('left', 'right', 'body')
     :param raw: If True the raw data are returned without preprocessing, otherwise frame count is

ibllib/io/session_params.py

@@ -135,7 +135,7 @@ def read_params(path) -> dict:
 
     """
     if (path := Path(path)).is_dir():
-        yaml_file = next(path.glob('_ibl_experiment.description*'), None)
+        yaml_file = next(path.glob('_ibl_experiment.description*.yaml'), None)
     else:
         yaml_file = path if path.exists() else None
     if not yaml_file:
@@ -165,6 +165,11 @@ def merge_params(a, b, copy=False):
     dict
         A merged dictionary consisting of fields from `a` and `b`.
     """
+    def to_hashable(dict_item):
+        """Convert protocol -> dict map to hashable tuple of protocol + sorted key value pairs."""
+        hashable = (dict_item[0], *chain.from_iterable(sorted(dict_item[1].items())))
+        return tuple(tuple(x) if isinstance(x, list) else x for x in hashable)
+
     if copy:
         a = deepcopy(a)
     for k in b:
@@ -176,8 +181,6 @@ def merge_params(a, b, copy=False):
                 # For tasks, keep order and skip duplicates
                 # Assert tasks is a list of single value dicts
                 assert (not prev or set(map(len, prev)) == {1}) and set(map(len, b[k])) == {1}
-                # Convert protocol -> dict map to hashable tuple of protocol + sorted key value pairs
-                to_hashable = lambda itm: (itm[0], *chain.from_iterable(sorted(itm[1].items())))  # noqa
                 # Get the set of previous tasks
                 prev_tasks = set(map(to_hashable, chain.from_iterable(map(dict.items, prev))))
                 tasks = chain.from_iterable(map(dict.items, b[k]))

ibllib/tests/qc/test_critical_reasons.py

@@ -82,7 +82,7 @@ def test_note_already_existing(self):
 
         note = one.alyx.rest('notes', 'list', django=f'object_id,{eid}', no_cache=True)
         self.assertEqual(len(note), 1)
-        self.assertNotEquals(original_note_id, note[0]['id'])
+        self.assertNotEqual(original_note_id, note[0]['id'])
 
     def test_guiinput_ins(self):
         eid = self.ins_id  # probe id

ibllib/tests/test_io.py

@@ -589,6 +589,12 @@ def test_merge_params(self):
         # Test assertion on duplicate sync
         b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}}
         self.assertRaises(AssertionError, session_params.merge_params, a, b)
+        # Test how it handles the extractors key, which is an unhashable list
+        f = {'tasks': [{'fooChoiceWorld': {'collection': 'bar', 'sync_label': 'bpod', 'extractors': ['a', 'b']}}]}
+        g = session_params.merge_params(a, f, copy=True)
+        self.assertCountEqual(['devices', 'procedures', 'projects', 'sync', 'tasks', 'version'], g.keys())
+        self.assertEqual(4, len(g['tasks']))
+        self.assertDictEqual(f['tasks'][0], g['tasks'][-1])
 
     def test_get_protocol_number(self):
         """Test ibllib.io.session_params.get_task_protocol_number function."""

release_notes.md

@@ -1,9 +1,12 @@
 ## Release Note 2.40.0
 
 ### features
-- iblsorter >= 1.9 sorting tasks with waveform extraction and channel sorting 
+- iblsorter >= 1.9 sorting tasks with waveform extraction and channel sorting
 - s3 patcher prototype
 
+#### 2.40.1
+- Bugfix: ibllib.io.sess_params.merge_params supports tasks extractors key
+
 ## Release Note 2.39.0
 
 ### features
@@ -18,7 +21,7 @@
 
 #### 2.39.1
 - Bugfix: brainbox.metrics.single_unit.quick_unit_metrics fix for indexing of n_spike_below2
-- 
+-
 #### 2.39.2
 - Bugfix: routing of protocol to extractor through the project repository checks that the
 target is indeed an extractor class.
@@ -81,7 +84,7 @@ target is indeed an extractor class.
 - oneibl.register_datasets accounts for non existing sessions when checking protected dsets
 
 #### 2.34.1
-- Ensure mesoscope frame QC files are sorted before concatenating 
+- Ensure mesoscope frame QC files are sorted before concatenating
 - Look for SESSION_TEMPLATE_ID key of task settings for extraction of pre-generated choice world sequences
 - Download required ap.meta files when building pipeline for task_qc command
 
@@ -145,7 +148,7 @@ target is indeed an extractor class.
 - Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function
 
 ### bugfixes
-- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 
+- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8
 
 ### other
 - Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9
@@ -198,7 +201,7 @@ target is indeed an extractor class.
 ### features
 - Training status pipeline now compatible with dynamic pipeline
 - Dynamic DLC task using description file
-- Full photometry lookup table 
+- Full photometry lookup table
 
 ### bugfixes
 - fix for untrainable, unbiasable don't repopulate if already exists
@@ -213,7 +216,7 @@ target is indeed an extractor class.
 - split swanson areas
 ### bugfixes
 - trainig plots
-- fix datahandler on SDSC for ONEv2 
+- fix datahandler on SDSC for ONEv2
 
 ### Release Notes 2.23.0 2023-05-19
 - quiescence period extraction
@@ -227,7 +230,7 @@ target is indeed an extractor class.
 ### Release Notes 2.22.2 2023-05-03
 ### bugfixes
 - training plots
-- 
+-
 ### features
 - can change download path for atlas
 ### Release Notes 2.22.1 2023-05-02
@@ -331,7 +334,7 @@ target is indeed an extractor class.
 
 ### Release Notes 2.17.0 2022-10-04
 - units quality metrics use latest algorithms for refractory period violations and noise cut-off
- 
+
 ## Release Notes 2.16
 ### Release Notes 2.16.1 2022-09-28
 ### bugfixes
@@ -350,7 +353,7 @@ target is indeed an extractor class.
 - SessionLoader error handling and bug fix
 
 ### Release Notes 2.15.2 - 2022-09-22
-- extraction pipeline: fix unpacking of empty arguments field from alyx dict that prevents running task 
+- extraction pipeline: fix unpacking of empty arguments field from alyx dict that prevents running task
 
 ### Release Notes 2.15.1 - 2022-09-21
 - atlas: gene-expression backend and MRI Toronto atlas stretch and squeeze factors (Dan/Olivier)
@@ -364,7 +367,7 @@ target is indeed an extractor class.
 - new modalities:
   - photometry extraction (Mainen lab)
   - widefield extraction (Churchland lab)
-  
+
 #### bugfixes
 - Spike sorting task: parse new pykilosort log format
 - Session loader
@@ -439,7 +442,7 @@ target is indeed an extractor class.
 
 ### Release Notes 2.10.6 2022-03-15
 - Allow parent tasks to be 'Incomplete' to run task on local server
-- Change one base_rul for dlc_qc_plot on cortexlab 
+- Change one base_rul for dlc_qc_plot on cortexlab
 
 ### Release Notes 2.10.5 2022-03-11
 - Fix moot release accident
@@ -480,7 +483,7 @@ target is indeed an extractor class.
 
 ### Release Notes 2.9.0 2022-01-24
 - Adding EphysDLC task in ephys_preprocessing pipeline
-- NOTE: requires DLC environment to be set up on local servers! 
+- NOTE: requires DLC environment to be set up on local servers!
 - Fixes to EphysPostDLC dlc_qc_plot
 
 ## Release Notes 2.8