Develop (#719)

* Fix loading of personal projects extractor map
* flake8
* Fix unrelated test

Author: k1o0

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.28.1'
+__version__ = '2.28.2'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/extractors/base.py

@@ -395,7 +395,7 @@ def _get_task_extractor_map():
         # look if there are custom extractor types in the personal projects repo
         import projects.base
         custom_extractors = Path(projects.base.__file__).parent.joinpath(FILENAME)
-        with open(custom_extractors) as fp:
+        with open(custom_extractors, 'r') as fp:
             custom_task_types = json.load(fp)
         task_extractors.update(custom_task_types)
     except (ModuleNotFoundError, FileNotFoundError):

ibllib/io/raw_data_loaders.py

@@ -3,9 +3,9 @@
 # @Author: Niccolò Bonacchi, Miles Wells
 # @Date: Monday, July 16th 2018, 1:28:46 pm
 """
-Raw Data Loader functions for PyBpod rig
+Raw Data Loader functions for PyBpod rig.
 
-Module contains one loader function per raw datafile
+Module contains one loader function per raw datafile.
 """
 import re
 import json
@@ -107,7 +107,7 @@ def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data
 
 
 def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame:
-    """ Loads binary frame data from Bonsai camera recording workflow.
+    """Loads binary frame data from Bonsai camera recording workflow.
 
     Args:
         session_path (StrPath): Path to session folder

ibllib/plots/snapshot.py

@@ -216,7 +216,7 @@ def register_image(self, image_file, text='', json_field=None, width=None):
         """
         # the protocol is not compatible with byte streaming and json, so serialize the json object here
         # Make sure that user is logged in, if not, try to log in
-        assert self.one.alyx.is_logged_in, "No Alyx user is logged in, try running one.alyx.authenticate() first"
+        assert self.one.alyx.is_logged_in, 'No Alyx user is logged in, try running one.alyx.authenticate() first'
         note = {
             'user': self.one.alyx.user, 'content_type': self.content_type, 'object_id': self.object_id,
             'text': text, 'width': width, 'json': json.dumps(json_field)}
@@ -232,16 +232,15 @@ def register_image(self, image_file, text='', json_field=None, width=None):
         # Catch error that results from object_id - content_type mismatch
         try:
             note_db = self.one.alyx.rest('notes', 'create', data=note, files={'image': fig_open})
-            fig_open.close()
             return note_db
         except requests.HTTPError as e:
-            if "matching query does not exist.'" in str(e):
-                fig_open.close()
+            if 'matching query does not exist' in str(e):
                 _logger.error(f'The object_id {self.object_id} does not match an object of type {self.content_type}')
                 _logger.debug(traceback.format_exc())
             else:
-                fig_open.close()
-                raise
+                raise e
+        finally:
+            fig_open.close()
 
     def register_images(self, image_list=None, texts=None, widths=None, jsons=None):
         """

ibllib/tests/extractors/test_extractors_base.py

@@ -0,0 +1,73 @@
+import json
+import unittest
+import tempfile
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+from ibllib.io.extractors import base
+
+
+class TestExtractorMaps(unittest.TestCase):
+    """Tests for functions that return Bpod extractor classes."""
+    def setUp(self):
+        # Store original __import__
+        self.orig_import = __import__
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.custom_extractors_path = Path(tmp.name).joinpath('task_extractor_map.json')
+        self.custom_extractors = {'fooChoiceWorld': 'Bar'}
+        self.projects = MagicMock()
+        self.projects.base.__file__ = str(self.custom_extractors_path.with_name('__init__.py'))
+        with open(self.custom_extractors_path, 'w') as fp:
+            json.dump(self.custom_extractors, fp)
+
+    def import_mock(self, name, *args):
+        """Return mock for project_extraction imports."""
+        if name == 'projects' or name == 'projects.base':
+            return self.projects
+        return self.orig_import(name, *args)
+
+    def test_get_task_extractor_map(self):
+        """Test ibllib.io.extractors.base._get_task_extractor_map function."""
+        # Check the custom map is loaded
+        with patch('builtins.__import__', side_effect=self.import_mock):
+            extractors = base._get_task_extractor_map()
+            self.assertTrue(self.custom_extractors.items() < extractors.items())
+        # Test handles case where module not installed
+        with patch('builtins.__import__', side_effect=ModuleNotFoundError):
+            extractors = base._get_task_extractor_map()
+            self.assertFalse(set(self.custom_extractors.items()).issubset(set(extractors.items())))
+        # Remove the file and check exception is caught
+        self.custom_extractors_path.unlink()
+        extractors = base._get_task_extractor_map()
+        self.assertFalse(set(self.custom_extractors.items()).issubset(set(extractors.items())))
+
+    def test_get_bpod_extractor_class(self):
+        """Test ibllib.io.extractors.base.get_bpod_extractor_class function."""
+        # installe
+        # alf_path = self.custom_extractors_path.parent.joinpath('subject', '2020-01-01', '001', 'raw_task_data_00')
+        # alf_path.mkdir(parents=True)
+        settings_file = Path(__file__).parent.joinpath(
+            'data', 'session_biased_ge5', 'raw_behavior_data', '_iblrig_taskSettings.raw.json'
+        )
+        # shutil.copy(settings_file, alf_path)
+        session_path = settings_file.parents[1]
+        self.assertEqual('BiasedTrials', base.get_bpod_extractor_class(session_path))
+        session_path = str(session_path).replace('session_biased_ge5', 'session_training_ge5')
+        self.assertEqual('TrainingTrials', base.get_bpod_extractor_class(session_path))
+        session_path = str(session_path).replace('session_training_ge5', 'foobar')
+        self.assertRaises(ValueError, base.get_bpod_extractor_class, session_path)
+
+    def test_protocol2extractor(self):
+        """Test ibllib.io.extractors.base.protocol2extractor function."""
+        # Test fuzzy match
+        (proc, expected), = self.custom_extractors.items()
+        with patch('builtins.__import__', side_effect=self.import_mock):
+            extractor = base.protocol2extractor('_mw_' + proc)
+            self.assertEqual(expected, extractor)
+        # Test unknown protocol
+        self.assertRaises(ValueError, base.protocol2extractor, proc)
+
+
+if __name__ == '__main__':
+    unittest.main()

ibllib/tests/test_plots.py

@@ -22,7 +22,7 @@ class TestSnapshot(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        # Make a small image an store in tmp file
+        # Make a small image and store in tmp file
         cls.tmp_dir = tempfile.TemporaryDirectory()
         cls.img_file = Path(cls.tmp_dir.name).joinpath('test.png')
         image = Image.new('RGBA', size=(WIDTH, HEIGHT), color=(155, 0, 0))
@@ -40,7 +40,7 @@ def setUpClass(cls):
         cls.eid = str(eid)
 
     def _get_image(self, url):
-        # This is a bit of a hack because when running a the server locally, the request to the media folder fail
+        # This is a bit of a hack because when running the server locally, the request to the media folder fails
         rel_path = urlparse(url).path[1:]
         try:
             img_file = list(Path('/var/www/').rglob(rel_path))[0]

release_notes.md

@@ -13,6 +13,9 @@
 - Typo in ibllib.pipes.video_tasks.EphysPostDLC class
 - ibllib.io.raw_data_loaders.patch_settings works with iblrigv8 settings files
 
+#### 2.28.2
+- Fix loading of personal projects extractor map
+
 ## Release Notes 2.27
 
 ### features