Fix reading of UTF8 annotations (#276)

* make utf8 default reading format

* make utf8 default reading format

* Add test for reading UTF8 annotations

Author: skjerns

pyedflib/edfreader.py

@@ -167,12 +167,17 @@ def _get_float(self, v: np.ndarray) -> np.ndarray:
         return result
 
     def _convert_string(self, s: Union[bytes, str]) -> str:
+        """decode a byte to string using utf8"""
         if isinstance(s, bytes):
-            return s.decode("latin")
+            try:
+                return s.decode("utf_8", "strict")  # Prioritize UTF-8
+            except UnicodeDecodeError:
+                warnings.warn("Could not decode string '{s=}', using fallback latin encoding")
+                return s.decode("latin1", errors="replace")  # Fallback
         elif isinstance(s, str):
             return s
         else:
-            return s.decode("utf_8", "strict")  # type: ignore
+            raise TypeError('unknown string type: {type(s)=}')
 
     def getHeader(self) -> Dict[str, Union[str, datetime]]:
         """

pyedflib/tests/data/test_utf8.edf

@@ -22,6 +22,7 @@ class TestEdfReader(unittest.TestCase):
     def setUpClass(cls):
         # data_dir = os.path.join(os.getcwd(), 'data')
         data_dir = os.path.join(os.path.dirname(__file__), 'data')
+        cls.edf_utf8 = os.path.join(data_dir, 'test_utf8.edf')
         cls.edf_data_file = os.path.join(data_dir, 'test_generator.edf')
         cls.bdf_data_file = os.path.join(data_dir, 'test_generator.bdf')
         cls.bdf_data_file_datarec_2 = os.path.join(data_dir, 'test_generator_datarec_generator_2.bdf')
@@ -393,6 +394,14 @@ def test_EdfReader_Legacy_Header_Info(self):
             for attr_name, expected_value in expected_header.items():
                 self.assertEqual(getattr(f, attr_name), expected_value)
 
+
+    def test_read_annotations_utf8(self):
+        """properly test for UTF8 reading of existing file"""
+        with pyedflib.EdfReader(self.edf_utf8) as f:
+            ann_time, ann_duration, ann_text = f.readAnnotations()
+            self.assertEqual(ann_text[2], '中文测试八个字')
+
+
 if __name__ == '__main__':
     # run_module_suite(argv=sys.argv)
     unittest.main()