Merge pull request #1997 from pupil-labs/develop

Pupil v2.3 Release Candidate 1

Author: papr

pupil_src/shared_modules/camera_models.py

@@ -28,7 +28,7 @@
 
 logger = logging.getLogger(__name__)
 
-NEWEST_SUPPORTED_VERSION = Version("1.2")
+NEWEST_SUPPORTED_VERSION = Version("1.3")
 
 
 def transform_invisible_to_corresponding_new_style(rec_dir: str):
@@ -144,14 +144,15 @@ def _convert_gaze(recording: PupilRecording):
         "topic": "gaze.pi",
         "norm_pos": None,
         "timestamp": None,
-        "confidence": 1.0,
+        "confidence": None,
     }
     with fm.PLData_Writer(recording.rec_dir, "gaze") as writer:
-        for ((x, y), ts) in pi_gaze_items(root_dir=recording.rec_dir):
+        for ((x, y), ts, conf) in pi_gaze_items(root_dir=recording.rec_dir):
             template_datum["timestamp"] = ts
             template_datum["norm_pos"] = m.normalize(
                 (x, y), size=(width, height), flip_y=True
             )
+            template_datum["confidence"] = conf
             writer.append(template_datum)
         logger.info(f"Converted {len(writer.ts_queue)} gaze positions.")
 

pupil_src/shared_modules/pupil_recording/update/invisible.py

@@ -21,9 +21,9 @@
 import numpy as np
 from scipy.interpolate import interp1d
 
-import camera_models as cm
 import csv_utils
 import file_methods as fm
+from camera_models import Camera_Model
 from version_utils import VersionFormat
 
 from .. import Version
@@ -507,7 +507,7 @@ def make_update():
         f = world_video.streams.video[0].format
         resolution = f.width, f.height
 
-        intrinsics = cm.load_intrinsics(rec_dir, "world", resolution)
+        intrinsics = Camera_Model.from_file(rec_dir, "world", resolution)
 
         DEPRECATED_SQUARE_MARKER_KEY = "realtime_square_marker_surfaces"
         if DEPRECATED_SQUARE_MARKER_KEY not in surface_definitions_dict:

pupil_src/shared_modules/pupil_recording/update/old_style.py

@@ -31,7 +31,7 @@ def _try_patch_world_instrinsics_file(rec_dir: str, videos: T.Sequence[Path]) ->
     # Make sure the default value always correlates to the frame size of BrokenStream
     frame_size = (1280, 720)
     # TODO: Due to the naming conventions for multipart-recordings, we can't
-    # easily lookup 'any' video name in the pre_recorded_calibrations, since it
+    # easily lookup 'any' video name in the default_world_intrinsics, since it
     # might be a multipart recording. Therefore we need to compute a hint here
     # for the lookup. This could be improved.
     camera_hint = ""
@@ -41,7 +41,7 @@ def _try_patch_world_instrinsics_file(rec_dir: str, videos: T.Sequence[Path]) ->
         except av.AVError:
             continue
 
-        for camera in cm.pre_recorded_calibrations:
+        for camera in cm.default_world_intrinsics:
             if camera in video.name:
                 camera_hint = camera
                 break
@@ -51,7 +51,7 @@ def _try_patch_world_instrinsics_file(rec_dir: str, videos: T.Sequence[Path]) ->
         )
         break
 
-    intrinsics = cm.load_intrinsics(rec_dir, camera_hint, frame_size)
+    intrinsics = cm.Camera_Model.from_file(rec_dir, camera_hint, frame_size)
     intrinsics.save(rec_dir, "world")
 
 

pupil_src/shared_modules/pupil_recording/update/update_utils.py

@@ -286,8 +286,9 @@ def start(self):
             frame = self.g_pool.capture._recent_frame
             if frame is None:
                 logger.error(
-                    "Recording a Pupil Mobile stream requires a connection!"
-                    " Aborting recording."
+                    "Your connection does not seem to be stable enough for "
+                    "recording Pupil Mobile via WiFi. We recommend recording "
+                    "on the phone."
                 )
                 return
             if abs(frame.timestamp - start_time_synced) > TIMESTAMP_ERROR_THRESHOLD:

pupil_src/shared_modules/recorder.py

@@ -28,8 +28,6 @@
 
 import numpy as np
 
-from camera_models import load_intrinsics
-
 from .base_backend import (
     Base_Manager,
     Base_Source,

pupil_src/shared_modules/video_capture/__init__.py

@@ -22,7 +22,7 @@
 import numpy as np
 from pyglui import ui
 
-from camera_models import load_intrinsics
+from camera_models import Camera_Model
 from pupil_recording import PupilRecording
 
 from .base_backend import Base_Manager, Base_Source, EndofVideoError, Playback_Source
@@ -245,7 +245,7 @@ def __init__(
         self.buffering = buffered_decoding
         # Load video split for first frame
         self.reset_video()
-        self._intrinsics = load_intrinsics(rec, set_name, self.frame_size)
+        self._intrinsics = Camera_Model.from_file(rec, set_name, self.frame_size)
 
         self.show_plugin_menu = show_plugin_menu
 

pupil_src/shared_modules/video_capture/file_backend.py

@@ -17,7 +17,7 @@
 from pyglui import ui
 
 import os_utils
-from camera_models import load_intrinsics
+from camera_models import Camera_Model
 
 from .base_backend import Base_Manager, Base_Source, SourceInfo
 
@@ -221,7 +221,7 @@ def on_notify(self, notification):
     @property
     def intrinsics(self):
         if self._intrinsics is None or self._intrinsics.resolution != self.frame_size:
-            self._intrinsics = load_intrinsics(
+            self._intrinsics = Camera_Model.from_file(
                 self.g_pool.user_dir, self.name, self.frame_size
             )
         return self._intrinsics

pupil_src/shared_modules/video_capture/ndsi_backend.py

@@ -468,6 +468,16 @@ def find_raw_path(timestamps_path):
         assert raw_path.exists(), f"The file does not exist at path: {raw_path}"
         return raw_path
 
+    def find_worn_path(timestamps_path):
+        worn_name = timestamps_path.name
+        worn_name = worn_name.replace("gaze", "worn")
+        worn_name = worn_name.replace("_timestamps", "")
+        worn_path = timestamps_path.with_name(worn_name).with_suffix(".raw")
+        if worn_path.exists():
+            return worn_path
+        else:
+            return None
+
     def load_timestamps_data(path):
         timestamps = np.load(str(path))
         return timestamps
@@ -478,6 +488,13 @@ def load_raw_data(path):
         raw_data.shape = (-1, 2)
         return np.asarray(raw_data, dtype=raw_data_dtype)
 
+    def load_worn_data(path):
+        if not (path and path.exists()):
+            return None
+
+        confidences = np.fromfile(str(path), "<u1") / 255.0
+        return np.clip(confidences, 0.0, 1.0)
+
     # This pattern will match any filename that:
     # - starts with "gaze ps"
     # - is followed by one or more digits
@@ -496,4 +513,16 @@ def load_raw_data(path):
             size = min(len(raw_data), len(timestamps))
             raw_data = raw_data[:size]
             timestamps = timestamps[:size]
-        yield from zip(raw_data, timestamps)
+
+        conf_data = load_worn_data(find_worn_path(timestamps_path))
+        if conf_data is not None and len(conf_data) != len(timestamps):
+            logger.warning(
+                f"There is a mismatch between the number of confidence data ({len(conf_data)}) "
+                f"and the number of timestamps ({len(timestamps)})! Not using confidence data."
+            )
+            conf_data = None
+
+        if conf_data is None:
+            conf_data = (1.0 for _ in range(len(timestamps)))
+
+        yield from zip(raw_data, timestamps, conf_data)

pupil_src/shared_modules/video_capture/utils.py

@@ -23,7 +23,7 @@
 
 import gl_utils
 import uvc
-from camera_models import load_intrinsics
+from camera_models import Camera_Model
 from version_utils import VersionFormat
 
 from .base_backend import Base_Manager, Base_Source, InitialisationError, SourceInfo
@@ -135,7 +135,7 @@ def __init__(
             self.frame_size_backup = frame_size
             self.frame_rate_backup = frame_rate
             self.exposure_time_backup = None
-            self._intrinsics = load_intrinsics(
+            self._intrinsics = Camera_Model.from_file(
                 self.g_pool.user_dir, self.name, self.frame_size
             )
         else:
@@ -556,7 +556,7 @@ def frame_size(self, new_size):
         self.uvc_capture.frame_size = size
         self.frame_size_backup = size
 
-        self._intrinsics = load_intrinsics(
+        self._intrinsics = Camera_Model.from_file(
             self.g_pool.user_dir, self.name, self.frame_size
         )