Merge pull request #307 from pollen-robotics/306-sounddevice-may-not-open-the-device-properly

bug #306: soundevice fails if respeaker is selected in system default

Author: FabienDanieau

examples/debug/sound_record.py

@@ -13,7 +13,7 @@
 OUTPUT_FILE = "recorded_audio.wav"
 
 
-def main(backend: str):
+def main(backend: str) -> None:
     """Record audio for 5 seconds and save to a WAV file."""
     logging.basicConfig(
         level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s"
@@ -28,7 +28,7 @@ def main(backend: str):
             sample = mini.media.get_audio_sample()
             if sample is not None:
                 if backend == "gstreamer":
-                    sample = np.frombuffer(sample, dtype=np.int16).reshape(-1, 1)
+                    sample = np.frombuffer(sample, dtype=np.float32).reshape(-1, 1)
                 audio_samples.append(sample)
             else:
                 print("No audio data available yet...")

src/reachy_mini/media/audio_base.py

@@ -23,6 +23,8 @@ class AudioBackend(Enum):
 class AudioBase(ABC):
     """Abstract class for opening and managing audio devices."""
 
+    SAMPLE_RATE = 16000  # respeaker samplerate
+
     def __init__(self, backend: AudioBackend, log_level: str = "INFO") -> None:
         """Initialize the audio device."""
         self.logger = logging.getLogger(__name__)
@@ -39,11 +41,6 @@ def get_audio_sample(self) -> Optional[bytes | npt.NDArray[np.float32]]:
         """Read audio data from the device. Returns the data or None if error."""
         pass
 
-    @abstractmethod
-    def get_audio_samplerate(self) -> int:
-        """Return the samplerate of the audio device."""
-        pass
-
     @abstractmethod
     def stop_recording(self) -> None:
         """Close the audio device and release resources."""

src/reachy_mini/media/audio_gstreamer.py

@@ -35,8 +35,6 @@ def __init__(self, log_level: str = "INFO") -> None:
         self._thread_bus_calls = Thread(target=lambda: self._loop.run(), daemon=True)
         self._thread_bus_calls.start()
 
-        self._samplerate = 24000
-
         self._pipeline_record = Gst.Pipeline.new("audio_recorder")
         self._appsink_audio: Optional[GstApp] = None
         self._init_pipeline_record(self._pipeline_record)
@@ -56,7 +54,7 @@ def __init__(self, log_level: str = "INFO") -> None:
     def _init_pipeline_record(self, pipeline: Gst.Pipeline) -> None:
         self._appsink_audio = Gst.ElementFactory.make("appsink")
         caps = Gst.Caps.from_string(
-            f"audio/x-raw,channels=1,rate={self._samplerate},format=S16LE"
+            f"audio/x-raw,channels=2,rate={self.SAMPLE_RATE},format=F32LE"
         )
         self._appsink_audio.set_property("caps", caps)
         self._appsink_audio.set_property("drop", True)  # avoid overflow
@@ -98,7 +96,7 @@ def _init_pipeline_playback(self, pipeline: Gst.Pipeline) -> None:
         self._appsrc.set_property("format", Gst.Format.TIME)
         self._appsrc.set_property("is-live", True)
         caps = Gst.Caps.from_string(
-            f"audio/x-raw,format=F32LE,channels=1,rate={self._samplerate},layout=interleaved"
+            f"audio/x-raw,format=F32LE,channels=1,rate={self.SAMPLE_RATE},layout=interleaved"
         )
         self._appsrc.set_property("caps", caps)
 
@@ -136,7 +134,7 @@ def start_recording(self) -> None:
         """Open the audio card using GStreamer."""
         self._pipeline_record.set_state(Gst.State.PLAYING)
 
-    def _get_sample(self, appsink: Gst.AppSink) -> Optional[bytes]:
+    def _get_sample(self, appsink: GstApp.AppSink) -> Optional[bytes]:
         sample = appsink.try_pull_sample(20_000_000)
         if sample is None:
             return None
@@ -158,10 +156,6 @@ def get_audio_sample(self) -> Optional[bytes]:
         """
         return self._get_sample(self._appsink_audio)
 
-    def get_audio_samplerate(self) -> int:
-        """Return the samplerate of the audio device."""
-        return self._samplerate
-
     def stop_recording(self) -> None:
         """Release the camera resource."""
         self._pipeline_record.set_state(Gst.State.NULL)

src/reachy_mini/media/audio_sounddevice.py

@@ -30,17 +30,16 @@ def __init__(
         self.stream = None
         self._output_stream = None
         self._buffer: List[npt.NDArray[np.float32]] = []
-        self._device_id = self.get_output_device_id("respeaker")
-        self._samplerate = (
-            -1
-        )  # will be set on first use to avoid issues if device is not present (CI)
+        self._output_device_id = self.get_output_device_id("respeaker")
+        self._input_device_id = self.get_input_device_id("respeaker")
 
     def start_recording(self) -> None:
         """Open the audio input stream, using ReSpeaker card if available."""
         self.stream = sd.InputStream(
             blocksize=self.frames_per_buffer,
-            device=self._device_id,
+            device=self._input_device_id,
             callback=self._callback,
+            samplerate=self.SAMPLE_RATE,
         )
         if self.stream is None:
             raise RuntimeError("Failed to open SoundDevice audio stream.")
@@ -69,14 +68,6 @@ def get_audio_sample(self) -> Optional[bytes | npt.NDArray[np.float32]]:
         self.logger.debug("No audio data available in buffer.")
         return None
 
-    def get_audio_samplerate(self) -> int:
-        """Return the samplerate of the audio device."""
-        if self._samplerate == -1:
-            self._samplerate = int(
-                sd.query_devices(self._device_id)["default_samplerate"]
-            )
-        return self._samplerate
-
     def stop_recording(self) -> None:
         """Close the audio stream and release resources."""
         if self.stream is not None:
@@ -97,8 +88,8 @@ def push_audio_sample(self, data: bytes) -> None:
     def start_playing(self) -> None:
         """Open the audio output stream."""
         self._output_stream = sd.OutputStream(
-            samplerate=self.get_audio_samplerate(),
-            device=self._device_id,
+            samplerate=self.SAMPLE_RATE,
+            device=self._output_device_id,
             channels=1,
         )
         if self._output_stream is None:
@@ -121,9 +112,9 @@ def play_sound(self, sound_file: str, autoclean: bool = False) -> None:
 
         data, samplerate_in = sf.read(file_path, dtype="float32")
 
-        if samplerate_in != self.get_audio_samplerate():
+        if samplerate_in != self.SAMPLE_RATE:
             data = scipy.signal.resample(
-                data, int(len(data) * (self.get_audio_samplerate() / samplerate_in))
+                data, int(len(data) * (self.SAMPLE_RATE / samplerate_in))
             )
         if data.ndim > 1:  # convert to mono
             data = np.mean(data, axis=1)
@@ -157,8 +148,8 @@ def callback(
         event = threading.Event()
 
         self._output_stream = sd.OutputStream(
-            samplerate=self.get_audio_samplerate(),
-            device=self._device_id,
+            samplerate=self.SAMPLE_RATE,
+            device=self._output_device_id,
             channels=1,
             callback=callback,
             finished_callback=event.set,  # release the device when done
@@ -198,10 +189,32 @@ def get_output_device_id(self, name_contains: str) -> int:
         devices = sd.query_devices()
 
         for idx, dev in enumerate(devices):
-            if name_contains.lower() in dev["name"].lower():
+            if (
+                name_contains.lower() in dev["name"].lower()
+                and dev["max_output_channels"] > 0
+            ):
                 return idx
         # Return default output device if not found
         self.logger.warning(
             f"No output device found containing '{name_contains}', using default."
         )
         return int(sd.default.device[1])
+
+    def get_input_device_id(self, name_contains: str) -> int:
+        """Return the input device id whose name contains the given string (case-insensitive).
+
+        If not found, return the default input device id.
+        """
+        devices = sd.query_devices()
+
+        for idx, dev in enumerate(devices):
+            if (
+                name_contains.lower() in dev["name"].lower()
+                and dev["max_input_channels"] > 0
+            ):
+                return idx
+        # Return default input device if not found
+        self.logger.warning(
+            f"No input device found containing '{name_contains}', using default."
+        )
+        return int(sd.default.device[1])

src/reachy_mini/media/camera_gstreamer.py

@@ -93,7 +93,7 @@ def open(self) -> None:
         self._thread_bus_calls = Thread(target=self._handle_bus_calls, daemon=True)
         self._thread_bus_calls.start()
 
-    def _get_sample(self, appsink: Gst.AppSink) -> Optional[bytes]:
+    def _get_sample(self, appsink: GstApp.AppSink) -> Optional[bytes]:
         sample = appsink.try_pull_sample(20_000_000)
         if sample is None:
             return None

src/reachy_mini/media/media_manager.py

@@ -150,7 +150,7 @@ def get_audio_samplerate(self) -> int:
         if self.audio is None:
             self.logger.warning("Audio system is not initialized.")
             return -1
-        return self.audio.get_audio_samplerate()
+        return self.audio.SAMPLE_RATE
 
     def stop_recording(self) -> None:
         """Stop recording audio."""