feat: tts base set metadata in audio frame

Author: wangyimin-agora

integration_tests/test_async_tts2_base/extension.py

@@ -45,6 +45,9 @@ async def request_tts(
         This method is called when the TTS request is made.
         It should yield audio data bytes.
         """
+        # Send audio_start to set current_audio_request_id (required for metadata)
+        await self.send_tts_audio_start(request_id=t.request_id)
+
         audio_data_bytes = [3, 100, 7]
         for b in audio_data_bytes:
             await self.send_tts_audio_data(bytearray(b))

integration_tests/test_async_tts2_base/tests/test_basic_tts.py

@@ -29,6 +29,10 @@ def __init__(self, sample_rate) -> None:
         self.target_sample_rate = sample_rate
         self.received_frames = 0
         self.received_text_result:TTSTextResult = None
+        self.expected_metadata = {
+            "session_id": "test_session",
+            "turn_id": 1
+        }
 
     async def on_start(self, ten_env: AsyncTenEnvTester) -> None:
         await asyncio.sleep(0.1)
@@ -75,6 +79,18 @@ async def on_audio_frame(
             == audio_frame.get_samples_per_channel() * 2
         )
 
+        # Verify metadata is attached to audio frame
+        metadata_json, err = audio_frame.get_property_to_json("metadata")
+        assert not err, f"Failed to get metadata from audio frame: {err}"
+
+        metadata = json.loads(metadata_json)
+        ten_env.log_info(f"Audio frame metadata: {metadata}")
+
+        # Verify metadata matches what was sent in the request
+        assert metadata == self.expected_metadata, (
+            f"Metadata mismatch! Expected: {self.expected_metadata}, "
+            f"Got: {metadata}"
+        )
 
         self.received_frames += 1
 
@@ -85,7 +101,7 @@ async def on_audio_frame(
             f"Number of Channels: {audio_frame.get_number_of_channels()}"
             f"Received Frames: {self.received_frames}"
         )
-        
+
         self.check_received(ten_env)
 
     def check_received(self, ten_env: AsyncTenEnvTester):

interface/ten_ai_base/tts2.py

@@ -97,6 +97,11 @@ def __init__(self, name: str):
         self.total_recv_audio_duration = 0
         self.total_recv_audio_chunks_len = 0
 
+        # Tracks which request_id's audio is currently being sent
+        # Set in send_tts_audio_start(), reset in send_tts_audio_end() and flush
+        # Used by send_tts_audio_data() to attach correct metadata to audio frames
+        self.current_audio_request_id = None
+
     def _can_transition_to(self, request_id: str, new_state: RequestState) -> bool:
         """Check if state transition is valid."""
         current_state = self.request_states.get(request_id)
@@ -298,8 +303,9 @@ async def _flush_input_items(self):
         self.request_states.clear()
         self.metadatas.clear()
 
-        # Reset processing request ID
+        # Reset processing request ID and current audio request ID
         self._processing_request_id = None
+        self.current_audio_request_id = None
 
         self.ten_env.log_debug("Cleared all request states, metadata, and pending messages after flush")
 
@@ -405,6 +411,7 @@ async def send_tts_audio_data(self, audio_data: bytes, timestamp: int = 0) -> No
                 )
                 f.alloc_buf(len(combined_data))
                 f.set_timestamp(timestamp)
+                f.set_property_from_json("metadata", json.dumps(self.metadatas.get(self.current_audio_request_id, {})))
                 buff = f.lock_buf()
                 buff[:] = combined_data
                 f.unlock_buf(buff)
@@ -443,6 +450,9 @@ async def send_tts_ttfb_metrics(
     async def send_tts_audio_start(
         self, request_id: str, turn_id: int = -1, extra_metadata: dict | None = None
     ) -> None:
+        # Set current_audio_request_id to track which request's audio is being sent
+        self.current_audio_request_id = request_id
+
         new_metadata = self.update_metadata(request_id, extra_metadata)
 
         data = Data.create("tts_audio_start")
@@ -489,6 +499,10 @@ async def send_tts_audio_end(
         # Clean up metadata when audio_end is sent
         self.metadatas.pop(request_id, None)
 
+        # Reset current_audio_request_id (audio phase complete)
+        if self.current_audio_request_id == request_id:
+            self.current_audio_request_id = None
+
     async def send_tts_error(
         self,
         request_id: str | None,
@@ -670,6 +684,10 @@ async def finish_request(
         # This is a defensive cleanup in case audio_end wasn't sent
         self.metadatas.pop(request_id, None)
 
+        # Defensive reset of current_audio_request_id for error paths
+        if self.current_audio_request_id == request_id:
+            self.current_audio_request_id = None
+
         # Handle request completion and buffered messages release
         # Only process if this is the currently processing request
         if self._processing_request_id == request_id: