fix: TypeError: byref() argument must be a ctypes instance, not 'VideoSubscriptionOptions'; feat: add save img from video yuv to rgb img

agora_rtc/agora/rtc/local_user.py

@@ -487,7 +487,7 @@ def unregister_video_frame_observer(self):
     def subscribe_video(self, user_id, options: VideoSubscriptionOptions):
         user_id_t = user_id.encode('utf-8')
 
-        ret = agora_local_user_subscribe_video(self.user_handle, user_id_t, ctypes.byref(options))
+        ret = agora_local_user_subscribe_video(self.user_handle, user_id_t, ctypes.byref(VideoSubscriptionOptionsInner.create(options)))
         return ret
 
     def subscribe_all_video(self, options: VideoSubscriptionOptions):

agora_rtc/examples/common/parse_args.py

@@ -23,7 +23,7 @@ def __init__(self):
         self.hours = 0
         self.mode = 1
         self.value = 0
-        self.dir_path = None #directory path
+        self.dir_path = None  # directory path
         self.msg = "hello agora python sdk"
 
 
@@ -34,15 +34,24 @@ def parse_args():
     parser.add_argument("--channelId", required=True, help="Channel Id / must")
     parser.add_argument("--connectionNumber", default=1, help="Enter the channel number")
     parser.add_argument("--userId", default="0", help="User Id / default is 0")
-    parser.add_argument("--audioFile", required=False, help="The audio file in raw PCM format to be sent")
+    parser.add_argument(
+        "--audioFile",
+        required=False,
+        help="The audio file in raw PCM format to be sent")
     parser.add_argument("--lowdelay", action="store_true", help="Enable the low delay")
     parser.add_argument("--videoFile", help="The video file in YUV420 format to be sent")
     parser.add_argument("--sampleRate", type=int, help="Example rate for the PCM file to be sent")
-    parser.add_argument("--numOfChannels", type=int, help="Number of channels for the PCM file to be sent")
+    parser.add_argument(
+        "--numOfChannels",
+        type=int,
+        help="Number of channels for the PCM file to be sent")
     parser.add_argument("--fps", type=int, help="Target frame rate for sending the video stream")
     parser.add_argument("--width", type=int, help="Image width for the YUV file to be sent")
     parser.add_argument("--height", type=int, help="Image height for the YUV file to be sent")
-    parser.add_argument("--bitrate", type=int, help="Target bitrate (bps) for encoding the YUV stream")
+    parser.add_argument(
+        "--bitrate",
+        type=int,
+        help="Target bitrate (bps) for encoding the YUV stream")
     parser.add_argument("--message", help="The message to be sent")
     parser.add_argument("--hours", default="0", help="The time to run")
     parser.add_argument("--saveToDisk", default=0, help="The time to run")
@@ -82,5 +91,6 @@ def parse_args_example() -> ExampleOptions:
     sample_options.mode = args.mode
     sample_options.value = args.value
     sample_options.dir_path = args.dir
+    print(sample_options)
 
     return sample_options

agora_rtc/examples/observer/video_frame_observer.py

@@ -1,40 +1,184 @@
 #!env python
 
 import os
+import numpy as np
+from PIL import Image
 import datetime
 from agora.rtc.video_frame_observer import IVideoFrameObserver, VideoFrame
 import logging
 logger = logging.getLogger(__name__)
 
-source_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+source_dir = os.path.dirname(
+    os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(
+                os.path.abspath(__file__)))))
 filename, _ = os.path.splitext(os.path.basename(__file__))
-log_folder = os.path.join(source_dir, 'logs', filename, datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
+log_folder = os.path.join(
+    source_dir,
+    'logs',
+    filename,
+    datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
 os.makedirs(log_folder, exist_ok=True)
 
 
+def yuv_to_rgb(y: np.ndarray, u: np.ndarray, v: np.ndarray) -> np.ndarray:
+    """Convert YUV to RGB using numpy operations"""
+    # YUV to RGB conversion matrix
+    y = y.astype(np.float32)
+    u = u.astype(np.float32) - 128.0
+    v = v.astype(np.float32) - 128.0
+
+    # RGB conversion using BT.601 standard
+    r = y + 1.402 * v
+    g = y - 0.344136 * u - 0.714136 * v
+    b = y + 1.772 * u
+
+    # Clip values to [0, 255] and convert to uint8
+    rgb = np.stack([r, g, b], axis=-1)
+    rgb = np.clip(rgb, 0, 255).astype(np.uint8)
+    return rgb
+
+
+def resize_plane(plane: np.ndarray, target_shape: tuple) -> np.ndarray:
+    """Resize UV plane using bilinear interpolation"""
+    h, w = plane.shape
+    target_h, target_w = target_shape
+
+    # Create coordinate matrices
+    x = np.linspace(0, w - 1, target_w)
+    y = np.linspace(0, h - 1, target_h)
+    x_coords, y_coords = np.meshgrid(x, y)
+
+    # Get integer and fractional parts
+    x0 = np.floor(x_coords).astype(int)
+    x1 = np.minimum(x0 + 1, w - 1)
+    y0 = np.floor(y_coords).astype(int)
+    y1 = np.minimum(y0 + 1, h - 1)
+
+    # Get weights
+    wx = x_coords - x0
+    wy = y_coords - y0
+
+    # Get values at corners
+    v00 = plane[y0, x0]
+    v10 = plane[y1, x0]
+    v01 = plane[y0, x1]
+    v11 = plane[y1, x1]
+
+    # Interpolate
+    result = (v00 * (1 - wx) * (1 - wy) +
+              v01 * wx * (1 - wy) +
+              v10 * (1 - wx) * wy +
+              v11 * wx * wy)
+
+    return result.astype(np.uint8)
+
+
+def convert_I420_to_RGB(video_frame: VideoFrame) -> Image.Image:
+    # YUV420P(I420) to RGB
+    # TODO:
+    # 使用numba进行JIT编译
+    # 实现SIMD优化
+    # 使用多线程处理不同的颜色平面 (or use cuda)
+
+    width = video_frame.width
+    height = video_frame.height
+
+    # Extract YUV planes
+    y_plane = np.frombuffer(
+        video_frame.y_buffer, dtype=np.uint8).reshape(height, width)
+    u_plane = np.frombuffer(
+        video_frame.u_buffer, dtype=np.uint8).reshape(height // 2, width // 2)
+    v_plane = np.frombuffer(
+        video_frame.v_buffer, dtype=np.uint8).reshape(height // 2, width // 2)
+
+    # Resize U and V planes
+    u_resized = resize_plane(u_plane, (height, width))
+    v_resized = resize_plane(v_plane, (height, width))
+
+    # Convert to RGB
+    rgb = yuv_to_rgb(y_plane, u_resized, v_resized)
+    image = Image.fromarray(rgb)
+
+    return image
+
+
+def convert_I420_to_RGB_with_cv(video_frame: VideoFrame) -> Image.Image:
+    import cv2
+    # 从YUV420P(I420)转换到RGB
+    width = video_frame.width
+    height = video_frame.height
+
+    # 从YUV缓冲区提取Y、U、V平面
+    y_size = width * height
+    u_size = (width * height) // 4
+
+    y_plane = np.frombuffer(
+        video_frame.y_buffer, dtype=np.uint8).reshape(height, width)
+    u_plane = np.frombuffer(
+        video_frame.u_buffer, dtype=np.uint8).reshape(height // 2, width // 2)
+    v_plane = np.frombuffer(
+        video_frame.v_buffer, dtype=np.uint8).reshape(height // 2, width // 2)
+
+    # 将U和V平面放大到与Y相同的尺寸
+    u_resized = cv2.resize(u_plane, (width, height), interpolation=cv2.INTER_LINEAR)
+    v_resized = cv2.resize(v_plane, (width, height), interpolation=cv2.INTER_LINEAR)
+
+    # 将YUV转换为RGB
+    yuv = cv2.merge([y_plane, u_resized, v_resized])
+    # 如果出现图像质量问题,可以尝试调整插值方法(如改用cv2.INTER_CUBIC)
+    rgb = cv2.cvtColor(yuv, cv2.COLOR_YUV2RGB)
+
+    # 转换为PIL Image
+    image = Image.fromarray(rgb)
+    return image
+
+
+def save_image(image: Image.Image, channel_id, remote_uid):
+    file_path = os.path.join(log_folder, channel_id + "_" + remote_uid + '.jpeg')
+    image.save(file_path)
+    return file_path
+
+
+def save_yuv(channel_id, remote_uid, log_folder, frame):
+    file_path = os.path.join(log_folder, channel_id + "_" + remote_uid + '.yuv')
+    y_size = frame.y_stride * frame.height
+    uv_size = (frame.u_stride * frame.height // 2)
+    # logger.info(f"on_frame, file_path={file_path}, y_size={y_size}, uv_size={uv_size}, len_y={len(frame.y_buffer)}, len_u={len(frame.u_buffer)}, len_v={len(frame.v_buffer)}")
+    with open(file_path, 'ab') as f:
+        f.write(frame.y_buffer[:y_size])
+        f.write(frame.u_buffer[:uv_size])
+        f.write(frame.v_buffer[:uv_size])
+
+
 class ExampleVideoFrameObserver(IVideoFrameObserver):
     def __init__(self, save_to_disk=0):
         super(ExampleVideoFrameObserver, self).__init__()
         self._save_to_disk = save_to_disk
 
     def on_frame(self, channel_id, remote_uid, frame: VideoFrame):
-        # logger.info(f"on_frame, channel_id={channel_id}, remote_uid={remote_uid}, width={frame.width}, height={frame.height}, y_stride={frame.y_stride}, u_stride={frame.u_stride}, v_stride={frame.v_stride}, len_y={len(frame.y_buffer)}, len_u={len(frame.u_buffer)}, len_v={len(frame.v_buffer)}")
+        logger.info(
+            f"on_frame, channel_id={channel_id}, remote_uid={remote_uid},type={frame.type}, width={frame.width}, height={frame.height}, y_stride={frame.y_stride}, u_stride={frame.u_stride}, v_stride={frame.v_stride}, len_y={ len(frame.y_buffer)}, len_u={ len(frame.u_buffer)}, len_v={len(frame.v_buffer)}, len_alpha_buffer={len(frame.alpha_buffer) if frame.alpha_buffer else 0}")
 
-        logger.info(f"on_frame, channel_id={channel_id}, remote_uid={remote_uid},len_alpha_buffer={len(frame.alpha_buffer) if frame.alpha_buffer else 0}")
+        # logger.info(f"on_frame, channel_id={channel_id}, remote_uid={remote_uid},len_alpha_buffer={len(frame.alpha_buffer) if frame.alpha_buffer else 0}")
 
         if self._save_to_disk:
-            file_path = os.path.join(log_folder, channel_id + "_" + remote_uid + '.yuv')
-            y_size = frame.y_stride * frame.height
-            uv_size = (frame.u_stride * frame.height // 2)
-            # logger.info(f"on_frame, file_path={file_path}, y_size={y_size}, uv_size={uv_size}, len_y={len(frame.y_buffer)}, len_u={len(frame.u_buffer)}, len_v={len(frame.v_buffer)}")
-            with open(file_path, 'ab') as f:
-                f.write(frame.y_buffer[:y_size])
-                f.write(frame.u_buffer[:uv_size])
-                f.write(frame.v_buffer[:uv_size])
+            # image = convert_I420_to_RGB(frame)
+            image = convert_I420_to_RGB_with_cv(frame)
+            file_path = save_image(image, channel_id, remote_uid)
+            print(f"save to {file_path}")
+            # save_yuv(channel_id, remote_uid, log_folder, frame)
         return 1
 
-    def on_user_video_track_subscribed(self, agora_local_user, user_id, info, agora_remote_video_track):
-        logger.info(f"on_user_video_track_subscribed, agora_local_user={agora_local_user}, user_id={user_id}, info={info}, agora_remote_video_track={agora_remote_video_track}")
+    def on_user_video_track_subscribed(
+            self,
+            agora_local_user,
+            user_id,
+            info,
+            agora_remote_video_track):
+        logger.info(
+            f"on_user_video_track_subscribed, agora_local_user={agora_local_user}, user_id={user_id}, info={info}, agora_remote_video_track={agora_remote_video_track}")
         return 0
 
     # def on_user_video_track_subscribed(self, agora_local_user, user_id, agora_remote_video_track:RemoteVideoTrack, video_track_info):