refactor: remove raw buffer event and need src fmt for creating new codecs

Author: TzuHuanTai

src/capturer/libcamera_capturer.cpp

@@ -300,8 +300,9 @@ void LibcameraCapturer::RequestComplete(libcamera::Request *request) {
     tv.tv_sec = buffer->metadata().timestamp / 1000000000;
     tv.tv_usec = (buffer->metadata().timestamp % 1000000000) / 1000;
 
-    V4L2Buffer v4l2_buffer((uint8_t *)data, length, V4L2_BUF_FLAG_KEYFRAME, tv);
-    NextBuffer(v4l2_buffer);
+    auto v4l2_buffer = V4L2Buffer::FromLibcamera((uint8_t *)data, length, tv, format_);
+    frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, v4l2_buffer);
+    NextFrameBuffer(frame_buffer_);
 
     request->reuse(libcamera::Request::ReuseBuffers);
 
@@ -317,12 +318,6 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> LibcameraCapturer::GetI420Frame(
     return frame_buffer_->ToI420();
 }
 
-void LibcameraCapturer::NextBuffer(V4L2Buffer &buffer) {
-    frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, buffer, format_);
-    NextFrameBuffer(frame_buffer_);
-    NextRawBuffer(buffer);
-}
-
 void LibcameraCapturer::StartCapture() {
     int ret = camera_->configure(camera_config_.get());
     if (ret < 0) {

src/capturer/libcamera_capturer.h

@@ -50,7 +50,6 @@ class LibcameraCapturer : public VideoCapturer {
     std::map<int, std::pair<void *, unsigned int>> mapped_buffers_;
 
     rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer_;
-    void NextBuffer(V4L2Buffer &raw_buffer);
 
     LibcameraCapturer &SetResolution(int width, int height) override;
     LibcameraCapturer &SetFps(int fps) override;

src/capturer/v4l2_capturer.cpp

@@ -143,8 +143,7 @@ void V4L2Capturer::CaptureImage() {
         return;
     }
 
-    V4L2Buffer buffer((uint8_t *)capture_.buffers[buf.index].start, buf.bytesused, buf.flags,
-                      buf.timestamp);
+    auto buffer = V4L2Buffer::FromV4L2((uint8_t *)capture_.buffers[buf.index].start, buf, format_);
     NextBuffer(buffer);
 
     if (!V4L2Util::QueueBuffer(fd_, &buf)) {
@@ -162,35 +161,32 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> V4L2Capturer::GetI420Frame() {
 }
 
 void V4L2Capturer::NextBuffer(V4L2Buffer &buffer) {
-    if (hw_accel_) {
-        // hardware encoding
+    if (hw_accel_ && format_ == V4L2_PIX_FMT_H264) {
+        has_first_keyframe_ = (buffer.flags & V4L2_BUF_FLAG_KEYFRAME) != 0;
         if (!has_first_keyframe_) {
-            has_first_keyframe_ = (buffer.flags & V4L2_BUF_FLAG_KEYFRAME) != 0;
+            return;
         }
+    }
 
-        if (IsCompressedFormat()) {
-            decoder_->EmplaceBuffer(buffer, [this](V4L2Buffer decoded_buffer) {
-                frame_buffer_ =
-                    V4L2FrameBuffer::Create(width_, height_, decoded_buffer, V4L2_PIX_FMT_YUV420);
-                NextFrameBuffer(frame_buffer_);
-            });
-        } else {
-            frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, buffer, format_);
-            NextFrameBuffer(frame_buffer_);
-        }
+    if (!hw_accel_ && format_ == V4L2_PIX_FMT_H264) {
+        INFO_PRINT("Software decoding H264 camera source is not supported.");
+        exit(EXIT_FAILURE);
+    }
+
+    if (hw_accel_ && IsCompressedFormat()) {
+        decoder_->EmplaceBuffer(buffer, [this, buffer](V4L2Buffer &decoded_buffer) {
+            // hw decoder doesn't output timestamps.
+            decoded_buffer.timestamp = buffer.timestamp;
+            HandleDecodedBuffer(decoded_buffer);
+        });
     } else {
-        // software decoding
-        if (format_ != V4L2_PIX_FMT_H264) {
-            frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, buffer, format_);
-            NextFrameBuffer(frame_buffer_);
-        } else {
-            // todo: h264 decoding
-            INFO_PRINT("Software decoding h264 camera source is not support now.");
-            exit(1);
-        }
+        HandleDecodedBuffer(buffer);
     }
+}
 
-    NextRawBuffer(buffer);
+void V4L2Capturer::HandleDecodedBuffer(V4L2Buffer &buffer) {
+    frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, buffer);
+    NextFrameBuffer(frame_buffer_);
 }
 
 void V4L2Capturer::StartCapture() {

src/capturer/v4l2_capturer.h

@@ -41,9 +41,7 @@ class V4L2Capturer : public VideoCapturer {
     V4L2BufferGroup capture_;
     std::unique_ptr<Worker> worker_;
     std::unique_ptr<V4L2Decoder> decoder_;
-
     rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer_;
-    void NextBuffer(V4L2Buffer &raw_buffer);
 
     V4L2Capturer &SetResolution(int width, int height) override;
     V4L2Capturer &SetFps(int fps) override;
@@ -54,6 +52,8 @@ class V4L2Capturer : public VideoCapturer {
     void CaptureImage();
     bool CheckMatchingDevice(std::string unique_name);
     int GetCameraIndex(webrtc::VideoCaptureModule::DeviceInfo *device_info);
+    void NextBuffer(V4L2Buffer &buffer);
+    void HandleDecodedBuffer(V4L2Buffer &buffer);
 };
 
 #endif

src/capturer/video_capturer.h

@@ -14,10 +14,7 @@
 class VideoCapturer {
   public:
     VideoCapturer() = default;
-    ~VideoCapturer() {
-        raw_buffer_subject_.UnSubscribe();
-        frame_buffer_subject_.UnSubscribe();
-    }
+    ~VideoCapturer() { frame_buffer_subject_.UnSubscribe(); }
 
     virtual int fps() const = 0;
     virtual int width() const = 0;
@@ -33,23 +30,16 @@ class VideoCapturer {
     virtual VideoCapturer &SetRotation(int angle) = 0;
     virtual VideoCapturer &SetControls(int key, int value) = 0;
 
-    std::shared_ptr<Observable<V4L2Buffer>> AsRawBufferObservable() {
-        return raw_buffer_subject_.AsObservable();
-    }
-
     std::shared_ptr<Observable<rtc::scoped_refptr<V4L2FrameBuffer>>> AsFrameBufferObservable() {
         return frame_buffer_subject_.AsObservable();
     }
 
   protected:
-    void NextRawBuffer(V4L2Buffer raw_buffer) { raw_buffer_subject_.Next(raw_buffer); }
-
     void NextFrameBuffer(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) {
         frame_buffer_subject_.Next(frame_buffer);
     }
 
   private:
-    Subject<V4L2Buffer> raw_buffer_subject_;
     Subject<rtc::scoped_refptr<V4L2FrameBuffer>> frame_buffer_subject_;
 };
 

src/codecs/v4l2/v4l2_codec.cpp

@@ -5,6 +5,7 @@
 
 V4L2Codec::V4L2Codec()
     : fd_(0),
+      dst_fmt_(0),
       abort_(false) {}
 
 V4L2Codec::~V4L2Codec() {
@@ -48,6 +49,7 @@ bool V4L2Codec::PrepareBuffer(V4L2BufferGroup *gbuffer, int width, int height, u
             output_buffer_index_.push(i);
         }
     } else if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+        dst_fmt_ = pix_fmt;
         if (!V4L2Util::QueueBuffers(fd_, gbuffer)) {
             return false;
         }
@@ -57,6 +59,11 @@ bool V4L2Codec::PrepareBuffer(V4L2BufferGroup *gbuffer, int width, int height, u
 }
 
 void V4L2Codec::Start() {
+    if (dst_fmt_ == 0) {
+        ERROR_PRINT("The target format is not set.");
+        exit(EXIT_FAILURE);
+    }
+
     abort_ = false;
     worker_ = std::make_unique<Worker>(file_name_ + std::to_string(pthread_self()), [this]() {
         CaptureBuffer();
@@ -137,11 +144,9 @@ bool V4L2Codec::CaptureBuffer() {
             return false;
         }
 
-        V4L2Buffer buffer;
-        buffer.start = capture_.buffers[buf.index].start;
-        buffer.length = buf.m.planes[0].bytesused;
-        buffer.dmafd = capture_.buffers[buf.index].dmafd;
-        buffer.flags = buf.flags;
+        auto buffer = V4L2Buffer::FromCapturedPlane(
+            capture_.buffers[buf.index].start, buf.m.planes[0].bytesused,
+            capture_.buffers[buf.index].dmafd, buf.flags, dst_fmt_);
 
         if (abort_) {
             return false;

src/codecs/v4l2/v4l2_codec.h

@@ -24,6 +24,7 @@ class V4L2Codec {
     void Start();
 
   private:
+    uint32_t dst_fmt_;
     std::atomic<bool> abort_;
     std::unique_ptr<Worker> worker_;
     ThreadSafeQueue<int> output_buffer_index_;

src/codecs/v4l2/v4l2_decoder.cpp

@@ -42,7 +42,7 @@ void V4L2Decoder::HandleEvent() {
                 DEBUG_PRINT("Source changed!");
                 V4L2Util::StreamOff(fd_, capture_.type);
                 V4L2Util::DeallocateBuffer(fd_, &capture_);
-                V4L2Util::SetFormat(fd_, &capture_, 0, 0, 0);
+                V4L2Util::SetFormat(fd_, &capture_, 0, 0, capture_format_);
                 V4L2Util::AllocateBuffer(fd_, &capture_, BUFFER_NUM);
                 V4L2Util::StreamOn(fd_, capture_.type);
                 break;

src/codecs/v4l2/v4l2_decoder.h

@@ -11,6 +11,9 @@ class V4L2Decoder : public V4L2Codec {
   protected:
     bool Configure(int width, int height, uint32_t src_pix_fmt, bool is_dma_dst);
     void HandleEvent() override;
+
+  private:
+    uint32_t capture_format_ = V4L2_PIX_FMT_YUV420;
 };
 
 #endif // V4L2_DECODER_H_

src/codecs/v4l2/v4l2_encoder.cpp

@@ -2,12 +2,13 @@
 #include "common/logging.h"
 
 const char *ENCODER_FILE = "/dev/video11";
-const int BUFFER_NUM = 4;
+const int BUFFER_NUM = 2;
 const int KEY_FRAME_INTERVAL = 600;
 
-std::unique_ptr<V4L2Encoder> V4L2Encoder::Create(int width, int height, bool is_dma_src) {
+std::unique_ptr<V4L2Encoder> V4L2Encoder::Create(int width, int height, uint32_t src_pix_fmt,
+                                                 bool is_dma_src) {
     auto encoder = std::make_unique<V4L2Encoder>();
-    encoder->Configure(width, height, is_dma_src);
+    encoder->Configure(width, height, src_pix_fmt, is_dma_src);
     encoder->Start();
     return encoder;
 }
@@ -17,7 +18,7 @@ V4L2Encoder::V4L2Encoder()
       framerate_(30),
       bitrate_bps_(10000000) {}
 
-bool V4L2Encoder::Configure(int width, int height, bool is_dma_src) {
+bool V4L2Encoder::Configure(int width, int height, uint32_t src_pix_fmt, bool is_dma_src) {
     if (!Open(ENCODER_FILE)) {
         DEBUG_PRINT("Failed to turn on encoder: %s", ENCODER_FILE);
         return false;
@@ -30,7 +31,7 @@ bool V4L2Encoder::Configure(int width, int height, bool is_dma_src) {
     V4L2Util::SetExtCtrl(fd_, V4L2_CID_MPEG_VIDEO_H264_I_PERIOD, KEY_FRAME_INTERVAL);
 
     auto src_memory = is_dma_src ? V4L2_MEMORY_DMABUF : V4L2_MEMORY_MMAP;
-    PrepareBuffer(&output_, width, height, V4L2_PIX_FMT_YUV420, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+    PrepareBuffer(&output_, width, height, src_pix_fmt, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
                   src_memory, BUFFER_NUM);
     PrepareBuffer(&capture_, width, height, V4L2_PIX_FMT_H264, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
                   V4L2_MEMORY_MMAP, BUFFER_NUM);