AI: WebRTC: Support G.711 (PCMU/PCMA) audio codec for WebRTC. v7.0.124 (ossrs#4075)

Author: ossrs-ai

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2025-11-09, AI: WebRTC: Support G.711 (PCMU/PCMA) audio codec for WebRTC. v7.0.124 (#4075)
 * v7.0, 2025-11-08, AI: WebRTC: Support VP9 codec for WebRTC-to-WebRTC streaming. v7.0.123 (#4548)
 * v7.0, 2025-11-08, AI: API: Add audio_frames and video_frames to HTTP API. v7.0.122 (#4559)
 * v7.0, 2025-11-07, AI: WHIP: Return detailed HTTP error responses with proper status codes. v7.0.121 (#4502)

trunk/src/utest/srs_utest_manual_mock.cpp

@@ -262,6 +262,60 @@ std::string MockSdpFactory::create_chrome_publisher_offer_with_vp9()
     return ss.str();
 }
 
+std::string MockSdpFactory::create_chrome_publisher_offer_with_g711_pcmu()
+{
+    // Create a real Chrome-like WebRTC SDP offer with H.264 video and G.711 PCMU audio
+    // Use member variables for SSRC and payload type values
+    // PCMU payload type is 0 (standard)
+    uint8_t pcmu_pt = 0;
+    std::stringstream ss;
+    ss << "v=0\r\n"
+       << "o=- 4611731400430051339 2 IN IP4 127.0.0.1\r\n"
+       << "s=-\r\n"
+       << "t=0 0\r\n"
+       << "a=group:BUNDLE 0 1\r\n"
+       << "a=msid-semantic: WMS stream\r\n"
+       // Audio media description (PCMU)
+       << "m=audio 9 UDP/TLS/RTP/SAVPF " << (int)pcmu_pt << "\r\n"
+       << "c=IN IP4 0.0.0.0\r\n"
+       << "a=rtcp:9 IN IP4 0.0.0.0\r\n"
+       << "a=ice-ufrag:test1234\r\n"
+       << "a=ice-pwd:testpassword1234567890\r\n"
+       << "a=ice-options:trickle\r\n"
+       << "a=fingerprint:sha-256 AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99\r\n"
+       << "a=setup:actpass\r\n"
+       << "a=mid:0\r\n"
+       << "a=sendonly\r\n"
+       << "a=rtcp-mux\r\n"
+       << "a=rtpmap:" << (int)pcmu_pt << " PCMU/8000\r\n"
+       << "a=ssrc:" << audio_ssrc_ << " cname:test-audio-cname\r\n"
+       << "a=ssrc:" << audio_ssrc_ << " msid:stream audio\r\n"
+       // Video media description (H.264)
+       << "m=video 9 UDP/TLS/RTP/SAVPF " << (int)video_pt_ << "\r\n"
+       << "c=IN IP4 0.0.0.0\r\n"
+       << "a=rtcp:9 IN IP4 0.0.0.0\r\n"
+       << "a=ice-ufrag:test1234\r\n"
+       << "a=ice-pwd:testpassword1234567890\r\n"
+       << "a=ice-options:trickle\r\n"
+       << "a=fingerprint:sha-256 AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99\r\n"
+       << "a=setup:actpass\r\n"
+       << "a=mid:1\r\n"
+       << "a=sendonly\r\n"
+       << "a=rtcp-mux\r\n"
+       << "a=rtcp-rsize\r\n"
+       << "a=rtpmap:" << (int)video_pt_ << " H264/90000\r\n"
+       << "a=rtcp-fb:" << (int)video_pt_ << " goog-remb\r\n"
+       << "a=rtcp-fb:" << (int)video_pt_ << " transport-cc\r\n"
+       << "a=rtcp-fb:" << (int)video_pt_ << " ccm fir\r\n"
+       << "a=rtcp-fb:" << (int)video_pt_ << " nack\r\n"
+       << "a=rtcp-fb:" << (int)video_pt_ << " nack pli\r\n"
+       << "a=fmtp:" << (int)video_pt_ << " level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\n"
+       << "a=ssrc:" << video_ssrc_ << " cname:test-video-cname\r\n"
+       << "a=ssrc:" << video_ssrc_ << " msid:stream video\r\n";
+
+    return ss.str();
+}
+
 MockDtlsCertificate::MockDtlsCertificate()
 {
     fingerprint_ = "AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99";

trunk/src/utest/srs_utest_manual_mock.hpp

@@ -85,6 +85,8 @@ class MockSdpFactory
     std::string create_chrome_publisher_offer_with_av1();
     // Create a Chrome-like WebRTC publisher offer SDP with VP9
     std::string create_chrome_publisher_offer_with_vp9();
+    // Create a Chrome-like WebRTC publisher offer SDP with G.711 PCMU audio
+    std::string create_chrome_publisher_offer_with_g711_pcmu();
 };
 
 // Mock DTLS certificate for testing

trunk/src/utest/srs_utest_workflow_rtc_conn.cpp

@@ -823,3 +823,218 @@ VOID TEST(BasicWorkflowRtcConnTest, ManuallyVerifyForPublisherWithVP9)
     // Stop the publisher
     publisher->stop();
 }
+
+// This test is used to verify the basic workflow of the RTC connection with G.711 PCMU codec.
+// It's finished with the help of AI, but each step is manually designed
+// and verified. So this is not dominated by AI, but by humanbeing.
+VOID TEST(BasicWorkflowRtcConnTest, ManuallyVerifyForPublisherWithG711Pcmu)
+{
+    srs_error_t err;
+
+    // Create mock dependencies FIRST (they must outlive the connection)
+    SrsUniquePtr<MockCircuitBreaker> mock_circuit_breaker(new MockCircuitBreaker());
+    SrsUniquePtr<MockConnectionManager> mock_conn_manager(new MockConnectionManager());
+    SrsUniquePtr<MockRtcSourceManager> mock_rtc_sources(new MockRtcSourceManager());
+    SrsUniquePtr<MockAppConfig> mock_config(new MockAppConfig());
+    SrsUniquePtr<MockDtlsCertificate> mock_dtls_certificate(new MockDtlsCertificate());
+    SrsUniquePtr<MockSdpFactory> mock_sdp_factory(new MockSdpFactory());
+    SrsUniquePtr<MockAppFactoryForRtcConn> mock_app_factory(new MockAppFactoryForRtcConn());
+    SrsStreamPublishTokenManager token_manager;
+
+    mock_config->rtc_dtls_role_ = "passive";
+    mock_dtls_certificate->fingerprint_ = "test-fingerprint";
+    mock_app_factory->rtc_sources_ = mock_rtc_sources.get();
+    mock_app_factory->mock_protocol_utility_ = new MockProtocolUtility("192.168.1.100");
+    MockRtcSource *mock_rtc_source = new MockRtcSource();
+    mock_rtc_sources->mock_source_ = SrsSharedPtr<SrsRtcSource>(mock_rtc_source);
+
+    // Create a real ISrsRtcConnection using _srs_app_factory_
+    MockRtcAsyncTaskExecutor mock_exec;
+    SrsContextId cid;
+    cid.set_value("test-rtc-conn-publisher-g711-pcmu-workflow");
+
+    SrsUniquePtr<ISrsRtcConnection> conn_ptr(_srs_app_factory->create_rtc_connection(&mock_exec, cid));
+    SrsRtcConnection *conn = dynamic_cast<SrsRtcConnection *>(conn_ptr.get());
+    EXPECT_TRUE(conn != NULL);
+
+    // Mock the RTC conn, also mock the config in publisher_negotiator_ and player_negotiator_
+    conn->circuit_breaker_ = mock_circuit_breaker.get();
+    conn->conn_manager_ = mock_conn_manager.get();
+    conn->rtc_sources_ = mock_rtc_sources.get();
+    conn->config_ = mock_config.get();
+    conn->dtls_certificate_ = mock_dtls_certificate.get();
+    conn->app_factory_ = mock_app_factory.get();
+
+    SrsRtcPublisherNegotiator *pub_neg = dynamic_cast<SrsRtcPublisherNegotiator *>(conn->publisher_negotiator_);
+    pub_neg->config_ = mock_config.get();
+    SrsRtcPlayerNegotiator *play_neg = dynamic_cast<SrsRtcPlayerNegotiator *>(conn->player_negotiator_);
+    play_neg->config_ = mock_config.get();
+    play_neg->rtc_sources_ = mock_rtc_sources.get();
+
+    // Create RTC user config for add_publisher with G.711 PCMU codec
+    SrsUniquePtr<SrsRtcUserConfig> ruc(new SrsRtcUserConfig());
+    if (true) {
+        srs_freep(ruc->req_);
+        ruc->req_ = new MockRtcAsyncCallRequest("test.vhost", "live", "stream1");
+        ruc->publish_ = true;
+        ruc->dtls_ = true;
+        ruc->srtp_ = true;
+        ruc->audio_before_video_ = false;
+        ruc->acodec_ = "pcmu"; // Specify PCMU codec
+
+        ruc->remote_sdp_str_ = mock_sdp_factory->create_chrome_publisher_offer_with_g711_pcmu();
+        HELPER_EXPECT_SUCCESS(ruc->remote_sdp_.parse(ruc->remote_sdp_str_));
+    }
+
+    // Add publisher, which negotiate the SDP and generate local SDP
+    SrsSdp local_sdp;
+    local_sdp.session_config_.dtls_role_ = mock_config->get_rtc_dtls_role(ruc->req_->vhost_);
+
+    if (true) {
+        HELPER_EXPECT_SUCCESS(conn->add_publisher(ruc.get(), local_sdp));
+
+        // Verify publishers and SSRC mappings
+        EXPECT_TRUE(conn->publishers_.size() == 1);
+        EXPECT_TRUE(conn->publishers_ssrc_map_.size() == 2);
+        EXPECT_TRUE(conn->publishers_ssrc_map_.find(mock_sdp_factory->audio_ssrc_) != conn->publishers_ssrc_map_.end());
+        EXPECT_TRUE(conn->publishers_ssrc_map_.find(mock_sdp_factory->video_ssrc_) != conn->publishers_ssrc_map_.end());
+
+        // Verify the source stream desription, should have two tracks.
+        SrsRtcSourceDescription *stream_desc = mock_rtc_sources->mock_source_->stream_desc_;
+        EXPECT_TRUE(stream_desc->audio_track_desc_ != NULL);
+        EXPECT_TRUE(stream_desc->video_track_descs_.size() == 1);
+
+        // Verify the audio track ssrc and payload type.
+        EXPECT_TRUE(stream_desc->audio_track_desc_->ssrc_ == mock_sdp_factory->audio_ssrc_);
+        // PCMU uses payload type 0
+        EXPECT_TRUE(stream_desc->audio_track_desc_->media_->pt_ == 0);
+
+        // Verify the codec is PCMU
+        EXPECT_TRUE(stream_desc->audio_track_desc_->media_->name_ == "PCMU");
+
+        // Verify the video track ssrc and payload type.
+        EXPECT_TRUE(stream_desc->video_track_descs_[0]->ssrc_ == mock_sdp_factory->video_ssrc_);
+        EXPECT_TRUE(stream_desc->video_track_descs_[0]->media_->pt_ == mock_sdp_factory->video_pt_);
+
+        // Verify the local SDP was generated with media information
+        EXPECT_TRUE(local_sdp.version_ == "0");
+        EXPECT_TRUE(local_sdp.group_policy_ == "BUNDLE");
+        EXPECT_TRUE(local_sdp.msids_.size() == 1);
+        EXPECT_TRUE(local_sdp.msids_[0] == "live/stream1");
+        EXPECT_TRUE(local_sdp.media_descs_.size() == 2);
+
+        // First should be audio media desc with PCMU
+        SrsMediaDesc *audio_desc = &local_sdp.media_descs_[0];
+        EXPECT_TRUE(audio_desc->type_ == "audio");
+        EXPECT_TRUE(audio_desc->recvonly_);
+        EXPECT_TRUE(audio_desc->payload_types_.size() == 1);
+        EXPECT_TRUE(audio_desc->payload_types_[0].payload_type_ == 0);
+        EXPECT_TRUE(audio_desc->payload_types_[0].encoding_name_ == "PCMU");
+        EXPECT_TRUE(audio_desc->payload_types_[0].clock_rate_ == 8000);
+
+        // Second should be video media desc
+        SrsMediaDesc *video_desc = &local_sdp.media_descs_[1];
+        EXPECT_TRUE(video_desc->type_ == "video");
+        EXPECT_TRUE(video_desc->recvonly_);
+        EXPECT_TRUE(video_desc->payload_types_.size() == 1);
+        EXPECT_TRUE(video_desc->payload_types_[0].payload_type_ == mock_sdp_factory->video_pt_);
+        EXPECT_TRUE(video_desc->payload_types_[0].encoding_name_ == "H264");
+        EXPECT_TRUE(video_desc->payload_types_[0].clock_rate_ == 90000);
+    }
+
+    // Generate local SDP and setup SDP.
+    std::string username;
+    if (true) {
+        bool status = true;
+        conn->set_all_tracks_status(ruc->req_->get_stream_url(), ruc->publish_, status);
+
+        HELPER_EXPECT_SUCCESS(conn->generate_local_sdp(ruc.get(), local_sdp, username));
+        conn->set_remote_sdp(ruc->remote_sdp_);
+        conn->set_local_sdp(local_sdp);
+        conn->set_state_as_waiting_stun();
+
+        // Verify the local SDP was generated ice pwd
+        SrsMediaDesc *audio_desc = &local_sdp.media_descs_[0];
+        EXPECT_TRUE(!audio_desc->session_info_.ice_pwd_.empty());
+        EXPECT_TRUE(!audio_desc->session_info_.fingerprint_.empty());
+        EXPECT_TRUE(audio_desc->candidates_.size() == 1);
+        EXPECT_TRUE(audio_desc->candidates_[0].ip_ == "192.168.1.100");
+        EXPECT_TRUE(audio_desc->session_info_.setup_ == "passive");
+
+        SrsMediaDesc *video_desc = &local_sdp.media_descs_[1];
+        EXPECT_TRUE(!video_desc->session_info_.ice_pwd_.empty());
+        EXPECT_TRUE(!video_desc->session_info_.fingerprint_.empty());
+        EXPECT_TRUE(video_desc->candidates_.size() == 1);
+        EXPECT_TRUE(video_desc->candidates_[0].ip_ == "192.168.1.100");
+        EXPECT_TRUE(video_desc->session_info_.setup_ == "passive");
+
+        EXPECT_TRUE(local_sdp.session_negotiate_.dtls_role_ == "passive");
+    }
+
+    // Initialize the connection
+    if (true) {
+        HELPER_EXPECT_SUCCESS(conn->initialize(ruc->req_, ruc->dtls_, ruc->srtp_, username));
+        EXPECT_TRUE(conn->nack_enabled_);
+
+        // Create and set publish token
+        SrsStreamPublishToken *publish_token_raw = NULL;
+        HELPER_EXPECT_SUCCESS(token_manager.acquire_token(ruc->req_, publish_token_raw));
+        SrsSharedPtr<ISrsStreamPublishToken> publish_token(publish_token_raw);
+
+        conn->set_publish_token(publish_token);
+        EXPECT_TRUE(conn->publish_token_->is_acquired());
+    }
+
+    // DTLS done, start publisher
+    SrsRtcPublishStream *publisher = NULL;
+    if (true) {
+        HELPER_EXPECT_SUCCESS(conn->on_dtls_handshake_done());
+
+        // Wait for coroutine to start. Normally it should be ready wait for PLI requests.
+        srs_usleep(1 * SRS_UTIME_MILLISECONDS);
+
+        // Verify the publisher is created and started
+        EXPECT_TRUE(conn->publishers_.size() == 1);
+        publisher = dynamic_cast<SrsRtcPublishStream *>(conn->publishers_.begin()->second);
+        EXPECT_TRUE(publisher->is_sender_started_);
+    }
+
+    // Got a RTP audio packet with PCMU payload type.
+    for (int i = 0; i < 3; i++) {
+        SrsRtpPacket pkt;
+        pkt.header_.set_ssrc(mock_sdp_factory->audio_ssrc_);
+        pkt.header_.set_sequence(100);
+        pkt.header_.set_timestamp(1000);
+        pkt.header_.set_payload_type(0); // PCMU payload type
+
+        SrsUniquePtr<char[]> data(new char[1500]);
+        SrsBuffer buf(data.get(), 1500);
+        HELPER_EXPECT_SUCCESS(pkt.encode(&buf));
+
+        HELPER_EXPECT_SUCCESS(conn->on_rtp_cipher(data.get(), buf.pos()));
+        HELPER_EXPECT_SUCCESS(conn->on_rtp_plaintext(data.get(), buf.pos()));
+
+        EXPECT_EQ(mock_rtc_source->rtp_audio_count_, i + 1);
+    }
+
+    // Got a RTP video packet.
+    for (int i = 0; i < 3; i++) {
+        SrsRtpPacket pkt;
+        pkt.header_.set_ssrc(mock_sdp_factory->video_ssrc_);
+        pkt.header_.set_sequence(100);
+        pkt.header_.set_timestamp(1000);
+        pkt.header_.set_payload_type(mock_sdp_factory->video_pt_);
+
+        SrsUniquePtr<char[]> data(new char[1500]);
+        SrsBuffer buf(data.get(), 1500);
+        HELPER_EXPECT_SUCCESS(pkt.encode(&buf));
+
+        HELPER_EXPECT_SUCCESS(conn->on_rtp_cipher(data.get(), buf.pos()));
+        HELPER_EXPECT_SUCCESS(conn->on_rtp_plaintext(data.get(), buf.pos()));
+
+        EXPECT_EQ(mock_rtc_source->rtp_video_count_, i + 1);
+    }
+
+    // Stop the publisher
+    publisher->stop();
+}