wip

wkazmierczak · wkazmierczak · commit 144eb3034250 · 2026-04-03T12:37:30.000+02:00
diff --git a/smelter-core/src/graphics_context.rs b/smelter-core/src/graphics_context.rs
@@ -80,6 +80,58 @@ impl GraphicsContext {
     pub fn has_vulkan_encoder_support(&self) -> bool {
         false
     }
+
+    #[cfg(feature = "vk-video")]
+    pub fn vulkan_h264_encode_profile_level_support(&self) -> Option<H264ProfileLevelSupport> {
+        let vulkan_ctx = self.vulkan_ctx.as_ref()?;
+        let caps = vulkan_ctx.device.encode_capabilities().h264?;
+
+        Some(H264ProfileLevelSupport {
+            baseline_max_level_idc: caps.baseline_profile.map(|p| p.max_level_idc),
+            main_max_level_idc: caps.main_profile.map(|p| p.max_level_idc),
+            high_max_level_idc: caps.high_profile.map(|p| p.max_level_idc),
+        })
+    }
+
+    #[cfg(not(feature = "vk-video"))]
+    pub fn vulkan_h264_encode_profile_level_support(&self) -> Option<H264ProfileLevelSupport> {
+        None
+    }
+
+    #[cfg(feature = "vk-video")]
+    pub fn vulkan_h264_decode_profile_level_support(&self) -> Option<H264ProfileLevelSupport> {
+        let vulkan_ctx = self.vulkan_ctx.as_ref()?;
+        let caps = vulkan_ctx.device.decode_capabilities().h264?;
+
+        Some(H264ProfileLevelSupport {
+            baseline_max_level_idc: caps.baseline_profile.map(|p| p.max_level_idc),
+            main_max_level_idc: caps.main_profile.map(|p| p.max_level_idc),
+            high_max_level_idc: caps.high_profile.map(|p| p.max_level_idc),
+        })
+    }
+
+    #[cfg(not(feature = "vk-video"))]
+    pub fn vulkan_h264_decode_profile_level_support(&self) -> Option<H264ProfileLevelSupport> {
+        None
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct H264ProfileLevelSupport {
+    pub baseline_max_level_idc: Option<u8>,
+    pub main_max_level_idc: Option<u8>,
+    pub high_max_level_idc: Option<u8>,
+}
+
+impl H264ProfileLevelSupport {
+    pub fn max_level_for_profile(self, profile_idc: u8) -> Option<u8> {
+        match profile_idc {
+            0x42 => self.baseline_max_level_idc,
+            0x4d => self.main_max_level_idc,
+            0x64 => self.high_max_level_idc,
+            _ => None,
+        }
+    }
 }
 
 #[derive(Debug, thiserror::Error)]
diff --git a/smelter-core/src/pipeline/webrtc/h264_offer_filter.rs b/smelter-core/src/pipeline/webrtc/h264_offer_filter.rs
@@ -4,9 +4,11 @@ use tracing::warn;
 use webrtc::{
     api::media_engine::MIME_TYPE_H264,
     peer_connection::sdp::session_description::RTCSessionDescription,
-    rtp_transceiver::rtp_codec::RTCRtpCodecParameters,
+    rtp_transceiver::rtp_codec::{RTCRtpCodecCapability, RTCRtpCodecParameters},
 };
 
+use super::supported_codec_parameters::get_video_rtcp_feedback;
+
 #[derive(Debug, Clone)]
 struct OfferH264Params {
     payload_type: u8,
@@ -20,40 +22,24 @@ struct ParsedH264Fmtp {
     packetization_mode: String,
 }
 
-/// Mirrors H264 variants from the SDP offer.
+/// Builds H264 codec parameters by copying every H264 variant from the SDP offer.
 ///
 /// For each offered H264 payload type, we emit a local codec entry with the same
-/// payload type and negotiated fmtp constraints, so WHIP answers can include every
-/// offered H264 profile/level and WHEP can expose as many H264 subtypes as possible.
-pub(crate) fn filter_h264_codecs_by_offer(
-    offer: &RTCSessionDescription,
-    codecs: Vec<RTCRtpCodecParameters>,
-) -> Vec<RTCRtpCodecParameters> {
+/// payload type and fmtp, so negotiation produces an exact match. This allows
+/// accepting any H264 profile/level the peer supports.
+///
+/// Returns an empty list if no H264 codecs are found in the offer.
+pub(crate) fn h264_codecs_from_offer(offer: &RTCSessionDescription) -> Vec<RTCRtpCodecParameters> {
     let offer_h264_params = match extract_h264_params_from_offer(offer) {
         Some(params) if !params.is_empty() => params,
-        Some(_) => {
-            // Offer has no explicit H264 fmtp lines.
-            return codecs;
-        }
+        Some(_) => return Vec::new(),
         None => {
-            warn!("Failed to parse SDP offer for H264 codec filtering, using all codecs");
-            return codecs;
+            warn!("Failed to parse SDP offer for H264 codecs");
+            return Vec::new();
         }
     };
 
-    let (h264_templates, mut passthrough): (Vec<_>, Vec<_>) = codecs
-        .into_iter()
-        .partition(|codec| codec.capability.mime_type == MIME_TYPE_H264);
-
-    if h264_templates.is_empty() {
-        return passthrough;
-    }
-
-    // Any H264 template works: payload type and fmtp are overwritten from the offer.
-    let Some(template) = h264_templates.into_iter().next() else {
-        return passthrough;
-    };
-
+    let mut codecs = Vec::new();
     let mut seen = HashSet::new();
     for offer in offer_h264_params {
         if !seen.insert((
@@ -64,17 +50,23 @@ pub(crate) fn filter_h264_codecs_by_offer(
             continue;
         }
 
-        let mut codec = template.clone();
-
-        codec.payload_type = offer.payload_type;
-        codec.capability.sdp_fmtp_line = build_h264_fmtp(
-            offer.profile_level_id.as_deref(),
-            offer.packetization_mode.as_str(),
-        );
-        passthrough.push(codec);
+        codecs.push(RTCRtpCodecParameters {
+            capability: RTCRtpCodecCapability {
+                mime_type: MIME_TYPE_H264.to_owned(),
+                clock_rate: 90000,
+                channels: 0,
+                sdp_fmtp_line: build_h264_fmtp(
+                    offer.profile_level_id.as_deref(),
+                    offer.packetization_mode.as_str(),
+                ),
+                rtcp_feedback: get_video_rtcp_feedback(),
+            },
+            payload_type: offer.payload_type,
+            ..Default::default()
+        });
     }
 
-    passthrough
+    codecs
 }
 
 /// Extracts H264 payload parameters from offer fmtp/rtpmap lines.
diff --git a/smelter-core/src/pipeline/webrtc/h264_vulkan_capability_filter.rs b/smelter-core/src/pipeline/webrtc/h264_vulkan_capability_filter.rs
@@ -4,29 +4,14 @@ use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecParameters;
 
 use crate::pipeline::PipelineCtx;
 
-#[derive(Debug, Clone, Copy)]
-struct H264ProfileLevelSupport {
-    baseline_max_level_idc: Option<u8>,
-    main_max_level_idc: Option<u8>,
-    high_max_level_idc: Option<u8>,
-}
-
-impl H264ProfileLevelSupport {
-    fn max_level_for_profile(self, profile_idc: u8) -> Option<u8> {
-        match profile_idc {
-            0x42 => self.baseline_max_level_idc,
-            0x4d => self.main_max_level_idc,
-            0x64 => self.high_max_level_idc,
-            _ => None,
-        }
-    }
-}
-
 pub(crate) fn filter_h264_codecs_for_vulkan_encoder(
     ctx: &Arc<PipelineCtx>,
     codecs: Vec<RTCRtpCodecParameters>,
 ) -> Vec<RTCRtpCodecParameters> {
-    let Some(support) = vulkan_h264_encode_profile_level_support(ctx) else {
+    let Some(support) = ctx
+        .graphics_context
+        .vulkan_h264_encode_profile_level_support()
+    else {
         return codecs;
     };
 
@@ -37,7 +22,10 @@ pub(crate) fn filter_h264_codecs_for_vulkan_decoder(
     ctx: &Arc<PipelineCtx>,
     codecs: Vec<RTCRtpCodecParameters>,
 ) -> Vec<RTCRtpCodecParameters> {
-    let Some(support) = vulkan_h264_decode_profile_level_support(ctx) else {
+    let Some(support) = ctx
+        .graphics_context
+        .vulkan_h264_decode_profile_level_support()
+    else {
         return codecs;
     };
 
@@ -46,7 +34,7 @@ pub(crate) fn filter_h264_codecs_for_vulkan_decoder(
 
 fn filter_h264_codecs_by_profile_level_support(
     codecs: Vec<RTCRtpCodecParameters>,
-    support: H264ProfileLevelSupport,
+    support: crate::graphics_context::H264ProfileLevelSupport,
 ) -> Vec<RTCRtpCodecParameters> {
     codecs
         .into_iter()
@@ -81,45 +69,3 @@ fn h264_profile_level_idc_from_fmtp(fmtp: &str) -> Option<(u8, u8)> {
 
     None
 }
-
-#[cfg(feature = "vk-video")]
-fn vulkan_h264_encode_profile_level_support(
-    ctx: &Arc<PipelineCtx>,
-) -> Option<H264ProfileLevelSupport> {
-    let vulkan_ctx = ctx.graphics_context.vulkan_ctx.as_ref()?;
-    let caps = vulkan_ctx.device.encode_capabilities().h264?;
-
-    Some(H264ProfileLevelSupport {
-        baseline_max_level_idc: caps.baseline_profile.map(|p| p.max_level_idc),
-        main_max_level_idc: caps.main_profile.map(|p| p.max_level_idc),
-        high_max_level_idc: caps.high_profile.map(|p| p.max_level_idc),
-    })
-}
-
-#[cfg(not(feature = "vk-video"))]
-fn vulkan_h264_encode_profile_level_support(
-    _ctx: &Arc<PipelineCtx>,
-) -> Option<H264ProfileLevelSupport> {
-    None
-}
-
-#[cfg(feature = "vk-video")]
-fn vulkan_h264_decode_profile_level_support(
-    ctx: &Arc<PipelineCtx>,
-) -> Option<H264ProfileLevelSupport> {
-    let vulkan_ctx = ctx.graphics_context.vulkan_ctx.as_ref()?;
-    let caps = vulkan_ctx.device.decode_capabilities().h264?;
-
-    Some(H264ProfileLevelSupport {
-        baseline_max_level_idc: caps.baseline_profile.map(|p| p.max_level_idc),
-        main_max_level_idc: caps.main_profile.map(|p| p.max_level_idc),
-        high_max_level_idc: caps.high_profile.map(|p| p.max_level_idc),
-    })
-}
-
-#[cfg(not(feature = "vk-video"))]
-fn vulkan_h264_decode_profile_level_support(
-    _ctx: &Arc<PipelineCtx>,
-) -> Option<H264ProfileLevelSupport> {
-    None
-}
diff --git a/smelter-core/src/pipeline/webrtc/supported_codec_parameters.rs b/smelter-core/src/pipeline/webrtc/supported_codec_parameters.rs
@@ -50,33 +50,30 @@ pub fn h264_codec_params() -> Vec<RTCRtpCodecParameters> {
         "640033", // high, 5.1
     ];
 
-    let mut params = Vec::with_capacity(profile_level_ids.len() * 2);
-    let mut payload_type = 100u8;
+    let opus_payload_types: [u8; 2] = [110, 111];
+    let payload_types = (100u8..).filter(|pt| !opus_payload_types.contains(pt));
 
-    for plid in profile_level_ids {
-        for pmode in [1, 0] {
-            params.push(RTCRtpCodecParameters {
-                capability: RTCRtpCodecCapability {
-                    mime_type: MIME_TYPE_H264.to_owned(),
-                    clock_rate: 90000,
-                    channels: 0,
-                    sdp_fmtp_line: format!(
-                        "level-asymmetry-allowed=1;packetization-mode={pmode};profile-level-id={plid}"
-                    ),
-                    rtcp_feedback: get_video_rtcp_feedback(),
-                },
-                payload_type,
-                ..Default::default()
-            });
-
-            payload_type += 1;
-        }
-    }
-
-    params
+    profile_level_ids
+        .iter()
+        .flat_map(|plid| [1, 0].map(|pmode| (plid, pmode)))
+        .zip(payload_types)
+        .map(|((plid, pmode), payload_type)| RTCRtpCodecParameters {
+            capability: RTCRtpCodecCapability {
+                mime_type: MIME_TYPE_H264.to_owned(),
+                clock_rate: 90000,
+                channels: 0,
+                sdp_fmtp_line: format!(
+                    "level-asymmetry-allowed=1;packetization-mode={pmode};profile-level-id={plid}"
+                ),
+                rtcp_feedback: get_video_rtcp_feedback(),
+            },
+            payload_type,
+            ..Default::default()
+        })
+        .collect()
 }
 
-fn get_video_rtcp_feedback() -> Vec<RTCPFeedback> {
+pub(crate) fn get_video_rtcp_feedback() -> Vec<RTCPFeedback> {
     vec![
         RTCPFeedback {
             typ: "goog-remb".to_owned(),
diff --git a/smelter-core/src/pipeline/webrtc/whep_input/input.rs b/smelter-core/src/pipeline/webrtc/whep_input/input.rs
@@ -34,6 +34,7 @@ pub(crate) struct WhepInput {
     ctx: Arc<PipelineCtx>,
     session_url: Url,
     client: Arc<WhipWhepHttpClient>,
+    _peer_connection: RecvonlyPeerConnection,
 }
 
 impl WhepInput {
@@ -157,6 +158,7 @@ async fn init_whep_client(
             ctx,
             session_url,
             client,
+            _peer_connection: pc,
         }),
         InputInitInfo::Other,
         QueueDataReceiver {
diff --git a/smelter-core/src/pipeline/webrtc/whep_output/peer_connection.rs b/smelter-core/src/pipeline/webrtc/whep_output/peer_connection.rs
@@ -31,9 +31,9 @@ use webrtc::{
 
 use crate::pipeline::webrtc::{
     error::WhipWhepServerError,
-    h264_offer_filter::filter_h264_codecs_by_offer,
+    h264_offer_filter::h264_codecs_from_offer,
     h264_vulkan_capability_filter::filter_h264_codecs_for_vulkan_encoder,
-    supported_codec_parameters::{h264_codec_params, vp8_codec_params, vp9_codec_params},
+    supported_codec_parameters::{vp8_codec_params, vp9_codec_params},
 };
 
 use crate::prelude::*;
@@ -291,13 +291,12 @@ fn register_codecs(
     if let Some(encoder) = video_encoder {
         match encoder {
             VideoEncoderOptions::FfmpegH264(_) => {
-                let codecs = filter_h264_codecs_by_offer(offer, h264_codec_params());
-                for codec in codecs {
+                for codec in h264_codecs_from_offer(offer) {
                     media_engine.register_codec(codec, RTPCodecType::Video)?;
                 }
             }
             VideoEncoderOptions::VulkanH264(_) => {
-                let codecs = filter_h264_codecs_by_offer(offer, h264_codec_params());
+                let codecs = h264_codecs_from_offer(offer);
                 for codec in filter_h264_codecs_for_vulkan_encoder(ctx, codecs) {
                     media_engine.register_codec(codec, RTPCodecType::Video)?;
                 }
diff --git a/smelter-core/src/pipeline/webrtc/whip_input/video_preferences.rs b/smelter-core/src/pipeline/webrtc/whip_input/video_preferences.rs
diff --git a/vk-video/src/device/caps.rs b/vk-video/src/device/caps.rs
