diff --git a/mm-client/build.rs b/mm-client/build.rs index 45f50a1..2a18ce2 100644 --- a/mm-client/build.rs +++ b/mm-client/build.rs @@ -42,6 +42,7 @@ fn compile_shader( let mut compile_request = session.create_compile_request(); compile_request + .add_search_path("../shader-common") .set_codegen_target(slang::CompileTarget::Spirv) .set_optimization_level(slang::OptimizationLevel::Maximal) .set_target_profile(session.find_profile("glsl_460")); diff --git a/mm-client/src/audio.rs b/mm-client/src/audio.rs index 599faf8..2795ab4 100644 --- a/mm-client/src/audio.rs +++ b/mm-client/src/audio.rs @@ -146,6 +146,7 @@ where if frames_remaining < frames_needed { out.fill(Default::default()); + trace!("audio buffer underrun"); return; } diff --git a/mm-client/src/bin/mmclient.rs b/mm-client/src/bin/mmclient.rs index d46997f..dd36ae1 100644 --- a/mm-client/src/bin/mmclient.rs +++ b/mm-client/src/bin/mmclient.rs @@ -96,9 +96,14 @@ struct Cli { /// resizes. #[arg(long, required = false, default_value = "auto")] resolution: Resolution, + /// Request 10-bit video output from the server. This will only work if + /// both your display and the application in question support rendering + /// HDR color. + #[arg(long, required = false)] + hdr: bool, /// The UI scale to communicate to the server. If not specified, this will /// be determined from the client-side window scale factor. - #[arg(long)] + #[arg(long, required = false)] ui_scale: Option, /// Video codec to use. #[arg(long, default_value = "h265")] @@ -121,6 +126,7 @@ struct MainLoop { struct App { configured_resolution: Resolution, configured_codec: protocol::VideoCodec, + configured_profile: protocol::VideoProfile, configured_framerate: u32, window: Arc, @@ -621,6 +627,7 @@ impl App { session_id: self.session_id, streaming_resolution: self.remote_display_params.resolution.clone(), video_codec: self.configured_codec.into(), + video_profile: self.configured_profile.into(), ..Default::default() }, None, @@ -815,6 +822,12 @@ fn main() -> Result<()> { Some(v) => bail!("invalid codec: {:?}", v), }; + let configured_profile = if args.hdr { + protocol::VideoProfile::Hdr10 + } else { + protocol::VideoProfile::Hd + }; + // TODO: anyhow errors are garbage for end-users. debug!("establishing connection to {:}", &args.host); let mut conn = Conn::new(&args.host).context("failed to establish connection")?; @@ -927,7 +940,7 @@ fn main() -> Result<()> { window.clone(), cfg!(debug_assertions), )?); - let renderer = Renderer::new(vk.clone(), window.clone())?; + let renderer = Renderer::new(vk.clone(), window.clone(), args.hdr)?; debug!("attaching session {:?}", session.session_id); let attachment_sid = conn.send( @@ -937,6 +950,7 @@ fn main() -> Result<()> { session_id: session.session_id, streaming_resolution: Some(streaming_resolution), video_codec: configured_codec.into(), + video_profile: configured_profile.into(), ..Default::default() }, None, @@ -964,6 +978,7 @@ fn main() -> Result<()> { configured_codec, configured_framerate: args.framerate, configured_resolution: args.resolution, + configured_profile, window, _proxy: proxy.clone(), diff --git a/mm-client/src/render.rs b/mm-client/src/render.rs index 8c7cedb..847c4c2 100644 --- a/mm-client/src/render.rs +++ b/mm-client/src/render.rs @@ -23,16 +23,36 @@ use crate::vulkan::*; const FONT_SIZE: f32 = 8.0; +// Matches the definition in render.slang. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +enum TextureColorSpace { + Bt709 = 0, + Bt2020Pq = 1, +} + +impl From for TextureColorSpace { + fn from(cs: crate::video::ColorSpace) -> Self { + match cs { + crate::video::ColorSpace::Bt709 => TextureColorSpace::Bt709, + crate::video::ColorSpace::Bt2020Pq => TextureColorSpace::Bt2020Pq, + } + } +} + #[derive(Copy, Clone, Debug)] #[repr(C)] struct PushConstants { aspect: glam::Vec2, + texture_color_space: TextureColorSpace, + output_color_space: vk::ColorSpaceKHR, } pub struct Renderer { width: u32, height: u32, scale_factor: f64, + hdr_mode: bool, imgui: imgui::Context, imgui_platform: imgui_winit_support::WinitPlatform, @@ -43,12 +63,18 @@ pub struct Renderer { swapchain: Option, swapchain_dirty: bool, - video_texture: Option, + new_video_texture: Option<(Arc, VideoStreamParams)>, vk: Arc, window: Arc, } +struct VideoTexture { + image: Arc, + view: vk::ImageView, + color_space: TextureColorSpace, +} + struct Swapchain { swapchain: vk::SwapchainKHR, frames: Vec, @@ -57,11 +83,13 @@ struct Swapchain { sampler_conversion: vk::SamplerYcbcrConversion, sampler: vk::Sampler, - bound_video_texture: Option<(Arc, vk::ImageView)>, + bound_video_texture: Option, + /// The normalized relationship between the output and the video texture, /// after scaling. For example, a 500x500 video texture in a 1000x500 /// swapchain would have the aspect (2.0, 1.0), as would a 250x250 texture. aspect: (f64, f64), + surface_format: vk::SurfaceFormatKHR, descriptor_set_layout: vk::DescriptorSetLayout, descriptor_pool: vk::DescriptorPool, pipeline_layout: vk::PipelineLayout, @@ -85,13 +113,12 @@ struct SwapImage { view: vk::ImageView, } -struct VideoTexture { - params: VideoStreamParams, - texture: Arc, -} - impl Renderer { - pub fn new(vk: Arc, window: Arc) -> Result { + pub fn new( + vk: Arc, + window: Arc, + hdr_mode: bool, + ) -> Result { let window_size = window.inner_size(); let scale_factor = window.scale_factor(); @@ -112,6 +139,7 @@ impl Renderer { width: window_size.width, height: window_size.height, scale_factor, + hdr_mode, window, imgui, imgui_platform, @@ -120,7 +148,7 @@ impl Renderer { imgui_time: time::Instant::now(), swapchain: None, swapchain_dirty: false, - video_texture: None, + new_video_texture: None, vk, }; @@ -134,8 +162,7 @@ impl Renderer { let start = time::Instant::now(); let device = &self.vk.device; - let surface_format = select_surface_format(self.vk.clone())?; - trace!(?surface_format, "surface format"); + let surface_format = select_surface_format(self.vk.clone(), self.hdr_mode)?; let surface_capabilities = self .vk @@ -219,12 +246,16 @@ impl Renderer { // We need to create a sampler, even if we don't have a video stream yet // and don't know what the fields should be. - let video_params = match self.video_texture.as_ref() { - Some(tex) => tex.params, - None => VideoStreamParams::default(), + let (video_texture_format, video_params) = match self.new_video_texture.as_ref() { + Some((tex, params)) => (tex.format, *params), + None => ( + vk::Format::G8_B8_R8_3PLANE_420_UNORM, + VideoStreamParams::default(), + ), }; - let sampler_conversion = sampler_conversion(device, &video_params)?; + let sampler_conversion = + create_ycbcr_sampler_conversion(device, video_texture_format, &video_params)?; let sampler = { let mut conversion_info = vk::SamplerYcbcrConversionInfo::builder() @@ -243,22 +274,26 @@ impl Renderer { unsafe { device.create_sampler(&create_info, None)? } }; - let bound_video_texture = if let Some(tex) = self.video_texture.as_ref() { + let bound_video_texture = if let Some((tex, params)) = self.new_video_texture.as_ref() { let view = create_image_view( &self.vk.device, - tex.texture.image, - tex.texture.format, + tex.image, + tex.format, Some(sampler_conversion), )?; // Increment the reference count on the texture. - Some((tex.texture.clone(), view)) + Some(VideoTexture { + image: tex.clone(), + view, + color_space: params.color_space.into(), + }) } else { None }; - let aspect = if let Some((tex, _)) = bound_video_texture.as_ref() { - calculate_aspect(self.width, self.height, tex.width, tex.height) + let aspect = if let Some(tex) = bound_video_texture.as_ref() { + calculate_aspect(self.width, self.height, tex.image.width, tex.image.height) } else { (1.0, 1.0) }; @@ -294,7 +329,7 @@ impl Renderer { let pipeline_layout = { let pc_ranges = [vk::PushConstantRange::builder() - .stage_flags(vk::ShaderStageFlags::VERTEX) + .stage_flags(vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT) .offset(0) .size(std::mem::size_of::() as u32) .build()]; @@ -435,10 +470,10 @@ impl Renderer { .unwrap(); // TODO: do the write in bind_video_texture? - if let Some((_, view)) = bound_video_texture.as_ref() { + if let Some(tex) = bound_video_texture.as_ref() { let info = vk::DescriptorImageInfo::builder() .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) - .image_view(*view); + .image_view(tex.view); let image_info = &[info.build()]; let sampler_write = vk::WriteDescriptorSet::builder() @@ -522,6 +557,7 @@ impl Renderer { sampler, bound_video_texture, aspect, + surface_format, pipeline_layout, pipeline, @@ -592,7 +628,7 @@ impl Renderer { params: VideoStreamParams, ) -> Result<()> { // TODO: no need to recreate the sampler if the params match. - self.video_texture = Some(VideoTexture { params, texture }); + self.new_video_texture = Some((texture, params)); self.swapchain_dirty = true; Ok(()) } @@ -603,7 +639,7 @@ impl Renderer { // 1.0). pub fn get_texture_aspect(&self) -> Option<(f64, f64)> { if let Some(Swapchain { - bound_video_texture: Some((_, _)), + bound_video_texture: Some(_), aspect, .. }) = self.swapchain.as_ref() @@ -738,21 +774,23 @@ impl Renderer { ); } - if self.video_texture.is_none() || swapchain.aspect != (1.0, 1.0) { + if self.new_video_texture.is_none() || swapchain.aspect != (1.0, 1.0) { // TODO Draw the background // https://www.toptal.com/designers/subtlepatterns/prism/ } // Draw the video texture. - if let Some((_texture, _)) = &swapchain.bound_video_texture { + if let Some(tex) = &swapchain.bound_video_texture { let pc = PushConstants { aspect: glam::Vec2::new(swapchain.aspect.0 as f32, swapchain.aspect.1 as f32), + texture_color_space: tex.color_space, + output_color_space: swapchain.surface_format.color_space, }; device.cmd_push_constants( frame.render_cb, swapchain.pipeline_layout, - vk::ShaderStageFlags::VERTEX, + vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT, 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, @@ -899,8 +937,8 @@ impl Renderer { device.destroy_sampler(swapchain.sampler, None); device.destroy_sampler_ycbcr_conversion(swapchain.sampler_conversion, None); - if let Some((_img, view)) = swapchain.bound_video_texture.take() { - device.destroy_image_view(view, None); + if let Some(tex) = swapchain.bound_video_texture.take() { + device.destroy_image_view(tex.view, None); // We probably drop the last reference to the image here, which then // gets destroyed. } @@ -912,8 +950,11 @@ impl Renderer { } } -fn select_surface_format(vk: Arc) -> Result { - let surface_formats = unsafe { +fn select_surface_format( + vk: Arc, + hdr_mode: bool, +) -> Result { + let mut surface_formats = unsafe { vk.surface_loader .get_physical_device_surface_formats(vk.pdevice, vk.surface)? }; @@ -924,16 +965,36 @@ fn select_surface_format(vk: Arc) -> Result= 0.0031) - return 1.055 * pow(s, 1.0 / 2.4) - 0.055; - else - return s * 12.92; + if (vk_color_space == VK_COLOR_SPACE_BT709_NONLINEAR_EXT) + { + return color; + } + + let linear = bt709_eotf(color); + switch (vk_color_space) + { + case VK_COLOR_SPACE_SRGB_NONLINEAR_EXT: + return srgb_inverse_eotf(linear); + case VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT: + return linear; + // case VK_COLOR_SPACE_DISPLAY_P3_NONLINEAR_EXT: + // return srgb_inverse_eotf(transform(color, Primaries::BT709, Primaries::P3)) + case VK_COLOR_SPACE_HDR10_ST2084_EXT: + return pq_inverse_eotf(transform(linear, Primaries::BT709, Primaries::BT2020)); + default: + return srgb_inverse_eotf(linear); + } } -float bt709_linearize(float s) +float3 bt2020_pq_to_display(float3 color, int vk_color_space) { - if (s > 0.081) - return pow((s + 0.099) / 1.099, 1.0 / 0.45); - else - return s / 4.5; + if (vk_color_space == VK_COLOR_SPACE_HDR10_ST2084_EXT) + { + return color; + } + + let linear = transform(pq_eotf(color) * PQ_MAX_WHITE / SDR_REFERENCE_WHITE, Primaries::BT2020, Primaries::BT709); + switch (vk_color_space) + { + case VK_COLOR_SPACE_SRGB_NONLINEAR_EXT: + return srgb_inverse_eotf(linear); + case VK_COLOR_SPACE_BT709_NONLINEAR_EXT: + return bt709_inverse_eotf(clamp(linear, 0.0, 1.0)); + case VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT: + return linear; + // case VK_COLOR_SPACE_DISPLAY_P3_NONLINEAR_EXT: + // return srgb_inverse_eotf(transform(color, Primaries::BT2020, Primaries::P3)) + default: + return srgb_inverse_eotf(linear); + } } [shader("fragment")] float4 frag(float2 uv: TextureCoord) : SV_Target { - float4 color = texture.Sample(uv); + float4 color = clamp(texture.Sample(uv), 0.0, 1.0); // When sampling the video texture, vulkan does the matrix multiplication - // for us, but doesn't apply any transfer function. So we need to convert - // from the BT.709 transfer function to the sRGB one. - return float4( - srgb_unlinear(bt709_linearize(color.r)), - srgb_unlinear(bt709_linearize(color.g)), - srgb_unlinear(bt709_linearize(color.b)), - color.a); + // for us, but doesn't apply any transfer function, so the values are + // still nonlinear in either BT.709 or BT.2020/ST2048. + switch (pc.texture_color_space) + { + case TextureColorSpace::Bt709: + return float4(bt709_to_display(color.rgb, pc.vk_color_space), 1.0); + case TextureColorSpace::Bt2020Pq: + return float4(bt2020_pq_to_display(color.rgb, pc.vk_color_space), 1.0); + default: + return float4(0.0, 0.5, 1.0, 1.0); + } } diff --git a/mm-client/src/video.rs b/mm-client/src/video.rs index 1bc38f4..7015794 100644 --- a/mm-client/src/video.rs +++ b/mm-client/src/video.rs @@ -7,7 +7,7 @@ use std::{ time, }; -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, bail, Context}; use ash::vk; use bytes::{Bytes, BytesMut}; use ffmpeg_next as ffmpeg; @@ -31,20 +31,25 @@ pub struct FrameMetadata { pub pts: u64, } +#[derive(Debug, Clone)] struct YUVPicture { - y: Bytes, - u: Bytes, - v: Bytes, + planes: [Bytes; 3], + num_planes: usize, info: FrameMetadata, } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ColorSpace { + Bt709, + Bt2020Pq, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct VideoStreamParams { pub width: u32, pub height: u32, - pub pixel_format: ffmpeg::format::Pixel, - pub color_space: ffmpeg::color::Space, - pub color_range: ffmpeg::color::Range, + pub color_space: ColorSpace, + pub color_full_range: bool, } impl Default for VideoStreamParams { @@ -52,9 +57,8 @@ impl Default for VideoStreamParams { Self { width: 0, height: 0, - pixel_format: ffmpeg::format::Pixel::YUV420P, - color_space: ffmpeg::color::Space::BT709, - color_range: ffmpeg::color::Range::MPEG, + color_space: ColorSpace::Bt709, + color_full_range: false, } } } @@ -387,10 +391,29 @@ impl DecoderInit { match self.decoder.receive_frame(&mut frame) { Ok(()) => { self.first_frame = match frame.format() { - ffmpeg::format::Pixel::YUV420P => Some((frame, info)), ffmpeg::format::Pixel::VIDEOTOOLBOX => { + let sw_format = unsafe { + let ctx_ref = (*self.decoder.as_ptr()).hw_frames_ctx; + assert!(!ctx_ref.is_null()); + + let mut transfer_fmt_list = std::ptr::null_mut(); + if ffmpeg_sys::av_hwframe_transfer_get_formats( + ctx_ref, + ffmpeg_sys::AVHWFrameTransferDirection::AV_HWFRAME_TRANSFER_DIRECTION_FROM, + &mut transfer_fmt_list, + 0) < 0 + { + bail!("call to av_hwframe_transfer_get_formats failed"); + }; + + let transfer_formats = read_format_list(transfer_fmt_list); + assert!(!transfer_formats.is_empty()); + + transfer_formats[0] + }; + let mut sw_frame = ffmpeg::frame::Video::new( - ffmpeg::format::Pixel::YUV420P, + sw_format, self.decoder.width(), self.decoder.height(), ); @@ -409,7 +432,7 @@ impl DecoderInit { Some((sw_frame, info)) } } - f => return Err(anyhow!("unexpected stream format: {:?}", f)), + _ => Some((frame, info)), }; Ok(true) @@ -439,25 +462,31 @@ impl DecoderInit { None => return Err(anyhow!("no frames received yet")), }; - // debug_assert_eq!(self.decoder.color_space(), ffmpeg::color::Space::BT709); - - let output_format = first_frame.0.format(); - assert_eq!(output_format, ffmpeg::format::Pixel::YUV420P); - // If we're using VideoToolbox, create a "hardware" frame to use with // receive_frame. + let output_format = first_frame.0.format(); + let ((mut frame, info), mut hw_frame) = match decoder_format { - ffmpeg::format::Pixel::YUV420P => (first_frame, None), ffmpeg::format::Pixel::VIDEOTOOLBOX => { let hw_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::VIDEOTOOLBOX, width, height); (first_frame, Some(hw_frame)) } - _ => return Err(anyhow!("unexpected stream format: {:?}", decoder_format)), + _ => (first_frame, None), }; - let texture_format = vk::Format::G8_B8_R8_3PLANE_420_UNORM; + // For 10-bit textures, we need to end up in on the GPU in P010LE, + // because that's better supported. To make the copy easier, we'll use + // swscale to convert to a matching intermediate format. + let (intermediate_format, texture_format) = match output_format { + ffmpeg::format::Pixel::YUV420P => (None, vk::Format::G8_B8_R8_3PLANE_420_UNORM), + ffmpeg::format::Pixel::YUV420P10 | ffmpeg::format::Pixel::YUV420P10LE => ( + Some(ffmpeg::format::Pixel::P010LE), + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + ), + _ => return Err(anyhow!("unexpected pixel format: {:?}", output_format)), + }; debug_assert_eq!(frame.width(), width); debug_assert_eq!(frame.height(), height); @@ -472,22 +501,26 @@ impl DecoderInit { )); } - let y_stride = frame.stride(0); - let u_stride = frame.stride(1); - let v_stride = frame.stride(2); - - let y_len = y_stride * frame.plane_height(0) as usize; - let u_len = u_stride * frame.plane_height(1) as usize; - let v_len = v_stride * frame.plane_height(2) as usize; - - // This is vanishingly unlikely, and I have no idea how the pixels - // would be layed out in that case. - debug_assert_eq!(y_len % 4, 0); - - // Precalculate the offsets into the buffer for each plane. - let buffer_size = y_len + u_len + v_len; - let buffer_offsets = [0, y_len, y_len + u_len]; - let buffer_strides = [y_stride, u_stride, v_stride]; + let mut intermediate_frame = + intermediate_format.map(|fmt| ffmpeg::frame::Video::new(fmt, width, height)); + + // For the purposes of determining the size of and offsets into the + // staging buffer, we use the intermediate frame if it exists, otherwise + // the output frame. + let model_frame = intermediate_frame.as_ref().unwrap_or(&frame); + + // Precalculate the layout of the staging buffer. + let mut buffer_strides = [0; 3]; + let mut buffer_offsets = [0; 3]; + let mut buffer_size = 0; + for plane in 0..model_frame.planes() { + let stride = model_frame.stride(plane); + let len = stride * model_frame.plane_height(plane) as usize; + + buffer_strides[plane] = stride; + buffer_offsets[plane] = buffer_size; + buffer_size += len; + } let staging_buffer = create_host_buffer( &self.vk.device, @@ -496,21 +529,32 @@ impl DecoderInit { buffer_size, )?; - let color_space = match self.decoder.color_space() { - ffmpeg::color::Space::BT709 => ffmpeg::color::Space::BT709, - ffmpeg::color::Space::BT2020NCL => ffmpeg::color::Space::BT2020NCL, - cs => { - warn!("unexpected color space: {:?}", cs); - ffmpeg::color::Space::BT709 + let color_space = match ( + self.decoder.color_space(), + self.decoder.color_transfer_characteristic(), + ) { + (ffmpeg::color::Space::BT709, ffmpeg::color::TransferCharacteristic::BT709) => { + ColorSpace::Bt709 + } + (ffmpeg::color::Space::BT2020NCL, ffmpeg::color::TransferCharacteristic::SMPTE2084) => { + ColorSpace::Bt2020Pq } + ( + ffmpeg::color::Space::Unspecified, + ffmpeg::color::TransferCharacteristic::Unspecified, + ) => { + warn!("video stream has unspecified color primaries or transfer function"); + ColorSpace::Bt709 + } + (cs, ctrc) => bail!("unexpected color description: {:?} / {:?}", cs, ctrc), }; - let color_range = match self.decoder.color_range() { - ffmpeg::color::Range::MPEG => ffmpeg::color::Range::MPEG, - ffmpeg::color::Range::JPEG => ffmpeg::color::Range::JPEG, + let color_full_range = match self.decoder.color_range() { + ffmpeg::color::Range::MPEG => false, + ffmpeg::color::Range::JPEG => true, cr => { warn!("unexpected color range: {:?}", cr); - ffmpeg::color::Range::MPEG + false } }; @@ -536,7 +580,12 @@ impl DecoderInit { // Send the frame we have from before. decoded_send - .send(copy_frame(&mut frame, &mut BytesMut::new(), info)) + .send(copy_frame( + &mut frame, + intermediate_frame.as_mut(), + &mut BytesMut::new(), + info, + )) .unwrap(); // Spawn another thread that receives packets on one channel and sends @@ -580,11 +629,12 @@ impl DecoderInit { loop { match receive_frame(&mut decoder, &mut frame, hw_frame.as_mut()) { Ok(()) => { - let pic = copy_frame(&mut frame, &mut scratch, info); - - debug_assert_eq!(pic.y.len(), y_len); - debug_assert_eq!(pic.u.len(), u_len); - debug_assert_eq!(pic.v.len(), v_len); + let pic = copy_frame( + &mut frame, + intermediate_frame.as_mut(), + &mut scratch, + info, + ); let span = trace_span!("send"); let _guard = span.enter(); @@ -640,9 +690,8 @@ impl DecoderInit { let params = VideoStreamParams { width, height, - pixel_format: output_format, color_space, - color_range, + color_full_range, }; Ok((dec, video_texture, params)) @@ -715,14 +764,15 @@ impl CPUDecoder { // Copy data into the staging buffer. self.yuv_buffer_offsets .iter() - .zip([pic.y, pic.u, pic.v]) + .zip(pic.planes.iter()) + .take(pic.num_planes) .for_each(|(offset, src)| { let dst = std::slice::from_raw_parts_mut( (self.staging_buffer.access as *mut u8).add(*offset), src.len(), ); - dst.copy_from_slice(&src); + dst.copy_from_slice(src); }); // Trace the upload, including loading timestamps for the previous upload. @@ -793,6 +843,12 @@ impl CPUDecoder { // Upload from the staging buffer to the texture. { + let num_planes = match self.video_texture.format { + vk::Format::G8_B8_R8_3PLANE_420_UNORM => 3, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 => 2, + _ => unreachable!(), + }; + let regions = [ vk::ImageAspectFlags::PLANE_0, vk::ImageAspectFlags::PLANE_1, @@ -800,6 +856,7 @@ impl CPUDecoder { ] .into_iter() .enumerate() + .take(num_planes) .map(|(plane, plane_aspect_mask)| { // Vulkan considers the image width/height to be 1/2 the size // for the U and V planes. @@ -809,10 +866,21 @@ impl CPUDecoder { (self.texture_width / 2, self.texture_height / 2) }; + let texel_width = match self.video_texture.format { + vk::Format::G8_B8_R8_3PLANE_420_UNORM => 1, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 => { + if plane == 0 { + 2 + } else { + 4 + } + } + _ => unreachable!(), + }; + vk::BufferImageCopy::builder() .buffer_offset(self.yuv_buffer_offsets[plane] as u64) - // This is actually in texels, but each plane uses 1bpp. - .buffer_row_length(self.yuv_buffer_strides[plane] as u32) + .buffer_row_length((self.yuv_buffer_strides[plane] / texel_width) as u32) // In texels. .image_subresource(vk::ImageSubresourceLayers { aspect_mask: plane_aspect_mask, mip_level: 0, @@ -935,58 +1003,93 @@ fn copy_packet(pkt: &mut ffmpeg::Packet, buf: Undecoded) -> anyhow::Result<()> { #[instrument(skip_all)] fn copy_frame( frame: &mut ffmpeg::frame::Video, + intermediate_frame: Option<&mut ffmpeg::frame::Video>, scratch: &mut BytesMut, info: FrameMetadata, ) -> YUVPicture { - scratch.truncate(0); - - scratch.extend_from_slice(frame.data(0)); - let y = scratch.split().freeze(); + let transfer_src = if let Some(intermediate) = intermediate_frame { + // TODO reuse + let mut ctx = ffmpeg::software::scaling::Context::get( + frame.format(), + frame.width(), + frame.height(), + intermediate.format(), + intermediate.width(), + intermediate.height(), + ffmpeg::software::scaling::Flags::empty(), + ) + .expect("failed to create sws ctx"); + + ctx.run(frame, intermediate).expect("failed to convert"); + + intermediate + } else { + frame + }; - scratch.extend_from_slice(frame.data(1)); - let u = scratch.split().freeze(); + let mut pic = YUVPicture { + planes: [Bytes::new(), Bytes::new(), Bytes::new()], + num_planes: transfer_src.planes(), + info, + }; - scratch.extend_from_slice(frame.data(2)); - let v = scratch.split().freeze(); + scratch.truncate(0); + for plane in 0..transfer_src.planes() { + scratch.extend_from_slice(transfer_src.data(plane)); + pic.planes[plane] = scratch.split().freeze(); + } - YUVPicture { y, u, v, info } + pic } #[no_mangle] unsafe extern "C" fn get_hw_format_videotoolbox( ctx: *mut ffmpeg_sys::AVCodecContext, - mut formats: *const ffmpeg_sys::AVPixelFormat, + list: *const ffmpeg_sys::AVPixelFormat, ) -> ffmpeg_sys::AVPixelFormat { use ffmpeg_sys::AVPixelFormat::*; - while *formats != AV_PIX_FMT_NONE { - if *formats == AV_PIX_FMT_VIDEOTOOLBOX { - let frames_ctx_ref = ffmpeg_sys::av_hwframe_ctx_alloc((*ctx).hw_device_ctx); - if frames_ctx_ref.is_null() { - error!("call to av_hwframe_ctx_alloc failed"); - break; - } + let sw_pix_fmt = (*ctx).sw_pix_fmt; + let formats = read_format_list(list); - let frames_ctx = (*frames_ctx_ref).data as *mut ffmpeg_sys::AVHWFramesContext; - (*frames_ctx).width = (*ctx).width; - (*frames_ctx).height = (*ctx).height; - (*frames_ctx).format = AV_PIX_FMT_VIDEOTOOLBOX; - (*frames_ctx).sw_format = AV_PIX_FMT_YUV420P; + if formats.contains(&ffmpeg::format::Pixel::VIDEOTOOLBOX) { + let frames_ctx_ref = ffmpeg_sys::av_hwframe_ctx_alloc((*ctx).hw_device_ctx); + if frames_ctx_ref.is_null() { + error!("call to av_hwframe_ctx_alloc failed"); + return sw_pix_fmt; + } - let res = ffmpeg_sys::av_hwframe_ctx_init(frames_ctx_ref); - if res < 0 { - error!("call to av_hwframe_ctx_init failed"); - break; - } + debug!(?formats, sw_pix_fmt = ?sw_pix_fmt, "get_hw_format_videotoolbox"); - debug!("using VideoToolbox hardware encoder"); - (*ctx).hw_frames_ctx = frames_ctx_ref; - return *formats; + let frames_ctx = (*frames_ctx_ref).data as *mut ffmpeg_sys::AVHWFramesContext; + (*frames_ctx).width = (*ctx).width; + (*frames_ctx).height = (*ctx).height; + (*frames_ctx).format = AV_PIX_FMT_VIDEOTOOLBOX; + (*frames_ctx).sw_format = AV_PIX_FMT_YUV420P; + + let res = ffmpeg_sys::av_hwframe_ctx_init(frames_ctx_ref); + if res < 0 { + error!("call to av_hwframe_ctx_init failed"); + return sw_pix_fmt; } - formats = formats.add(1); + debug!("using VideoToolbox hardware encoder"); + (*ctx).hw_frames_ctx = frames_ctx_ref; + return AV_PIX_FMT_VIDEOTOOLBOX; + } + + warn!("unable to determine ffmpeg hw format"); + sw_pix_fmt +} + +unsafe fn read_format_list( + mut ptr: *const ffmpeg_sys::AVPixelFormat, +) -> Vec { + let mut formats = Vec::new(); + while !ptr.is_null() && *ptr != ffmpeg_sys::AVPixelFormat::AV_PIX_FMT_NONE { + formats.push((*ptr).into()); + ptr = ptr.add(1); } - warn!("VideoToolbox setup failed, falling back to CPU decoder"); - AV_PIX_FMT_YUV420P + formats } diff --git a/mm-client/src/vulkan.rs b/mm-client/src/vulkan.rs index 417e785..32a734c 100644 --- a/mm-client/src/vulkan.rs +++ b/mm-client/src/vulkan.rs @@ -21,10 +21,11 @@ use ash::{ vk, }; use cstr::cstr; -use ffmpeg_next as ffmpeg; use raw_window_handle::{HasRawDisplayHandle, HasRawWindowHandle}; use tracing::{debug, error, info, warn}; +use crate::video::ColorSpace; + pub struct VkDebugContext { debug: DebugUtils, messenger: vk::DebugUtilsMessengerEXT, @@ -957,34 +958,24 @@ pub fn load_shader(device: &ash::Device, bytes: &[u8]) -> anyhow::Result anyhow::Result { let ycbcr_model = match params.color_space { - ffmpeg::color::Space::BT709 => vk::SamplerYcbcrModelConversion::YCBCR_709, - ffmpeg::color::Space::BT2020NCL => vk::SamplerYcbcrModelConversion::YCBCR_2020, - _ => return Err(anyhow!("unsupported color space: {:?}", params.color_space)), - }; - - let ycbcr_range = match params.color_range { - ffmpeg::color::Range::MPEG => vk::SamplerYcbcrRange::ITU_NARROW, - ffmpeg::color::Range::JPEG => vk::SamplerYcbcrRange::ITU_FULL, - _ => return Err(anyhow!("unsupported color range: {:?}", params.color_range)), + ColorSpace::Bt709 => vk::SamplerYcbcrModelConversion::YCBCR_709, + ColorSpace::Bt2020Pq => vk::SamplerYcbcrModelConversion::YCBCR_2020, }; - let texture_format = match params.pixel_format { - ffmpeg::format::Pixel::YUV420P => vk::Format::G8_B8_R8_3PLANE_420_UNORM, - _ => { - return Err(anyhow!( - "unsupported pixel format: {:?}", - params.pixel_format - )) - } + let ycbcr_range = if params.color_full_range { + vk::SamplerYcbcrRange::ITU_FULL + } else { + vk::SamplerYcbcrRange::ITU_NARROW }; let create_info = vk::SamplerYcbcrConversionCreateInfo::builder() - .format(texture_format) + .format(format) .ycbcr_model(ycbcr_model) .ycbcr_range(ycbcr_range) .chroma_filter(vk::Filter::LINEAR) diff --git a/mm-protocol/src/messages.proto b/mm-protocol/src/messages.proto index 0a4786f..ad9b10d 100644 --- a/mm-protocol/src/messages.proto +++ b/mm-protocol/src/messages.proto @@ -135,13 +135,13 @@ message PixelScale { // launch a session. message VirtualDisplayParameters { Size resolution = 1; // Required. - uint32 framerate_hz = 3; // Required. - PixelScale ui_scale = 2; // Required. + uint32 framerate_hz = 2; // Required. + PixelScale ui_scale = 3; // Required. } // ### Attachment type // -// This enum refers to the manner of attachment. +// This refers to the manner of attachment. enum AttachmentType { ATTACHMENT_TYPE_UNKNOWN = 0; ATTACHMENT_TYPE_OPERATOR = 1; @@ -150,7 +150,7 @@ enum AttachmentType { // ### Video codec // -// This enum refers to the codec used for a video stream. +// This refers to the codec used for a video stream. enum VideoCodec { VIDEO_CODEC_UNKNOWN = 0; VIDEO_CODEC_H264 = 1; @@ -158,9 +158,19 @@ enum VideoCodec { VIDEO_CODEC_AV1 = 3; } +// ### Video profile +// +// This refers to the profile used for a video stream. Profiles are fully +// defined in the output section, below. +enum VideoProfile { + VIDEO_PROFILE_UNKNOWN = 0; + VIDEO_PROFILE_HD = 1; + VIDEO_PROFILE_HDR10 = 2; +} + // ### Audio codec // -// This enum refers to the codec used for an audio stream. +// This refers to the codec used for an audio stream. enum AudioCodec { AUDIO_CODEC_UNKNOWN = 0; AUDIO_CODEC_OPUS = 1; @@ -448,6 +458,7 @@ message Attach { VideoCodec video_codec = 10; Size streaming_resolution = 11; + VideoProfile video_profile = 12; AudioCodec audio_codec = 15; AudioChannels channels = 16; @@ -466,8 +477,9 @@ message Attached { uint64 session_id = 1; // Required. uint64 attachment_id = 2; // Required. - VideoCodec video_codec = 10; // Required. - Size streaming_resolution = 11; // Required. + VideoCodec video_codec = 10; // Required. + Size streaming_resolution = 11; // Required. + VideoProfile video_profile = 12; // Required. AudioCodec audio_codec = 15; // Required. AudioChannels channels = 16; // Required. @@ -554,15 +566,29 @@ message Detach {} // // - The server must tag the video bitstream with resolution, framerate, and // YCbCr color space/range using whatever mechanism is supported by the codec -// (for example, PPS frames in H.264). Clients should use this information to -// verify that the parameters match the requested attachment parameters. +// (for example, PPS/VUI frames in H.264). Clients should use this +// information to verify that the parameters match the requested attachment +// parameters. // - The server must use YCbCr 4:2:0 chroma subsampling for the compressed // stream (this is sometimes called YUV420P, and is the default for most // implementations of H264, H265, and AV1). +// - For VIDEO_PROFILE_HD, a bit depth of 8, along with the Rec.709 color space +// and limited range must be used. For H.264, H.265, and AV1, this +// corresponds to `colour_primaries`, `transfer_characteristics`, and +// `matrix_coeffs` all equal to 1, and the `video_full_range_flag` set to 0 +// (named `color_range` for AV1). +// - For VIDEO_PROFILE_HDR10, a bit depth of 10, along with the Rec. 2100 color +// space and limited range must be used. For H.264, H.265, and AV1, this +// corresponds to `colour_primaries` and `matrix_coeffs` equal to 9, +// `transfer_characteristics` equal to 16, and the `video_full_range_flag` +// set to 0 (named `color_range` for AV1). The server should additionally use +// SEI headers (or metadata OBUs for AV1) to communicate HDR metadata such as +// mastering display color volume (MDCV) and content light level (CLL) +// information. // - The server may reuse an existing compression context for a new attachment, // but in this case the stream must be resumable by the client within a -// reasonable time frame. For H.265, for example, this means sending PPS -// frames with every keyframe, and keyframes every few seconds. +// reasonable time frame. For H.265, for example, this means sending headers +// with every keyframe, and keyframes every few seconds. // // ### Audio compression // diff --git a/mm-server/Cargo.toml b/mm-server/Cargo.toml index 5c2f738..0315c13 100644 --- a/mm-server/Cargo.toml +++ b/mm-server/Cargo.toml @@ -72,6 +72,7 @@ serde_json = "1.0.114" cursor-icon = "1.1.0" image = { version = "0.25.1", default-features = false, features = ["png"] } git-version = "0.3.9" +wayland-scanner = "0.31.1" [dependencies.pulseaudio] git = "https://github.com/colinmarc/pulseaudio-rs" @@ -79,7 +80,7 @@ rev = "70ddb748f20ceecc20e963e571188124aeb30186" [dependencies.svt] git = "https://github.com/colinmarc/svt-rs" -rev = "ab9dbd872d8f01c2cb96d3f9fd880b26d37d1f0e" +rev = "747915fa4b2f7d0bd2c70ef5af108c75031efc53" optional = true features = ["av1", "hevc"] diff --git a/mm-server/build.rs b/mm-server/build.rs index 418436c..10278f5 100644 --- a/mm-server/build.rs +++ b/mm-server/build.rs @@ -71,6 +71,7 @@ fn compile_shader<'a>( let mut compile_request = session.create_compile_request(); compile_request + .add_search_path("../shader-common") .set_codegen_target(slang::CompileTarget::Spirv) .set_optimization_level(slang::OptimizationLevel::Maximal) .set_target_profile(session.find_profile("glsl_460")); diff --git a/mm-server/src/color.rs b/mm-server/src/color.rs new file mode 100644 index 0000000..057579c --- /dev/null +++ b/mm-server/src/color.rs @@ -0,0 +1,73 @@ +use mm_protocol as protocol; + +/// A combination of color primaries, white point, and transfer function. We +/// generally ignore white point, since we deal only with colorspaces using the +/// D65 white point. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ColorSpace { + /// Uses BT.709 primaries and the sRGB transfer function. + Srgb, + /// Uses BT.709 primaries and a linear transfer function. Usually encoded as + /// a float with negative values and values above 1.0 used to represent the + /// extended space. + LinearExtSrgb, + /// Uses BT.2020 primaries and the ST2084 (PQ) transfer function. + Hdr10, +} + +impl ColorSpace { + pub fn from_primaries_and_tf( + primaries: Primaries, + transfer_function: TransferFunction, + ) -> Option { + match (primaries, transfer_function) { + (Primaries::Srgb, TransferFunction::Srgb) => Some(ColorSpace::Srgb), + (Primaries::Srgb, TransferFunction::Linear) => Some(ColorSpace::LinearExtSrgb), + (Primaries::Bt2020, TransferFunction::Pq) => Some(ColorSpace::Hdr10), + _ => None, + } + } +} + +// A configuration for a compressed video bitstream. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum VideoProfile { + // Uses a bit depth of 8, BT.709 primaries and transfer function. + Hd, + // Uses a bit depth of 10, BT.2020 primaries and the ST2084 (PQ) transfer function. + Hdr10, +} + +impl TryFrom for VideoProfile { + type Error = String; + + fn try_from(profile: protocol::VideoProfile) -> Result { + match profile { + protocol::VideoProfile::Hd => Ok(VideoProfile::Hd), + protocol::VideoProfile::Hdr10 => Ok(VideoProfile::Hdr10), + _ => Err("invalid video profile".into()), + } + } +} + +impl From for protocol::VideoProfile { + fn from(profile: VideoProfile) -> Self { + match profile { + VideoProfile::Hd => protocol::VideoProfile::Hd, + VideoProfile::Hdr10 => protocol::VideoProfile::Hdr10, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum TransferFunction { + Linear, + Srgb, + Pq, +} + +#[derive(Debug, Clone, Copy)] +pub enum Primaries { + Srgb, + Bt2020, +} diff --git a/mm-server/src/compositor.rs b/mm-server/src/compositor.rs index 04870b5..af6b0d3 100644 --- a/mm-server/src/compositor.rs +++ b/mm-server/src/compositor.rs @@ -122,6 +122,7 @@ pub struct State { compositor_state: smithay::wayland::compositor::CompositorState, dmabuf_state: smithay::wayland::dmabuf::DmabufState, _dmabuf_global: smithay::wayland::dmabuf::DmabufGlobal, + _color_global: handlers::color_management::ColorManagementGlobal, // output_manager_state: smithay::wayland::output::OutputManagerState, xdg_shell_state: smithay::wayland::shell::xdg::XdgShellState, shm_state: smithay::wayland::shm::ShmState, @@ -196,6 +197,8 @@ impl Compositor { let dmabuf_global = dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback); + let color_global = handlers::color_management::ColorManagementGlobal::new(&dh); + let xwayland_shell_state = smithay::wayland::xwayland_shell::XWaylandShellState::new::(&dh); @@ -237,6 +240,7 @@ impl Compositor { compositor_state, dmabuf_state, _dmabuf_global: dmabuf_global, + _color_global: color_global, // output_manager_state, xdg_shell_state, shm_state, diff --git a/mm-server/src/compositor/control.rs b/mm-server/src/compositor/control.rs index 7e34712..3f6162e 100644 --- a/mm-server/src/compositor/control.rs +++ b/mm-server/src/compositor/control.rs @@ -6,6 +6,7 @@ use crossbeam_channel::Sender; use crate::{ codec::{AudioCodec, VideoCodec}, + color::VideoProfile, pixel_scale::PixelScale, }; @@ -22,6 +23,7 @@ pub struct VideoStreamParams { pub width: u32, pub height: u32, pub codec: VideoCodec, + pub profile: VideoProfile, } #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/mm-server/src/compositor/handlers.rs b/mm-server/src/compositor/handlers.rs index 97450c4..fa4a42d 100644 --- a/mm-server/src/compositor/handlers.rs +++ b/mm-server/src/compositor/handlers.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BUSL-1.1 mod buffers; +pub mod color_management; mod input; mod x11; mod xdg_shell; @@ -53,7 +54,7 @@ impl compositor::CompositorHandler for State { removed_surfaces.push(surf.clone()); } Some(compositor::BufferAssignment::NewBuffer(buffer)) => { - buffers::buffer_commit(self, surf, &buffer).unwrap(); + buffers::buffer_commit(self, surf, data, &buffer).unwrap(); } None => (), }; diff --git a/mm-server/src/compositor/handlers/buffers.rs b/mm-server/src/compositor/handlers/buffers.rs index afa9dbf..8c285fb 100644 --- a/mm-server/src/compositor/handlers/buffers.rs +++ b/mm-server/src/compositor/handlers/buffers.rs @@ -7,10 +7,14 @@ use smithay::{ protocol::{wl_buffer, wl_surface}, Resource, }, - wayland::{buffer, dmabuf, shm}, + wayland::{buffer, compositor, dmabuf, shm}, }; use tracing::{debug, error, trace}; +use crate::{ + color::ColorSpace, compositor::handlers::color_management::ColorManagementCachedState, +}; + use super::State; impl buffer::BufferHandler for State { @@ -60,6 +64,7 @@ impl dmabuf::DmabufHandler for State { pub fn buffer_commit( state: &mut State, surface: &wl_surface::WlSurface, + surface_data: &compositor::SurfaceData, buffer: &wl_buffer::WlBuffer, ) -> anyhow::Result<()> { trace!( @@ -78,9 +83,15 @@ pub fn buffer_commit( match dmabuf::get_dmabuf(buffer) { Ok(dmabuf) => { + let colorspace = surface_data + .cached_state + .current::() + .colorspace + .unwrap_or(ColorSpace::Srgb); + return state .texture_manager - .attach_dma_buffer(surface, buffer, dmabuf); + .attach_dma_buffer(surface, buffer, dmabuf, colorspace); } Err(smithay::utils::UnmanagedResource) => (), // Fall through to shm handler. } diff --git a/mm-server/src/compositor/handlers/color_management.rs b/mm-server/src/compositor/handlers/color_management.rs new file mode 100644 index 0000000..0856cfa --- /dev/null +++ b/mm-server/src/compositor/handlers/color_management.rs @@ -0,0 +1,364 @@ +// Copyright 2024 Colin Marc +// +// SPDX-License-Identifier: BUSL-1.1 + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] + +use std::sync::{Arc, Mutex}; + +use smithay::{ + reexports::wayland_server::{ + backend::GlobalId, protocol::wl_surface, Client, DataInit, Dispatch, DisplayHandle, + GlobalDispatch, New, Resource, WEnum, + }, + wayland::compositor, +}; + +use crate::color::{ColorSpace, Primaries, TransferFunction}; + +use self::protocol::{ + mesa_color_management_surface_v1::MesaColorManagementSurfaceV1, + mesa_color_manager_v1::{self, Feature, MesaColorManagerV1, RenderIntent}, + mesa_image_description_creator_params_v1::MesaImageDescriptionCreatorParamsV1, + mesa_image_description_v1::{Cause, MesaImageDescriptionV1}, +}; + +impl From for mesa_color_manager_v1::TransferFunction { + fn from(tf: TransferFunction) -> Self { + match tf { + TransferFunction::Linear => mesa_color_manager_v1::TransferFunction::Linear, + TransferFunction::Srgb => mesa_color_manager_v1::TransferFunction::Srgb, + TransferFunction::Pq => mesa_color_manager_v1::TransferFunction::St2084Pq, + } + } +} + +impl TryFrom for TransferFunction { + type Error = (); + + fn try_from(tf: mesa_color_manager_v1::TransferFunction) -> Result { + match tf { + mesa_color_manager_v1::TransferFunction::Linear => Ok(TransferFunction::Linear), + mesa_color_manager_v1::TransferFunction::Srgb => Ok(TransferFunction::Srgb), + mesa_color_manager_v1::TransferFunction::St2084Pq => Ok(TransferFunction::Pq), + _ => Err(()), + } + } +} + +impl From for mesa_color_manager_v1::Primaries { + fn from(p: Primaries) -> Self { + match p { + Primaries::Srgb => mesa_color_manager_v1::Primaries::Srgb, + Primaries::Bt2020 => mesa_color_manager_v1::Primaries::Bt2020, + } + } +} + +impl TryFrom for Primaries { + type Error = (); + + fn try_from(p: mesa_color_manager_v1::Primaries) -> Result { + match p { + mesa_color_manager_v1::Primaries::Srgb => Ok(Primaries::Srgb), + mesa_color_manager_v1::Primaries::Bt2020 => Ok(Primaries::Bt2020), + _ => Err(()), + } + } +} + +use super::State; + +mod protocol { + use smithay::reexports::wayland_server; + use smithay::reexports::wayland_server::protocol::*; + + mod __interfaces { + use smithay::reexports::wayland_server; + use wayland_server::backend as wayland_backend; + use wayland_server::protocol::__interfaces::*; + wayland_scanner::generate_interfaces!( + "src/compositor/protocol/mesa-color-management-v1.xml" + ); + } + + use __interfaces::*; + wayland_scanner::generate_server_code!("src/compositor/protocol/mesa-color-management-v1.xml"); +} + +const SUPPORTED_TFS: [mesa_color_manager_v1::TransferFunction; 3] = [ + mesa_color_manager_v1::TransferFunction::Linear, + mesa_color_manager_v1::TransferFunction::Srgb, + mesa_color_manager_v1::TransferFunction::St2084Pq, +]; + +const SUPPORTED_PRIMARIES: [mesa_color_manager_v1::Primaries; 2] = [ + mesa_color_manager_v1::Primaries::Srgb, + mesa_color_manager_v1::Primaries::Bt2020, +]; + +pub struct ColorManagementGlobal { + _global: GlobalId, +} + +impl ColorManagementGlobal { + pub fn new(display: &DisplayHandle) -> Self { + let global = display.create_global::(1, ()); + Self { _global: global } + } +} + +struct ColorManagementSurfaceUserData { + wl_surface: wl_surface::WlSurface, +} + +/// Represents a pending image description for a color-managed surface. +#[derive(Debug, Default, Clone)] +pub struct ColorManagementCachedState { + pub colorspace: Option, +} + +impl compositor::Cacheable for ColorManagementCachedState { + fn commit(&mut self, _dh: &DisplayHandle) -> Self { + self.clone() + } + + fn merge_into(self, into: &mut Self, _dh: &DisplayHandle) { + *into = self; + } +} + +#[derive(Debug, Default)] +struct ImageDescParams { + primaries: Option, + transfer_function: Option, +} + +impl GlobalDispatch for State { + fn bind( + _state: &mut State, + _handle: &DisplayHandle, + _client: &Client, + resource: New, + _global_data: &(), + data_init: &mut DataInit<'_, State>, + ) { + let color_manager = data_init.init(resource, ()); + + color_manager.supported_intent(RenderIntent::Perceptual); + color_manager.supported_feature(Feature::Parametric); + + for tf in SUPPORTED_TFS.iter().copied() { + color_manager.supported_tf_named(tf); + } + + for primaries in SUPPORTED_PRIMARIES.iter().copied() { + color_manager.supported_primaries_named(primaries); + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + _resource: &MesaColorManagerV1, + request: ::Request, + _data: &(), + _dhandle: &DisplayHandle, + data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_color_manager_v1::Request::GetOutput { .. } => { + todo!() + } + protocol::mesa_color_manager_v1::Request::GetSurface { id, surface } => { + data_init.init( + id, + ColorManagementSurfaceUserData { + wl_surface: surface, + }, + ); + } + protocol::mesa_color_manager_v1::Request::NewIccCreator { obj } => { + data_init.post_error( + obj, + protocol::mesa_color_manager_v1::Error::UnsupportedFeature, + "ICC profiles are not supported.", + ); + } + protocol::mesa_color_manager_v1::Request::NewParametricCreator { obj } => { + data_init.init(obj, Arc::new(Mutex::new(Default::default()))); + } + protocol::mesa_color_manager_v1::Request::Destroy => (), + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaColorManagementSurfaceV1, + request: ::Request, + data: &ColorManagementSurfaceUserData, + _dhandle: &DisplayHandle, + _data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_color_management_surface_v1::Request::SetImageDescription { + image_description, + render_intent, + } => { + if render_intent != WEnum::Value(RenderIntent::Perceptual) { + resource.post_error( + protocol::mesa_color_management_surface_v1::Error::RenderIntent as u32, + "Only perceptual render intent is supported", + ); + + return; + } + + if let Some(colorspace) = image_description.data::() { + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = Some(*colorspace); + }); + } + } + protocol::mesa_color_management_surface_v1::Request::UnsetImageDescription => { + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = None; + }); + } + protocol::mesa_color_management_surface_v1::Request::GetPreferred { .. } => todo!(), + protocol::mesa_color_management_surface_v1::Request::Destroy => { + // XXX: Why does the protocol allow multiple? + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = None; + }); + } + } + } +} + +impl Dispatch>> for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaImageDescriptionCreatorParamsV1, + request: ::Request, + data: &Arc>, + _dhandle: &DisplayHandle, + data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_image_description_creator_params_v1::Request::SetTfNamed { tf } => { + match tf { + WEnum::Value(tf) if SUPPORTED_TFS.contains(&tf) => { + data.lock().unwrap().transfer_function = Some(tf); + } + _ => { + resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidTf as u32, + "Unsupported transfer function", + ); + } + } + } + protocol::mesa_image_description_creator_params_v1::Request::SetPrimariesNamed { primaries } => { + match primaries { + WEnum::Value(p) if SUPPORTED_PRIMARIES.contains(&p) => { + data.lock().unwrap().primaries = Some(p); + }, + _ => { + resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidPrimaries as u32, + "Unsupported primaries", + ); + } + } + } + protocol::mesa_image_description_creator_params_v1::Request::Create { image_description } => { + let params = data.lock().unwrap(); + if params.transfer_function.is_none() || params.primaries.is_none() { + data_init.post_error( + image_description, + protocol::mesa_image_description_creator_params_v1::Error::IncompleteSet as u32, + "Primaries and transfer function must be set", + ); + + return; + } + + let colorspace = match (params.primaries.unwrap().try_into(), params.transfer_function.unwrap().try_into()) { + (Ok(primaries), Ok(transfer_function)) => ColorSpace::from_primaries_and_tf(primaries, transfer_function), + _ => None, + }; + + if let Some(colorspace) = colorspace { + let image_description = data_init.init(image_description, colorspace); + image_description.ready(colorspace as u32); + } else { + // We init and then immediately fail. + let image_description = data_init.init(image_description, ColorSpace::Srgb); + image_description.failed(Cause::Unsupported, "Unsupported combination of transfer function and primaries".to_string()); + } + } + protocol::mesa_image_description_creator_params_v1::Request::SetTfPower { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidTf as u32, + "set_tf_power not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetPrimaries { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidPrimaries as u32, + "set_primaries not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMasteringDisplayPrimaries {.. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidMastering as u32, + "set_mastering_display_primaries not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMasteringLuminance { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidLuminance as u32, + "set_mastering_luminance not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMaxCll { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InconsistentSet as u32, + "set_max_cll not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMaxFall { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InconsistentSet as u32, + "set_max_fall not supported", + ), + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaImageDescriptionV1, + request: ::Request, + _data: &ColorSpace, + _dhandle: &DisplayHandle, + _data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_image_description_v1::Request::GetInformation { .. } => resource + .post_error( + protocol::mesa_image_description_v1::Error::NoInformation as u32, + "No information available", + ), + protocol::mesa_image_description_v1::Request::Destroy => (), + } + } +} diff --git a/mm-server/src/compositor/protocol/mesa-color-management-v1.xml b/mm-server/src/compositor/protocol/mesa-color-management-v1.xml new file mode 100644 index 0000000..b04a43d --- /dev/null +++ b/mm-server/src/compositor/protocol/mesa-color-management-v1.xml @@ -0,0 +1,1233 @@ + + + + Copyright 2019 Sebastian Wick + Copyright 2019 Erwin Burema + Copyright 2020 AMD + Copyright 2020, 2022, 2023 Collabora, Ltd. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice (including the next + paragraph) shall be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + The aim of the color management extension is to allow clients to know + the color properties of outputs, and to tell the compositor about the color + properties of their content on surfaces. Doing this enables a compositor + to perform automatic color management of content for different outputs + according to how content is intended to look like. + + The color properties are represented as an image description object which + is immutable after it has been created. A wl_output always has an + associated image description that clients can observe. A wl_surface + always has an associated preferred image description as a hint chosen by + the compositor that clients can also observe. Clients can set an image + description on a wl_surface to denote the color characteristics of the + surface contents. + + An image description includes SDR and HDR colorimetry and encoding, HDR + metadata, and viewing environment parameters. An image description does + not include the properties set through color-representation extension. + It is expected that the color-representation extension is used in + conjunction with the color management extension when necessary, + particularly with the YUV family of pixel formats. + + Recommendation ITU-T H.273 + "Coding-independent code points for video signal type identification" + shall be referred to as simply H.273 here. + + The color-and-hdr repository + (https://gitlab.freedesktop.org/pq/color-and-hdr) contains + background information on the protocol design and legacy color management. + It also contains a glossary, learning resources for digital color, tools, + samples and more. + + The terminology used in this protocol is based on common color science and + color encoding terminology where possible. The glossary in the color-and-hdr + repository shall be the authority on the definition of terms in this + protocol. + + + + + A global interface used for getting color management extensions for + wl_surface and wl_output objects, and for creating client defined image + description objects. The extension interfaces allow + getting the image description of outputs and setting the image + description of surfaces. + + + + + Destroy the mesa_color_manager_v1 object. This does not affect any other + objects in any way. + + + + + + + + + + See the ICC.1:2022 specification from the International Color Consortium + for more details about rendering intents. + + The principles of ICC defined rendering intents apply with all types + of image descriptions, not only those with ICC file profiles. + + Compositors must support the perceptual rendering intent. Other + rendering intents are optional. + + + + + + + + + + + + + + + + + + + The compositor supports set_mastering_display_primaries request + with a target color volume fully contained inside the primary + color volume. + + + + + The compositor additionally supports target color volumes that + extend outside of the primary color volume. + + This can only be advertised if feature set_mastering_display_primaries + is supported as well. + + + + + + + Named color primaries used to encode well-known sets of primaries. + H.273 is the authority, when it comes to the exact values of primaries and authoritative specifications, + where an equivalent code point exists. + Descriptions do list the specifications for convenience. + + + + Color primaries as defined by + - Rec. ITU-R BT.709-6 + - Rec. ITU-R BT.1361-0 conventional colour gamut system and extended colour gamut system (historical) + - IEC 61966-2-1 sRGB or sYCC + - IEC 61966-2-4 + - Society of Motion Picture and Television Engineers (SMPTE) RP 177 (1993) Annex B + Equivalent to H.273 ColourPrimaries code point 1. + + + + + Color primaries as defined by + - Rec. ITU-R BT.470-6 System M (historical) + - United States National Television System Committee 1953 Recommendation for transmission standards for color television + - United States Federal Communications Commission (2003) Title 47 Code of Federal Regulations 73.682 (a)(20) + Equivalent to H.273 ColourPrimaries code point 4. + + + + + Color primaries as defined by + - Rec. ITU-R BT.470-6 System B, G (historical) + - Rec. ITU-R BT.601-7 625 + - Rec. ITU-R BT.1358-0 625 (historical) + - Rec. ITU-R BT.1700-0 625 PAL and 625 SECAM + Equivalent to H.273 ColourPrimaries code point 5. + + + + + Color primaries as defined by + - Rec. ITU-R BT.601-7 525 + - Rec. ITU-R BT.1358-1 525 or 625 (historical) + - Rec. ITU-R BT.1700-0 NTSC + - SMPTE 170M (2004) + - SMPTE 240M (1999) (historical) + Equivalent to H.273 ColourPrimaries code point 6 and 7. + + + + + Color primaries as defined by H.273 for generic film. + Equivalent to H.273 ColourPrimaries code point 8. + + + + + Color primaries as defined by + - Rec. ITU-R BT.2020-2 + - Rec. ITU-R BT.2100-0 + Equivalent to H.273 ColourPrimaries code point 9. + + + + + Color primaries as defined as the maximum of the CIE 1931 XYZ color space by + - SMPTE ST 428-1 + - (CIE 1931 XYZ as in ISO 11664-1) + Equivalent to H.273 ColourPrimaries code point 10. + + + + + Color primaries as defined by Digital Cinema System and published in SMPTE RP 431-2 (2011). + Equivalent to H.273 ColourPrimaries code point 11. + + + + + Color primaries as defined by Digital Cinema System and published in SMPTE EG 432-1 (2010). + Equivalent to H.273 ColourPrimaries code point 12. + + + + + Color primaries as defined by Adobe as "Adobe RGB" and later published by ISO 12640-4 (2011). + + + + + + + Named Transfer Functions. + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.709-6 + - Rec. ITU-R BT.1361-0 conventional colour gamut system (historical) + Equivalent to H.273 TransferCharacteristics code point 1, 6, 14, 15. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.470-6 System M (historical) + - United States National Television System Committee 1953 Recommendation for transmission standards for color television + - United States Federal Communications Commission (2003) Title 47 Code of Federal Regulations 73.682 (a) (20) + - Rec. ITU-R BT.1700-0 625 PAL and 625 SECAM + Equivalent to H.273 TransferCharacteristics code point 4. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.470-6 System B, G (historical) + Equivalent to H.273 TransferCharacteristics code point 5. + + + + + Transfer characteristics as defined by + - SMPTE ST 240 (1999) + Equivalent to H.273 TransferCharacteristics code point 7. + + + + + Linear transfer characteristics. + Equivalent to H.273 TransferCharacteristics code point 8. + + + + + Logarithmic transfer characteristic (100:1 range). + Equivalent to H.273 TransferCharacteristics code point 9. + + + + + Logarithmic transfer characteristic (100 * Sqrt(10) : 1 range). + Equivalent to H.273 TransferCharacteristics code point 10. + + + + + Transfer characteristics as defined by + - IEC 61966-2-4 + Equivalent to H.273 TransferCharacteristics code point 11. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.1361-0 extended colour gamut system (historical) + Equivalent to H.273 TransferCharacteristics code point 12. + + + + + Transfer characteristics as defined by + - IEC 61966-2-1 sRGB + Equivalent to H.273 TransferCharacteristics code point 13 with MatrixCoefficients set to 0. + + + + + Transfer characteristics as defined by + - IEC 61966-2-1 sYCC + Equivalent to H.273 TransferCharacteristics code point 13 with MatrixCoefficients set to anything but 0. + + + + + Transfer characteristics as defined by + - SMPTE ST 2084 (2014) for 10-, 12-, 14- and 16-bit systems + - Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system + Equivalent to H.273 TransferCharacteristics code point 16. + + + + + Transfer characteristics as defined by + - SMPTE ST 428-1 (2019) + Equivalent to H.273 TransferCharacteristics code point 17. + + + + + Transfer characteristics as defined by + - ARIB STD-B67 (2015) + - Rec. ITU-R BT.2100-2 hybrid log-gamma (HLG) system + Equivalent to H.273 TransferCharacteristics code point 18. + + + + + + + This creates a new mesa_color_management_output_v1 object for the + given wl_output. + + See the mesa_color_management_output_v1 interface for more details. + + + + + + + + + This creates a new color mesa_color_management_surface_v1 object for the + given wl_surface. + + See the mesa_color_management_surface_v1 interface for more details. + + + + + + + + + Makes a new ICC-based image description creator object with all + properties initially unset. The client can then use the object's + interface to define all the required properties for an image description + and finally create a mesa_image_description_v1 object. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.icc_v1_v4. + Otherwise this request raises the protocol error unsupported_feature. + + + + + + + + Makes a new parametric image description creator object with all + properties initially unset. The client can then use the object's + interface to define all the required properties for an image description + and finally create a mesa_image_description_v1 object. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.parametric. + Otherwise this request raises the protocol error unsupported_feature. + + + + + + + + When this object is created, it shall immediately send this event + once for each rendering intent the compositor supports. + + + + + + + + When this object is created, it shall immediately send this event + once for each compositor supported feature listed in the enumeration. + + + + + + + + When this object is created, it shall immediately send this event + once for each named transfer function the compositor + supports with the parametric image description creator. + + + + + + + + When this object is created, it shall immediately send this event + once for each named set of primaries the compositor + supports with the parametric image description creator. + + + + + + + + + A mesa_color_management_output_v1 describes the color properties of an + output. + + The mesa_color_management_output_v1 is associated with the wl_output global + underlying the wl_output object. Therefore the client destroying the + wl_output object has no impact, but the compositor removing the output + global makes the mesa_color_management_output_v1 object inert. + + + + + Destroy the color mesa_color_management_output_v1 object. This does not + affect any remaining protocol objects. + + + + + + This event is sent whenever the image description of the + output changed, followed by one wl_output.done event common to + output events across all extensions. + + If the client wants to use the updated image description, it needs + to do get_image_description again, because image description objects + are immutable. + + + + + + This creates a new mesa_image_description_v1 object for the current image description + of the output. There always is exactly one image description active for an + output so the client should destroy the image description created by earlier + invocations of this request. This request is usually sent as a reaction + to the image_description_changed event or when creating a + mesa_color_management_output_v1 object. + + The created mesa_image_description_v1 object preserves the image description + of the output from the time the object was created. + + The resulting image description object allows get_information request. + + If this protocol object is inert, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the no_output cause. + + If the interface version is inadequate for the output's image + description, meaning that the client does not support all the + events needed to deliver the crucial information, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the low_version cause. + + Otherwise the object shall immediately deliver the ready event. + + + + + + + + + A mesa_color_management_surface_v1 allows the client to set the color + space and HDR properties of a surface. + + If the wl_surface associated with the mesa_color_management_surface_v1 is + destroyed, the mesa_color_management_surface_v1 object becomes inert. + + + + + Destroy the mesa_color_management_surface_v1 object. + + When the last mesa_color_management_surface_v1 object for a wl_surface + is destroyed, it does the same as unset_image_description. + + + + + + + + + + + + + Set the image description of the underlying surface. The image + description and rendering intent are double-buffered state, see + wl_surface.commit. + + It is the client's responsibility to understand the image description + it sets on a surface, and to provide content that matches that image + description. + + A rendering intent provides the client's preference on how content + colors should be mapped to each output. The render_intent value must + be one advertised by the compositor with + mesa_color_manager_v1.render_intent event, otherwise the protocol error + render_intent is raised. + + By default, a surface does not have an associated image description + nor a rendering intent. The handling of color on such surfaces is + compositor implementation defined. Compositors should handle such + surfaces as sRGB but may handle them differently if they have specific + requirements. + + + + + + + + + This request removes any image description from the surface. See + set_image_description for how a compositor handles a surface without + an image description. This is double-buffered state, see + wl_surface.commit. + + + + + + The preferred image description is the one which likely has the most + performance and/or quality benefits for the compositor if used by the + client for its wl_surface contents. This event is sent whenever the + compositor changes the wl_surface's preferred image description. + + This is not an initial event. + + This event is merely a notification. When the client wants to know + what the preferred image description is, it shall use the get_preferred + request. + + The preferred image description is not automatically used for anything. + It is only a hint, and clients may set any valid image description with + set_image_description but there might be performance and color accuracy + improvements by providing the wl_surface contents in the preferred + image description. Therefore clients that can, should render according + to the preferred image description + + + + + + If this protocol object is inert, the protocol error inert is raised. + + This creates a new mesa_image_description_v1 object for the currently + preferred image description for the wl_surface. The client should + stop using and destroy the image descriptions created by earlier + invocations of this request for the associated wl_surface. + This request is usually sent as a reaction to the preferred_changed + event or when creating a mesa_color_management_surface_v1 object if + the client is capable of adapting to image descriptions. + + The created mesa_image_description_v1 object preserves the preferred image + description of the wl_surface from the time the object was created. + + The resulting image description object allows get_information request. + + If the interface version is inadequate for the preferred image + description, meaning that the client does not support all the + events needed to deliver the crucial information, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the low_version cause, + otherwise the object shall immediately deliver the ready event. + + + + + + + + + This type of object is used for collecting all the information required + to create a mesa_image_description_v1 object from an ICC file. A complete + set of required parameters consists of these properties: + - ICC file + + Each required property must be set exactly once if the client is to create + an image description. The set requests verify that a property was not + already set. The create request verifies that all required properties are + set. There may be several alternative requests for setting each property, + and in that case the client must choose one of them. + + Once all properties have been set, the create request must be used to + create the image description object, destroying the creator in the + process. + + + + + + + + + + + + + + + Create an image description object based on the ICC information + previously set on this object. A compositor must parse the ICC data in + some undefined but finite amount of time. + + The completeness of the parameter set is verified. If the set is not + complete, the protocol error incomplete_set is raised. For the + definition of a complete set, see the description of this interface. + + If the particular combination of the information is not supported + by the compositor, the resulting image description object shall + immediately deliver the mesa_image_description_v1.failed event with the + 'unsupported' cause. If a valid image description was created from the + information, the mesa_image_description_v1.ready event will eventually + be sent instead. + + This request destroys the mesa_image_description_creator_icc_v1 object. + + The resulting image description object does not allow get_information + request. + + + + + + + + Sets the ICC profile file to be used as the basis of the image + description. + + The data shall be found through the given fd at the given offset, + having the given length. The fd must seekable and readable. Violating + these requirements raises the bad_fd protocol error. + + If reading the data fails due to an error independent of the client, the + compositor shall send the mesa_image_description_v1.failed event on the + created mesa_image_description_v1 with the 'operating_system' cause. + + The maximum size of the ICC profile is 4 MB. If length is greater + than that or zero, the protocol error bad_size is raised. + If offset + length exceeds the file size, the protocol error + out_of_file is raised. + + A compositor may read the file at any time starting from this request + and only until whichever happens first: + - If create request was issued, the mesa_image_description_v1 object + delivers either failed or ready event; or + - if create request was not issued, this + mesa_image_description_creator_icc_v1 object is destroyed. + + A compositor shall not modify the contents of the file, and the fd may + be sealed for writes and size changes. The client must ensure to its + best ability that the data does not change while the compositor is + reading it. + + The data must represent a valid ICC profile. + The ICC profile version must be 2 or 4, it must be a 3 channel profile + and the class must be 'display'. + Violating these requirements will not result in a protocol error but + will eventually send the mesa_image_description_v1.failed event on the + created mesa_image_description_v1 with the 'unsupported' cause. + + See the International Color Consortium specification ICC.1:2022 for more + details about ICC profiles. + + If ICC file has already been set on this object, the protocol error + already_set is raised. + + + + + + + + + + + This type of object is used for collecting all the parameters required + to create a mesa_image_description_v1 object. A complete set of required + parameters consists of these properties: + - transfer characteristic function (tf) + - chromaticities of primaries and white point (primary color volume) + + The following properties are optional and have a well-defined default + if not explicitly set: + - mastering display primaries and white point (target color volume) + - mastering luminance range + - maximum content light level + - maximum frame-average light level + + Each required property must be set exactly once if the client is to create + an image description. The set requests verify that a property was not + already set. The create request verifies that all required properties are + set. There may be several alternative requests for setting each property, + and in that case the client must choose one of them. + + Once all properties have been set, the create request must be used to + create the image description object, destroying the creator in the + process. + + + + + + + + + + + + + + + + + Create an image description object based on the parameters previously + set on this object. + + The completeness of the parameter set is verified. If the set is not + complete, the protocol error incomplete_set is raised. For the + definition of a complete set, see the description of this interface. + + If the particular combination of the parameter set is not supported + by the compositor, the resulting image description object shall + immediately deliver the mesa_image_description_v1.failed event with the + 'unsupported' cause. If a valid image description was created from the + parameter set, the mesa_image_description_v1.ready event will eventually + be sent instead. + + This request destroys the mesa_image_description_creator_params_v1 + object. + + The resulting image description object does not allow get_information + request. + + + + + + + + Sets the transfer characteristic using explicitly enumerated named functions. + + Only names advertised with mesa_color_manager_v1 + event supported_tf_named are allowed. Other values shall raise the + protocol error invalid_tf. + + If transfer characteristic has already been set on this object, the + protocol error already_set is raised. + + + + + + + + Sets the color component transfer characteristic to a power curve + with the given exponent. This curve represents the conversion from + electrical to optical pixel or color values. + + The curve exponent shall be multiplied by 10000 to get the argument + eexp value to carry the precision of 4 decimals. + + The curve exponent must be at least 1.0 and at most 10.0. Otherwise + the protocol error invalid_tf is raised. + + If transfer characteristic has already been set on this object, the + protocol error already_set is raised. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.set_tf_power. Otherwise this request raises + the protocol error invalid_tf. + + + + + + + + Sets the color primaries and white point using explicitly named sets. + This describes the primary color volume which is the basis + for color value encoding. + + Only names advertised with mesa_color_manager_v1 + event supported_primaries_named are allowed. Other values shall raise the + protocol error invalid_primaries. + + If primaries have already been set on this object, the protocol error + already_set is raised. + + + + + + + + Sets the color primaries and white point using CIE 1931 xy + chromaticity coordinates. This describes the primary color volume + which is the basis for color value encoding. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + If primaries have already been set on this object, the protocol error + already_set is raised. + + This request can be used if the compositor advertises + mesa_color_manager_v1.feature.set_primaries. Otherwise this request + raises the protocol error invalid_primaries. + + + + + + + + + + + + + + + Provides the color primaries and white point of the mastering display + using CIE 1931 xy chromaticity coordinates. This is compatible with the + SMPTE ST 2086 definition of HDR static metadata. + + The mastering display primaries define the target color volume. + + If mastering display primaries are not explicitly set, the target + color volume is assumed to be equal to the primary color volume. + + The target color volume is defined by all tristimulus values between 0.0 + and 1.0 (inclusive) of the color space defined by the given mastering + display primaries and white point. The colorimetry is identical between + the container color space and the mastering display color space, + including that no chromatic adaptation is applied even if the white + points differ. + + The target color volume can exceed the primary color volume to allow for + a greater color volume with an existing color space definition (for + example scRGB). It can be smaller than the primary color volume to + minimize gamut and tone mapping distances for big color spaces (HDR + metadata). + + To make use of the entire target color volume a suitable pixel format + has to be chosen (e.g. floating point to exceed the primary color + volume, or abusing limited quantization range as with xvYCC). + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + If mastering display primaries have already been set on this object, + the protocol error already_set is raised. + + This request can be used if the compositor advertises + mesa_color_manager_v1.feature.set_mastering_display_primaries. + Otherwise this request raises the protocol error invalid_mastering. + The advertisement implies support only for target color + volumes fully contained within the primary color volume. + + If a compositor additionally supports target color volume exceeding + the primary color volume, it must advertise + mesa_color_manager_v1.feature.extended_target_volume. + If a client uses target color volume exceeding the primary color volume + and the compositor does not support it, the result is implementation + defined. Compositors are recommended to detect this case and fail the + image description gracefully, but it may as well result in color + artifacts. + + + + + + + + + + + + + + + Sets the luminance range that was used during the content mastering + process as the minimum and maximum absolute luminance L. This is + compatible with the SMPTE ST 2086 definition of HDR static metadata. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + The mastering luminance range is undefined by default. + + If max L is less than or equal to min L, the protocol error + invalid_luminance is raised. + + Min L value is multiplied by 10000 to get the argument min_lum value + and carry precision of 4 decimals. Max L value is unscaled for max_lum. + + + + + + + + + Sets the maximum content light level (max_cll) as defined by CTA-861-H. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + max_cll is undefined by default. + + + + + + + + Sets the maximum frame-average light level (max_fall) as defined by + CTA-861-H. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + max_fall is undefined by default. + + + + + + + + + An image description carries information about the color encoding used + on a surface when attached to a wl_surface via + mesa_color_management_surface_v1.set_image_description. A compositor can + use this information to decode pixel values into colorimetrically + meaningful quantities. + + Note, that the mesa_image_description_v1 object is not ready to be used + immediately after creation. The object eventually delivers either the + 'ready' or the 'failed' event, specified in all requests creating it. The + object is deemed "ready" after receiving the 'ready' event. + + An object which is not ready is illegal to use, it can only be destroyed. + Any other request in this interface shall result in the 'not_ready' + protocol error. Attempts to use an object which is not ready through other + interfaces shall raise protocol errors defined there. + + Once created and regardless of how it was created, a mesa_image_description_v1 + object always refers to one fixed image description. It cannot change + after creation. + + + + + Destroy this object. It is safe to destroy an object which is not ready. + + Destroying a mesa_image_description_v1 object has no side-effects, not + even if a mesa_color_management_surface_v1.set_image_description has + not yet been followed by a wl_surface.commit. + + + + + + + + + + + + + + + + + + + + + + If creating a mesa_image_description_v1 object fails for a reason that + is not defined as a protocol error, this event is sent. + The requests that create image description objects define whether + and when this can occur. Only such creation requests can trigger this + event. This event cannot be triggered after the image description was + successfully formed. + + Once this event has been sent, the mesa_image_description_v1 object will + never become ready and it can only be destroyed. + + + + + + + + + Once this event has been sent, the mesa_image_description_v1 object is + deemed "ready". Ready objects can be used to send requests and can be + used through other interfaces. + + Every ready mesa_image_description_v1 protocol object refers to an + underlying image description record in the compositor. Multiple protocol + objects may end up referring to the same record. Clients may identify + these "copies" by comparing their id numbers: if the numbers from two + protocol objects are identical, the protocol objects refer to the same + image description record. Two different image description records + cannot have the same id number simultaneously. The id number does not + change during the lifetime of the image description record. + + The id number is valid only as long as the protocol object is alive. + If all protocol objects referring to the same image description record + are destroyed, the id number may be recycled for a different image + description record. + + Image description id number is not a protocol object id. Zero is + reserved as an invalid id number. It shall not be possible for a + client to refer to an image description by its id number in protocol. + The id numbers might not be portable between Wayland connections. + + This identity allows clients to de-duplicate image description records + and avoid get_information request if they already have the image + description information. + + + + + + + + Creates a mesa_image_description_info_v1 object which delivers the + information that makes up the image description. + + Not all image description protocol objects allow get_information + request. Whether it is allowed or not is defined by the request that + created the object. If get_information is not allowed, the protocol + error no_information is raised. + + + + + + + + + Sends all matching events describing an image description object exactly + once and finally sends the 'done' event. + + Once a mesa_image_description_info_v1 object has delivered a 'done' event + it is automatically destroyed. + + Every mesa_image_description_info_v1 created from the same + mesa_image_description_v1 shall always return the exact same data. + + + + + Signals the end of information events and destroys the object. + + + + + + The icc argument provides a file descriptor to the client which may be + memory-mapped to provide the ICC profile matching the image description. + The fd is read-only, and if mapped then it must be mapped with + MAP_PRIVATE by the client. + + The ICC profile version and other details are determined by the + compositor. There is no provision for a client to ask for a specific + kind of a profile. + + + + + + + + + + Delivers the primary color volume primaries and white point + using CIE 1931 xy chromaticity coordinates. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + + + + + + + + + + + + + + Delivers the primary color volume primaries and white point using a + explicitly enumerated named set. + + + + + + + + The color component transfer characteristic of this image description + is a pure power curve. This event provides the exponent of the power + function. This curve represents the conversion from electrical to + optical pixel or color values. + + The curve exponent has been multiplied by 10000 to get the argument + eexp value to carry the precision of 4 decimals. + + + + + + + + Delivers the transfer characteristic using an explicitly enumerated + named function. + + + + + + + + Provides the color primaries and white point of the target + color volume using CIE 1931 xy chromaticity coordinates. This is + compatible with the SMPTE ST 2086 definition of HDR static metadata + for mastering displays. + + While primary color volume is about how color is encoded, the + target color volume is the actually displayable color volume. + If target color volume is equal to the primary color volume, + then this event is not sent. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + + + + + + + + + + + + + + Provides the luminance range that the image description is targeting + as the minimum and maximum absolute luminance L. This is compatible + with the SMPTE ST 2086 definition of HDR static metadata. + + This luminance range is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + Min L value is multiplied by 10000 to get the argument min_lum value + and carry precision of 4 decimals. Max L value is unscaled for max_lum. + + + + + + + + + Provides the targeted max_cll of the image description. max_cll is + defined by CTA-861-H. + + This luminance is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + + + + + + + Provides the targeted max_fall of the image description. max_fall is + defined by CTA-861-H. + + This luminance is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + + + + + diff --git a/mm-server/src/compositor/video.rs b/mm-server/src/compositor/video.rs index d640986..23ae519 100644 --- a/mm-server/src/compositor/video.rs +++ b/mm-server/src/compositor/video.rs @@ -14,17 +14,21 @@ mod textures; mod timebase; mod vulkan_encode; -use cpu_encode::CpuEncoder; - +use anyhow::{anyhow, bail}; use smithay::{ reexports::wayland_server::{protocol::wl_surface, Resource}, wayland::compositor, }; use tracing::{error, instrument, trace, warn}; -use crate::{codec::VideoCodec, vulkan::*}; +use crate::{ + codec::VideoCodec, + color::{ColorSpace, VideoProfile}, + vulkan::*, +}; use super::{AttachedClients, DisplayParams, VideoStreamParams}; +use cpu_encode::CpuEncoder; pub use dmabuf::dmabuf_feedback; use textures::*; @@ -71,6 +75,10 @@ impl Encoder { } } + if params.profile != VideoProfile::Hd { + bail!("HDR requires vulkan encode") + } + Ok(Self::Cpu(CpuEncoder::new( vk, attachments, @@ -124,6 +132,7 @@ pub struct SwapFrame { pub struct EncodePipeline { display_params: DisplayParams, + streaming_params: VideoStreamParams, composite_pipeline: composite::CompositePipeline, convert_pipeline: convert::ConvertPipeline, @@ -142,10 +151,10 @@ impl EncodePipeline { stream_seq: u64, attachments: AttachedClients, display_params: DisplayParams, - stream_params: VideoStreamParams, + streaming_params: VideoStreamParams, ) -> anyhow::Result { - if stream_params.width != display_params.width - || stream_params.height != display_params.height + if streaming_params.width != display_params.width + || streaming_params.height != display_params.height { // Superres is not implemented yet. unimplemented!() @@ -155,7 +164,7 @@ impl EncodePipeline { vk.clone(), attachments.clone(), stream_seq, - stream_params, + streaming_params, display_params.framerate, )?; @@ -172,6 +181,7 @@ impl EncodePipeline { Ok(Self { display_params, + streaming_params, composite_pipeline, convert_pipeline, @@ -457,11 +467,16 @@ impl EncodePipeline { ); } + // We're converting the blend image, which is scRGB. + let input_color_space = ColorSpace::LinearExtSrgb; + self.convert_pipeline.cmd_convert( frame.render_cb, frame.blend_image.width, frame.blend_image.height, frame.convert_ds, + input_color_space, + self.streaming_params.profile, ); // Do a queue transfer for vulkan encode. @@ -630,16 +645,22 @@ fn new_swapframe( )?; let mut plane_views = Vec::new(); + let (single_plane_format, double_plane_format) = disjoint_plane_formats(encode_image.format) + .ok_or(anyhow!( + "couldn't find a disjoint plane formats for {:?}", + encode_image.format + ))?; + let disjoint_formats = if format_is_semiplanar(encode_image.format) { vec![ - vk::Format::R8_UNORM, // Y - vk::Format::R8G8_UNORM, // UV + single_plane_format, // Y + double_plane_format, // UV ] } else { vec![ - vk::Format::R8_UNORM, // Y - vk::Format::R8_UNORM, // U - vk::Format::R8_UNORM, // V + single_plane_format, // Y + single_plane_format, // U + single_plane_format, // V ] }; @@ -754,15 +775,55 @@ fn format_is_semiplanar(format: vk::Format) -> bool { format, vk::Format::G8_B8R8_2PLANE_420_UNORM | vk::Format::G8_B8R8_2PLANE_422_UNORM + | vk::Format::G8_B8R8_2PLANE_444_UNORM | vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 | vk::Format::G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 | vk::Format::G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16 | vk::Format::G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 | vk::Format::G16_B16R16_2PLANE_420_UNORM | vk::Format::G16_B16R16_2PLANE_422_UNORM - | vk::Format::G8_B8R8_2PLANE_444_UNORM - | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 - | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 | vk::Format::G16_B16R16_2PLANE_444_UNORM ) } + +fn disjoint_plane_formats(format: vk::Format) -> Option<(vk::Format, vk::Format)> { + match format { + vk::Format::G8_B8R8_2PLANE_420_UNORM + | vk::Format::G8_B8R8_2PLANE_422_UNORM + | vk::Format::G8_B8R8_2PLANE_444_UNORM + | vk::Format::G8_B8_R8_3PLANE_420_UNORM + | vk::Format::G8_B8_R8_3PLANE_422_UNORM + | vk::Format::G8_B8_R8_3PLANE_444_UNORM => { + Some((vk::Format::R8_UNORM, vk::Format::R8G8_UNORM)) + } + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16 => Some(( + vk::Format::R10X6_UNORM_PACK16, + vk::Format::R10X6G10X6_UNORM_2PACK16, + )), + vk::Format::G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16 => Some(( + vk::Format::R12X4_UNORM_PACK16, + vk::Format::R12X4G12X4_UNORM_2PACK16, + )), + vk::Format::G16_B16R16_2PLANE_420_UNORM + | vk::Format::G16_B16R16_2PLANE_422_UNORM + | vk::Format::G16_B16R16_2PLANE_444_UNORM + | vk::Format::G16_B16_R16_3PLANE_420_UNORM + | vk::Format::G16_B16_R16_3PLANE_422_UNORM + | vk::Format::G16_B16_R16_3PLANE_444_UNORM => { + Some((vk::Format::R16_UNORM, vk::Format::R16G16_UNORM)) + } + _ => None, + } +} diff --git a/mm-server/src/compositor/video/composite.rs b/mm-server/src/compositor/video/composite.rs index 58695a1..0a27437 100644 --- a/mm-server/src/compositor/video/composite.rs +++ b/mm-server/src/compositor/video/composite.rs @@ -7,13 +7,33 @@ use std::sync::Arc; use anyhow::Context; use ash::vk; use cstr::cstr; +use tracing::trace; -use crate::vulkan::*; +use crate::{color::ColorSpace, vulkan::*}; use super::SurfaceTexture; pub const BLEND_FORMAT: vk::Format = vk::Format::R16G16B16A16_SFLOAT; +// Also defined in composite.slang. +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +enum SurfaceColorSpace { + Srgb = 0, + LinearExtSrgb = 1, + Hdr10 = 2, +} + +impl From for SurfaceColorSpace { + fn from(cs: ColorSpace) -> Self { + match cs { + ColorSpace::Srgb => SurfaceColorSpace::Srgb, + ColorSpace::LinearExtSrgb => SurfaceColorSpace::LinearExtSrgb, + ColorSpace::Hdr10 => SurfaceColorSpace::Hdr10, + } + } +} + #[derive(Copy, Clone, Debug)] #[repr(C)] #[allow(dead_code)] @@ -25,6 +45,7 @@ struct SurfacePC { // TODO: suck it up and use a matrix transform (mat3) to support rotations. dst_pos: glam::Vec2, dst_size: glam::Vec2, + color_space: SurfaceColorSpace, } /// Composites surfaces into a blend image. @@ -68,7 +89,7 @@ impl CompositePipeline { let pipeline_layout = { let ranges = [vk::PushConstantRange::default() - .stage_flags(vk::ShaderStageFlags::VERTEX) + .stage_flags(vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT) .offset(0) .size(std::mem::size_of::() as u32)]; let set_layouts = [descriptor_set_layout]; @@ -239,11 +260,19 @@ impl CompositePipeline { ) -> anyhow::Result<()> { let device = &self.vk.device; + let color_space = match tex { + SurfaceTexture::Uploaded { .. } => ColorSpace::Srgb, + SurfaceTexture::Imported { color_space, .. } => *color_space, + }; + + trace!(?color_space, ?dst_pos, ?dst_size, "compositing surface"); + let pc = SurfacePC { src_pos: glam::Vec2::ZERO, src_size: glam::Vec2::ONE, dst_pos, dst_size, + color_space: color_space.into(), }; // Push the texture. @@ -280,7 +309,7 @@ impl CompositePipeline { device.cmd_push_constants( cb, self.pipeline_layout, - vk::ShaderStageFlags::VERTEX, + vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT, 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, diff --git a/mm-server/src/compositor/video/composite.slang b/mm-server/src/compositor/video/composite.slang index 060b7d3..fe45ac4 100644 --- a/mm-server/src/compositor/video/composite.slang +++ b/mm-server/src/compositor/video/composite.slang @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: BUSL-1.1 +import color; + const Sampler2D texture; struct VertOutput @@ -10,16 +12,26 @@ struct VertOutput float4 position : SV_Position; }; -struct VertPushConstants +// This must match the enum in composite.rs. +enum InputTextureColorSpace +{ + SRGB = 0, + LINEAR_EXTENDED_SRGB = 1, + HDR10 = 2, +} + +struct PushConstants { float2 src_pos; float2 src_size; float2 dst_pos; float2 dst_size; + + InputTextureColorSpace color_space; }; [[vk::push_constant]] -VertPushConstants vert_pc; +PushConstants pc; [shader("vertex")] VertOutput vert(uint vid: SV_VertexID) @@ -42,20 +54,37 @@ VertOutput vert(uint vid: SV_VertexID) } VertOutput output; - output.position = float4(vert_pc.dst_pos + vert_pc.dst_size * corner, 0.0, 1.0); - output.uv = vert_pc.src_pos + vert_pc.src_size * corner; + output.position = float4(pc.dst_pos + pc.dst_size * corner, 0.0, 1.0); + output.uv = pc.src_pos + pc.src_size * corner; return output; } -float srgb_linearize(float x) +float3 linearize(float3 color, InputTextureColorSpace color_space) { - if (x > 0.04045) - { - return pow((x + 0.055) / 1.055, 2.4); - } - else + switch (color_space) { - return x / 12.92; + case InputTextureColorSpace::SRGB: + return srgb_eotf(color); + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + return color; + case InputTextureColorSpace::HDR10: + float3 linear = pq_eotf(color); + + // The resulting values have the range 0-1, where 1.0 corresponds 10,000 + // nits. In order to effectively blend with SDR textures, we need to + // scale based on our virtual display brightness, producing values where + // 1.0 matches the maximum brightness that SDR content would produce. We + // use the Rec. 2408 value of 203 nits for this. On this scale, a value + // of 300 nits would result in a scaled value of about 1.47, and 1.0 + // would result in about 49.26. Either value would be clipped unless we + // use a floating-point blend format (which we do). + // TODO: allow HDR metadata to override the scaling factor. This is called + // "nominal diffuse white level" or NDWL. + linear *= PQ_MAX_WHITE / SDR_REFERENCE_WHITE; + + return transform(linear, Primaries::BT2020, Primaries::BT709); + default: + return srgb_eotf(color); } } @@ -66,30 +95,23 @@ float4 frag(float2 uv: TextureCoord) float4 color = texture.Sample(uv); // Wayland specifies that textures have premultiplied alpha. If we just - // import a dmabuf as sRGB, the colors are wrong, since vulkan expects sRGB - // textures to have not-premultiplied alpha. + // import a dmabuf as as an _SRGB format, the colors are wrong, since vulkan + // expects sRGB textures to have not-premultiplied alpha. // // Vulkan normally expects to do the sRGB -> linear conversion when sampling // in the shader. However, we're bypassing that operation here, by importing // the texture as UNORM (even though it's stored as sRGB) and then doing the // conversion manually. - // - // TODO: For imported textures with no alpha channel (XR24), we should skip - // this and use a sRGB view into the texture instead. if (color.a == 0) - { return float4(0); - } + else if (pc.color_space == InputTextureColorSpace::LINEAR_EXTENDED_SRGB) + // We're already in the right space for blending. + return color; color.rgb /= color.a; - - color.rgb = float3( - srgb_linearize(color.r), - srgb_linearize(color.g), - srgb_linearize(color.b)); - + color.rgb = linearize(color.rgb, pc.color_space); color.rgb *= color.a; - color.a = 1; + return color; } diff --git a/mm-server/src/compositor/video/convert.rs b/mm-server/src/compositor/video/convert.rs index d3148b4..6e0c1d8 100644 --- a/mm-server/src/compositor/video/convert.rs +++ b/mm-server/src/compositor/video/convert.rs @@ -7,21 +7,54 @@ use std::sync::Arc; use ash::vk; use tracing::instrument; -use crate::vulkan::*; +use crate::{ + color::{ColorSpace, VideoProfile}, + vulkan::*, +}; use super::VkPlaneView; -// GLSL requires vec4s for alignment. -const COLORSPACE_BT709: [[f32; 4]; 3] = [ - [0.2126, 0.7152, 0.0722, 0.0], - [-0.1146, -0.3854, 0.5, 0.0], - [0.5, -0.4542, -0.0458, 0.0], -]; +// Also defined in convert.slang. +#[repr(u32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum InputTextureColorSpace { + Srgb = 0, + LinearExtSrgb = 1, + Hdr10 = 2, +} + +impl From for InputTextureColorSpace { + fn from(cs: ColorSpace) -> Self { + match cs { + ColorSpace::Srgb => InputTextureColorSpace::Srgb, + ColorSpace::LinearExtSrgb => InputTextureColorSpace::LinearExtSrgb, + ColorSpace::Hdr10 => InputTextureColorSpace::Hdr10, + } + } +} + +// Also defined in convert.slang. +#[repr(u32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum OutputProfile { + Hd = 0, + Hdr10 = 1, +} + +impl From for OutputProfile { + fn from(profile: VideoProfile) -> Self { + match profile { + VideoProfile::Hd => OutputProfile::Hd, + VideoProfile::Hdr10 => OutputProfile::Hdr10, + } + } +} #[repr(C)] -#[derive(Copy, Clone, Debug)] -struct ColorspacePC { - color_space: [[f32; 4]; 3], +#[derive(Debug, Copy, Clone)] +struct ConvertPushConstants { + input_color_space: InputTextureColorSpace, + output_profile: OutputProfile, } pub struct ConvertPipeline { @@ -100,7 +133,7 @@ impl ConvertPipeline { let ranges = [vk::PushConstantRange::default() .stage_flags(vk::ShaderStageFlags::COMPUTE) .offset(0) - .size(std::mem::size_of::() as u32)]; + .size(std::mem::size_of::() as u32)]; let set_layouts = [descriptor_set_layout]; let create_info = vk::PipelineLayoutCreateInfo::default() @@ -150,6 +183,8 @@ impl ConvertPipeline { width: u32, height: u32, descriptor_set: vk::DescriptorSet, + input_color_space: ColorSpace, + video_profile: VideoProfile, ) { self.vk .device @@ -164,8 +199,9 @@ impl ConvertPipeline { &[], ); - let pc = ColorspacePC { - color_space: COLORSPACE_BT709, // TODO + let pc = ConvertPushConstants { + input_color_space: input_color_space.into(), + output_profile: video_profile.into(), }; self.vk.device.cmd_push_constants( @@ -175,7 +211,7 @@ impl ConvertPipeline { 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, - std::mem::size_of::(), + std::mem::size_of::(), ), ); diff --git a/mm-server/src/compositor/video/convert.slang b/mm-server/src/compositor/video/convert.slang index 533c577..f2167e1 100644 --- a/mm-server/src/compositor/video/convert.slang +++ b/mm-server/src/compositor/video/convert.slang @@ -2,59 +2,84 @@ // // SPDX-License-Identifier: BUSL-1.1 +import color; + const Sampler2D blend_image; -[[vk::image_format("r8")]] const RWTexture2D luminance; #ifdef SEMIPLANAR -[[vk::image_format("rg8")]] const RWTexture2D chroma_uv; #else -[[vk::image_format("r8")]] const RWTexture2D chroma_u; -[[vk::image_format("r8")]] const RWTexture2D chroma_v; #endif -struct PushConstants +// This must match the enum in convert.rs. +enum InputTextureColorSpace { - float3x3 color_transform; + SRGB = 0, + LINEAR_EXTENDED_SRGB = 1, + HDR10 = 2, } -[[vk::push_constant]] -PushConstants pc; +/// This must match the enum in convert.rs. +enum OutputProfile +{ + HD = 0, + HDR10 = 1, +} -float3 rgb_to_ycbcr(float3 color) +struct PushConstants { - let yuv = mul(color, transpose(pc.color_transform)); - - // The matrix multiplication gives us Y in [0, 1] and Cb and Cr in [-0.5, 0.5]. - // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. - return float3( - (219.0 * yuv.x + 16.0) / 256.0, - (224.0 * yuv.y + 128.0) / 256.0, - (224.0 * yuv.z + 128.0) / 256.0); + InputTextureColorSpace input_color_space; + OutputProfile output_profile; } -float rgb709_unlinear(float s) +[[vk::push_constant]] +PushConstants pc; + +float3 to_bt709(float3 rgb, InputTextureColorSpace color_space) { - if (s >= 0.018) + float3 linear; + switch (color_space) { - return 1.099 * pow(s, 1.0 / 2.2) - 0.099; - } - else + case InputTextureColorSpace::SRGB: + linear = srgb_eotf(rgb); + break; + case InputTextureColorSpace::HDR10: { - return 4.5 * s; + // Treat 203 nits as 1.0, and clip everything above that. + linear = pq_eotf(rgb); + linear = clamp(linear * (PQ_MAX_WHITE / SDR_REFERENCE_WHITE), 0.0, 1.0); + break; } + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + linear = clamp(rgb, 0.0, 1.0); + break; + } + + return bt709_inverse_eotf(linear); } -float3 unlinearize(float3 color) +float3 to_bt2020_pq(float3 rgb, InputTextureColorSpace color_space) { - return float3( - rgb709_unlinear(color.r), - rgb709_unlinear(color.g), - rgb709_unlinear(color.b)); + float3 bt2020_linear; + switch (color_space) + { + case InputTextureColorSpace::SRGB: + bt2020_linear = transform(srgb_eotf(rgb), Primaries::BT709, Primaries::BT2020); + break; + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + bt2020_linear = transform(rgb, Primaries::BT709, Primaries::BT2020); + break; + case InputTextureColorSpace::HDR10: + // Happy identity path. + return rgb; + } + + // Tone-map 1.0 to 203 nits, then delinearize. + return clamp(pq_inverse_eotf(bt2020_linear * (SDR_REFERENCE_WHITE / PQ_MAX_WHITE)), 0.0, 1.0); } [shader("compute")] @@ -72,8 +97,20 @@ void main(uint2 self_id: SV_DispatchThreadID) for (j = 0; j < 2; j += 1) { let texel_coords = coords + uint2(j, k); - let texel = blend_image.Load(uint3(texel_coords, 0)); - let yuv = rgb_to_ycbcr(unlinearize(texel.rgb)); + float4 texel = blend_image.Load(uint3(texel_coords, 0)); + + float3 yuv; + switch (pc.output_profile) + { + case OutputProfile::HD: + yuv = encode_ycbcr(to_bt709(texel.rgb, pc.input_color_space), + YCbCrModel::BT709, false); + break; + case OutputProfile::HDR10: + yuv = encode_ycbcr(to_bt2020_pq(texel.rgb, pc.input_color_space), + YCbCrModel::BT2020, false); + break; + } luminance[texel_coords] = yuv.x; diff --git a/mm-server/src/compositor/video/textures.rs b/mm-server/src/compositor/video/textures.rs index f749eb2..7044375 100644 --- a/mm-server/src/compositor/video/textures.rs +++ b/mm-server/src/compositor/video/textures.rs @@ -18,7 +18,7 @@ use smithay::{ }; use tracing::{debug, error, trace, warn}; -use crate::vulkan::*; +use crate::{color::ColorSpace, vulkan::*}; use super::dmabuf::import_dma_texture; @@ -87,6 +87,7 @@ pub enum SurfaceTexture { buffer: wl_buffer::WlBuffer, image: Rc, semaphore: vk::Semaphore, + color_space: ColorSpace, }, } @@ -247,6 +248,7 @@ impl TextureManager { surface: &wl_surface::WlSurface, buffer: &wl_buffer::WlBuffer, dmabuf: dmabuf::Dmabuf, + color_space: ColorSpace, ) -> anyhow::Result<()> { let DmabufCacheEntry { image, semaphore, .. @@ -262,6 +264,7 @@ impl TextureManager { semaphore, buffer: buffer.clone(), image, + color_space, }, ); diff --git a/mm-server/src/compositor/video/vulkan_encode.rs b/mm-server/src/compositor/video/vulkan_encode.rs index 764a1f0..3bbda3f 100644 --- a/mm-server/src/compositor/video/vulkan_encode.rs +++ b/mm-server/src/compositor/video/vulkan_encode.rs @@ -988,6 +988,14 @@ fn default_profile(op: vk::VideoCodecOperationFlagsKHR) -> vk::VideoProfileInfoK .luma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_8) } +fn default_hdr10_profile(op: vk::VideoCodecOperationFlagsKHR) -> vk::VideoProfileInfoKHR<'static> { + vk::VideoProfileInfoKHR::default() + .video_codec_operation(op) + .chroma_subsampling(vk::VideoChromaSubsamplingFlagsKHR::TYPE_420) + .chroma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_10) + .luma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_10) +} + fn default_encode_usage() -> vk::VideoEncodeUsageInfoKHR<'static> { vk::VideoEncodeUsageInfoKHR::default() .video_usage_hints(vk::VideoEncodeUsageFlagsKHR::STREAMING) diff --git a/mm-server/src/compositor/video/vulkan_encode/h264.rs b/mm-server/src/compositor/video/vulkan_encode/h264.rs index 316038d..c0958e0 100644 --- a/mm-server/src/compositor/video/vulkan_encode/h264.rs +++ b/mm-server/src/compositor/video/vulkan_encode/h264.rs @@ -14,6 +14,7 @@ use ash::vk::native::{ use bytes::Bytes; use tracing::trace; +use crate::color::VideoProfile; use crate::compositor::{AttachedClients, VideoStreamParams}; use crate::vulkan::*; @@ -70,16 +71,17 @@ impl H264Encoder { ) -> anyhow::Result { let (video_loader, encode_loader) = vk.video_apis.as_ref().unwrap(); - let profile_idc = 100; // HIGH + let op = vk::VideoCodecOperationFlagsKHR::ENCODE_H264_EXT; + let (profile, profile_idc) = match params.profile { + VideoProfile::Hd => (super::default_profile(op), 100), + VideoProfile::Hdr10 => (super::default_hdr10_profile(op), 110), + }; let h264_profile_info = vk::VideoEncodeH264ProfileInfoEXT::default().std_profile_idc(profile_idc); - let mut profile = H264EncodeProfile::new( - super::default_profile(vk::VideoCodecOperationFlagsKHR::ENCODE_H264_EXT), - super::default_encode_usage(), - h264_profile_info, - ); + let mut profile = + H264EncodeProfile::new(profile, super::default_encode_usage(), h264_profile_info); let mut caps = H264EncodeCapabilities::default(); @@ -95,10 +97,7 @@ impl H264Encoder { trace!("video capabilities: {:#?}", caps.video_caps); trace!("encode capabilities: {:#?}", caps.encode_caps); - trace!("h264 capabilities: {:#?}", caps.h264_caps); - - // let quality_level = caps.encode_caps.max_quality_levels - 1; - // let mut quality_props = H264QualityLevelProperties::default(); + trace!("h265 capabilities: {:#?}", caps.h264_caps); // unsafe { // let get_info = vk::PhysicalDeviceVideoEncodeQualityLevelInfoKHR::default() @@ -159,11 +158,16 @@ impl H264Encoder { trace!("crop right: {}, bottom: {}", crop_right, crop_bottom); + let (colour_primaries, transfer_characteristics, matrix_coefficients) = match params.profile + { + VideoProfile::Hd => (1, 1, 1), + VideoProfile::Hdr10 => (9, 16, 9), + }; + let mut vui = StdVideoH264SequenceParameterSetVui { - // BT.709. - colour_primaries: 1, - transfer_characteristics: 1, - matrix_coefficients: 1, + colour_primaries, + transfer_characteristics, + matrix_coefficients, // Unspecified. video_format: 5, ..unsafe { std::mem::zeroed() } @@ -179,11 +183,19 @@ impl H264Encoder { .ilog2() .saturating_sub(4) as u8; + let bit_depth = match params.profile { + VideoProfile::Hd => 8, + VideoProfile::Hdr10 => 10, + }; + let mut sps = StdVideoH264SequenceParameterSet { profile_idc, level_idc, chroma_format_idc: StdVideoH264ChromaFormatIdc_STD_VIDEO_H264_CHROMA_FORMAT_IDC_420, + bit_depth_chroma_minus8: bit_depth - 8, + bit_depth_luma_minus8: bit_depth - 8, + max_num_ref_frames: 1, pic_order_cnt_type: StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_0, log2_max_pic_order_cnt_lsb_minus4: log2_max_frame_num_minus4, diff --git a/mm-server/src/compositor/video/vulkan_encode/h265.rs b/mm-server/src/compositor/video/vulkan_encode/h265.rs index 9535573..2ef9acc 100644 --- a/mm-server/src/compositor/video/vulkan_encode/h265.rs +++ b/mm-server/src/compositor/video/vulkan_encode/h265.rs @@ -16,6 +16,7 @@ use ash::vk::native::{ use bytes::Bytes; use tracing::trace; +use crate::color::VideoProfile; use crate::compositor::{AttachedClients, VideoStreamParams}; use crate::vulkan::*; @@ -76,11 +77,14 @@ impl H265Encoder { let h265_profile_info = vk::VideoEncodeH265ProfileInfoEXT::default().std_profile_idc(profile_idc); - let mut profile = H265EncodeProfile::new( - super::default_profile(vk::VideoCodecOperationFlagsKHR::ENCODE_H265_EXT), - super::default_encode_usage(), - h265_profile_info, - ); + let op = vk::VideoCodecOperationFlagsKHR::ENCODE_H265_EXT; + let (profile, profile_idc) = match params.profile { + VideoProfile::Hd => (super::default_profile(op), 1), + VideoProfile::Hdr10 => (super::default_hdr10_profile(op), 2), + }; + + let mut profile = + H265EncodeProfile::new(profile, super::default_encode_usage(), h265_profile_info); let mut caps = H265EncodeCapabilities::default(); @@ -198,19 +202,23 @@ impl H265Encoder { trace!("crop right: {}, bottom: {}", crop_right, crop_bottom); + let (colour_primaries, transfer_characteristics, matrix_coeffs) = match params.profile { + VideoProfile::Hd => (1, 1, 1), + VideoProfile::Hdr10 => (9, 16, 9), + }; + let mut vui = StdVideoH265SequenceParameterSetVui { - // BT.709. - colour_primaries: 1, - transfer_characteristics: 1, - matrix_coeffs: 1, + colour_primaries, + transfer_characteristics, + matrix_coeffs, // Unspecified. video_format: 5, ..unsafe { std::mem::zeroed() } }; vui.flags.set_video_signal_type_present_flag(1); - vui.flags.set_video_full_range_flag(0); // Narrow range. vui.flags.set_colour_description_present_flag(1); + vui.flags.set_video_full_range_flag(0); // Narrow range. let ptl = StdVideoH265ProfileTierLevel { general_profile_idc: profile_idc, @@ -245,13 +253,18 @@ impl H265Encoder { let max_transform_hierarchy_depth = (max_ctb.ilog2() - min_tbs.ilog2()) as u8; + let bit_depth = match params.profile { + VideoProfile::Hd => 8, + VideoProfile::Hdr10 => 10, + }; + let mut sps = StdVideoH265SequenceParameterSet { chroma_format_idc: StdVideoH265ChromaFormatIdc_STD_VIDEO_H265_CHROMA_FORMAT_IDC_420, pic_width_in_luma_samples: aligned_width, pic_height_in_luma_samples: aligned_height, sps_max_sub_layers_minus1: layers_minus_1, - bit_depth_luma_minus8: 0, // TODO HDR - bit_depth_chroma_minus8: 0, // TODO HDR + bit_depth_luma_minus8: bit_depth - 8, + bit_depth_chroma_minus8: bit_depth - 8, log2_max_pic_order_cnt_lsb_minus4: 4, log2_min_luma_coding_block_size_minus3: (min_cb.ilog2() - 3) as u8, log2_diff_max_min_luma_coding_block_size: (max_cb.ilog2() - min_cb.ilog2()) as u8, diff --git a/mm-server/src/main.rs b/mm-server/src/main.rs index 85110f0..80f6f28 100644 --- a/mm-server/src/main.rs +++ b/mm-server/src/main.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BUSL-1.1 mod codec; +mod color; mod compositor; mod config; mod pixel_scale; diff --git a/mm-server/src/server/handlers.rs b/mm-server/src/server/handlers.rs index a9a2189..9b893d7 100644 --- a/mm-server/src/server/handlers.rs +++ b/mm-server/src/server/handlers.rs @@ -315,6 +315,7 @@ fn attach( }); let video_codec: protocol::VideoCodec = video_params.codec.into(); + let video_profile: protocol::VideoProfile = video_params.profile.into(); let audio_codec: protocol::AudioCodec = audio_params.codec.into(); let msg = protocol::Attached { session_id, @@ -325,6 +326,7 @@ fn attach( width: video_params.width, height: video_params.height, }), + video_profile: video_profile.into(), audio_codec: audio_codec.into(), sample_rate_hz: audio_params.sample_rate, diff --git a/mm-server/src/server/handlers/validation.rs b/mm-server/src/server/handlers/validation.rs index b5ab298..5c47e70 100644 --- a/mm-server/src/server/handlers/validation.rs +++ b/mm-server/src/server/handlers/validation.rs @@ -7,6 +7,7 @@ use tracing::debug; use crate::{ codec::{AudioCodec, VideoCodec}, + color::VideoProfile, compositor::{AudioStreamParams, DisplayParams, VideoStreamParams}, pixel_scale::PixelScale, waking_sender::WakingSender, @@ -45,6 +46,7 @@ pub fn validate_attachment( ) -> Result<(VideoStreamParams, AudioStreamParams)> { let (width, height) = validate_resolution(params.streaming_resolution)?; let video_codec = validate_video_codec(params.video_codec)?; + let video_profile = validate_profile(params.video_profile)?; let sample_rate = validate_sample_rate(params.sample_rate_hz)?; let channels = validate_channels(params.channels)?; @@ -55,6 +57,7 @@ pub fn validate_attachment( width, height, codec: video_codec, + profile: video_profile, }, AudioStreamParams { sample_rate, @@ -91,6 +94,20 @@ pub fn validate_ui_scale(ui_scale: Option) -> Result Result { + let p: protocol::VideoProfile = match profile.try_into() { + Ok(p) => p, + Err(_) => return Err(ValidationError::Invalid("invalid video profile".into())), + }; + + match p.try_into() { + Ok(p) => Ok(p), + _ => Err(ValidationError::NotSupported( + "unsupported video profile".into(), + )), + } +} + pub fn validate_video_codec(codec: i32) -> Result { let codec: protocol::VideoCodec = match codec.try_into() { Err(_) => return Err(ValidationError::Invalid("invalid video codec".into())), diff --git a/shader-common/color.slang b/shader-common/color.slang new file mode 100644 index 0000000..2c4c6bb --- /dev/null +++ b/shader-common/color.slang @@ -0,0 +1,221 @@ +// Copyright 2024 Colin Marc +// +// SPDX-License-Identifier: MIT + +module color; + +// A set of color primaries, defined in terms of a transformation to/from XYZ +// space. +public struct PrimariesTransform +{ + public float3x3 to_xyz; + public float3x3 from_xyz; +} + +// Named sets of color primaries. +namespace Primaries +{ +public static const PrimariesTransform BT709 = { + float3x3( + 0.4124564f, 0.3575761f, 0.1804375f, + 0.2126729f, 0.7151522f, 0.0721750f, + 0.0193339f, 0.1191920f, 0.9503041f), + float3x3( + 3.2404542f, -1.5371385f, -0.4985314f, + -0.9692660f, 1.8760108f, 0.0415560f, + 0.0556434f, -0.2040259f, 1.0572252f) +}; + +public static const PrimariesTransform BT2020 = { + float3x3( + 0.636958f, 0.1446169f, 0.1688810f, + 0.2627002f, 0.6779981f, 0.0593017f, + 0.0000000f, 0.0280727f, 1.0609851f), + float3x3( + 1.7166512, -0.3556708, -0.2533663, + -0.6666844, 1.6164812, 0.0157685, + 0.0176399, -0.0427706, 0.9421031), +}; +} + +// Applies the sRGB EOTF to a color, producing linear values. +public float3 srgb_eotf(float3 color) +{ + return float3( + srgb_eotf(color.r), + srgb_eotf(color.g), + srgb_eotf(color.b)); +} + +// Applies the sRGB EOTF to one channel of a color, producing a linear value. +public float srgb_eotf(float channel) +{ + return channel > 0.04045 ? pow((channel + 0.055) / 1.055, 2.4) : channel / 12.92; +} + +// Applies the inverse sRGB EOTF to a color, producing non-linear values. This +// is sometimes called gamma correction. +public float3 srgb_inverse_eotf(float3 color) +{ + return float3( + srgb_inverse_eotf(color.r), + srgb_inverse_eotf(color.g), + srgb_inverse_eotf(color.b)); +} + +// Applies the inverse sRGB EOTF to one channel of a color, producing non-linear +// values. This is sometimes called gamma correction. +public float srgb_inverse_eotf(float channel) +{ + return channel > 0.0031308 ? 1.055 * pow(channel, 1.0 / 2.4) - 0.055 : 12.92 * channel; +} + +// Applies the BT.709 EOTF to a color, producing linear values. +public float3 bt709_eotf(float3 color) +{ + return float3( + bt709_eotf(color.r), + bt709_eotf(color.g), + bt709_eotf(color.b)); +} + +// Applies the BT.709 EOTF to one channel of a color, producing a linear value. +public float bt709_eotf(float channel) +{ + return channel > 0.081 ? pow((channel + 0.099) / 1.099, 1.0 / 0.45) : channel / 4.5; +} + +// Applies the inverse BT.709 EOTF to a color, producing non-linear values. This +// is sometimes called gamma correction. +public float3 bt709_inverse_eotf(float3 color) +{ + return float3( + bt709_inverse_eotf(color.r), + bt709_inverse_eotf(color.g), + bt709_inverse_eotf(color.b)); +} + +// Applies the inverse BT.709 EOTF to one channel of a color, producing non-linear +// values. This is sometimes called gamma correction. +public float bt709_inverse_eotf(float channel) +{ + return channel >= 0.018 ? 1.099 * pow(channel, 1.0 / 2.2) - 0.099 : 4.5 * channel; +} + +static const float PQ_M1 = 0.1593017578125; +static const float PQ_M2 = 78.84375; +static const float PQ_C1 = 0.8359375; +static const float PQ_C2 = 18.8515625; +static const float PQ_C3 = 18.6875; + +public static const float SDR_REFERENCE_WHITE = 203.0; +public static const float PQ_MAX_WHITE = 10000.0; + +// Applies the Perceptual Quantizer EOTF to a color, producing linear values. +// The input should be in the range [0, 1], where 1 corresponds to the maximum +// 10,000 nits. +public float3 pq_eotf(float3 color) +{ + return float3( + pq_eotf(color.r), + pq_eotf(color.g), + pq_eotf(color.b)); +} + +// Applies the Perceptual Quantizer EOTF to a color channel, producing linear +// values. The input should be in the range [0, 1], where 1 corresponds to the +// maximum 10,000 nits. +float pq_eotf(float channel) +{ + let c = pow(channel, 1.0 / PQ_M2); + return pow( + max(c - PQ_C1, 0.0) / (PQ_C2 - PQ_C3 * c), + 1.0 / PQ_M1); +} + +// Applies the inverse Perceptual Quantizer EOTF to a color, producing non-linear +// values. The output will be in the range [0, 1], where 1 corresponds to the +// maximum 10,000 nits. +public float3 pq_inverse_eotf(float3 color) +{ + return float3( + pq_inverse_eotf(color.r), + pq_inverse_eotf(color.g), + pq_inverse_eotf(color.b)); +} + +// Applies the inverse Perceptual Quantizer EOTF to a color channel, producing a +// non-linear value. The output will be in the range [0, 1], where 1 corresponds +// to the maximum 10,000 nits. +float pq_inverse_eotf(float channel) +{ + let c = pow(channel, PQ_M1); + return pow( + (PQ_C1 + PQ_C2 * c) / (1.0 + PQ_C3 * c), + PQ_M2); +} + +// Transform a color from one set of primaries to another. The colors must be +// linear, that is, they must have already been linearized using the relevant +// OETF. +public float4 transform(float4 color, PrimariesTransform pa, PrimariesTransform pb) +{ + return float4( + transform(color.rgb, pa, pb), + color.a); +} + +// Transform a color from one set of primaries to another. The colors must be +// linear, that is, they must have already been linearized using the relevant +// inverse EOTF. +public float3 transform(float3 color, PrimariesTransform pa, PrimariesTransform pb) +{ + let mat = mul(pb.from_xyz, pa.to_xyz); + return mul(mat, color); +} + +// Available conversions to and from YCbCr color space. +public enum YCbCrModel +{ + BT709, + BT2020, +} + +static const float3x3 YCBCR_709_MATRIX = float3x3( + 0.2126, 0.7152, 0.0722, + -0.114572, -0.385428, 0.5, + 0.5, -0.454153, -0.045847); + +static const float3x3 YCBCR_2020_MATRIX = float3x3( + 0.2627, 0.6780, 0.0593, + -0.139630, -0.360370, 0.5, + 0.5, -0.459786, -0.040214); + +// Encode a color in the YCbCr color system. The color should already be in +// nonlinear space. +public float3 encode_ycbcr(float3 color, YCbCrModel model, bool full_range) +{ + float3 ycbcr; + switch (model) + { + case YCbCrModel::BT709: + ycbcr = mul(YCBCR_709_MATRIX, color); + break; + case YCbCrModel::BT2020: + ycbcr = mul(YCBCR_2020_MATRIX, color); + break; + } + + // The matrix multiplication gives us Y in [0, 1] and Cb and Cr in [-0.5, 0.5]. + ycbcr.y += 0.5; + ycbcr.z += 0.5; + + if (!full_range) + // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. + ycbcr = float3( + (219.0 * ycbcr.x + 16.0) / 256.0, + (224.0 * ycbcr.y + 16.0) / 256.0, + (224.0 * ycbcr.z + 16.0) / 256.0); + + return clamp(ycbcr, 0.0, 1.0); +}