From 91ea9e707f9fbe393067b817fdca8342b2bfb800 Mon Sep 17 00:00:00 2001 From: Colin Marc Date: Mon, 25 Mar 2024 20:02:46 +0100 Subject: [PATCH 1/5] feat(server): accept HDR textures via the wp-color-management-v1 wayland protocol This adds support for HDR application textures via dmabuf, using the (draft version of) the wayland color management protocol. The textures get blended into scRGB and eventually clipped, since we have no facility to output HDR yet. --- mm-server/Cargo.toml | 1 + mm-server/src/color.rs | 41 + mm-server/src/compositor.rs | 4 + mm-server/src/compositor/handlers.rs | 3 +- mm-server/src/compositor/handlers/buffers.rs | 15 +- .../compositor/handlers/color_management.rs | 364 +++++ .../protocol/mesa-color-management-v1.xml | 1233 +++++++++++++++++ mm-server/src/compositor/video/color.slang | 193 +++ mm-server/src/compositor/video/composite.rs | 32 +- .../src/compositor/video/composite.slang | 73 +- mm-server/src/compositor/video/convert.slang | 37 +- mm-server/src/compositor/video/textures.rs | 5 +- mm-server/src/main.rs | 1 + 13 files changed, 1942 insertions(+), 60 deletions(-) create mode 100644 mm-server/src/color.rs create mode 100644 mm-server/src/compositor/handlers/color_management.rs create mode 100644 mm-server/src/compositor/protocol/mesa-color-management-v1.xml create mode 100644 mm-server/src/compositor/video/color.slang diff --git a/mm-server/Cargo.toml b/mm-server/Cargo.toml index 5c2f738..b35bd31 100644 --- a/mm-server/Cargo.toml +++ b/mm-server/Cargo.toml @@ -72,6 +72,7 @@ serde_json = "1.0.114" cursor-icon = "1.1.0" image = { version = "0.25.1", default-features = false, features = ["png"] } git-version = "0.3.9" +wayland-scanner = "0.31.1" [dependencies.pulseaudio] git = "https://github.com/colinmarc/pulseaudio-rs" diff --git a/mm-server/src/color.rs b/mm-server/src/color.rs new file mode 100644 index 0000000..0020e08 --- /dev/null +++ b/mm-server/src/color.rs @@ -0,0 +1,41 @@ +/// A combination of color primaries, white point, and transfer function. We +/// generally ignore white point, since we deal only with colorspaces using the +/// D65 white point. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ColorSpace { + /// Uses BT.709 primaries and the sRGB transfer function. + Srgb, + /// Uses BT.709 primaries and a linear transfer function. Usually encoded as + /// a float with negative values and values above 1.0 used to represent the + /// extended space. + LinearExtSrgb, + /// Uses BT.2020 primaries and the ST2084 (PQ) transfer function. + Hdr10, +} + +impl ColorSpace { + pub fn from_primaries_and_tf( + primaries: Primaries, + transfer_function: TransferFunction, + ) -> Option { + match (primaries, transfer_function) { + (Primaries::Srgb, TransferFunction::Srgb) => Some(ColorSpace::Srgb), + (Primaries::Srgb, TransferFunction::Linear) => Some(ColorSpace::LinearExtSrgb), + (Primaries::Bt2020, TransferFunction::Pq) => Some(ColorSpace::Hdr10), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum TransferFunction { + Linear, + Srgb, + Pq, +} + +#[derive(Debug, Clone, Copy)] +pub enum Primaries { + Srgb, + Bt2020, +} diff --git a/mm-server/src/compositor.rs b/mm-server/src/compositor.rs index 04870b5..af6b0d3 100644 --- a/mm-server/src/compositor.rs +++ b/mm-server/src/compositor.rs @@ -122,6 +122,7 @@ pub struct State { compositor_state: smithay::wayland::compositor::CompositorState, dmabuf_state: smithay::wayland::dmabuf::DmabufState, _dmabuf_global: smithay::wayland::dmabuf::DmabufGlobal, + _color_global: handlers::color_management::ColorManagementGlobal, // output_manager_state: smithay::wayland::output::OutputManagerState, xdg_shell_state: smithay::wayland::shell::xdg::XdgShellState, shm_state: smithay::wayland::shm::ShmState, @@ -196,6 +197,8 @@ impl Compositor { let dmabuf_global = dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback); + let color_global = handlers::color_management::ColorManagementGlobal::new(&dh); + let xwayland_shell_state = smithay::wayland::xwayland_shell::XWaylandShellState::new::(&dh); @@ -237,6 +240,7 @@ impl Compositor { compositor_state, dmabuf_state, _dmabuf_global: dmabuf_global, + _color_global: color_global, // output_manager_state, xdg_shell_state, shm_state, diff --git a/mm-server/src/compositor/handlers.rs b/mm-server/src/compositor/handlers.rs index 97450c4..fa4a42d 100644 --- a/mm-server/src/compositor/handlers.rs +++ b/mm-server/src/compositor/handlers.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BUSL-1.1 mod buffers; +pub mod color_management; mod input; mod x11; mod xdg_shell; @@ -53,7 +54,7 @@ impl compositor::CompositorHandler for State { removed_surfaces.push(surf.clone()); } Some(compositor::BufferAssignment::NewBuffer(buffer)) => { - buffers::buffer_commit(self, surf, &buffer).unwrap(); + buffers::buffer_commit(self, surf, data, &buffer).unwrap(); } None => (), }; diff --git a/mm-server/src/compositor/handlers/buffers.rs b/mm-server/src/compositor/handlers/buffers.rs index afa9dbf..8c285fb 100644 --- a/mm-server/src/compositor/handlers/buffers.rs +++ b/mm-server/src/compositor/handlers/buffers.rs @@ -7,10 +7,14 @@ use smithay::{ protocol::{wl_buffer, wl_surface}, Resource, }, - wayland::{buffer, dmabuf, shm}, + wayland::{buffer, compositor, dmabuf, shm}, }; use tracing::{debug, error, trace}; +use crate::{ + color::ColorSpace, compositor::handlers::color_management::ColorManagementCachedState, +}; + use super::State; impl buffer::BufferHandler for State { @@ -60,6 +64,7 @@ impl dmabuf::DmabufHandler for State { pub fn buffer_commit( state: &mut State, surface: &wl_surface::WlSurface, + surface_data: &compositor::SurfaceData, buffer: &wl_buffer::WlBuffer, ) -> anyhow::Result<()> { trace!( @@ -78,9 +83,15 @@ pub fn buffer_commit( match dmabuf::get_dmabuf(buffer) { Ok(dmabuf) => { + let colorspace = surface_data + .cached_state + .current::() + .colorspace + .unwrap_or(ColorSpace::Srgb); + return state .texture_manager - .attach_dma_buffer(surface, buffer, dmabuf); + .attach_dma_buffer(surface, buffer, dmabuf, colorspace); } Err(smithay::utils::UnmanagedResource) => (), // Fall through to shm handler. } diff --git a/mm-server/src/compositor/handlers/color_management.rs b/mm-server/src/compositor/handlers/color_management.rs new file mode 100644 index 0000000..0856cfa --- /dev/null +++ b/mm-server/src/compositor/handlers/color_management.rs @@ -0,0 +1,364 @@ +// Copyright 2024 Colin Marc +// +// SPDX-License-Identifier: BUSL-1.1 + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] + +use std::sync::{Arc, Mutex}; + +use smithay::{ + reexports::wayland_server::{ + backend::GlobalId, protocol::wl_surface, Client, DataInit, Dispatch, DisplayHandle, + GlobalDispatch, New, Resource, WEnum, + }, + wayland::compositor, +}; + +use crate::color::{ColorSpace, Primaries, TransferFunction}; + +use self::protocol::{ + mesa_color_management_surface_v1::MesaColorManagementSurfaceV1, + mesa_color_manager_v1::{self, Feature, MesaColorManagerV1, RenderIntent}, + mesa_image_description_creator_params_v1::MesaImageDescriptionCreatorParamsV1, + mesa_image_description_v1::{Cause, MesaImageDescriptionV1}, +}; + +impl From for mesa_color_manager_v1::TransferFunction { + fn from(tf: TransferFunction) -> Self { + match tf { + TransferFunction::Linear => mesa_color_manager_v1::TransferFunction::Linear, + TransferFunction::Srgb => mesa_color_manager_v1::TransferFunction::Srgb, + TransferFunction::Pq => mesa_color_manager_v1::TransferFunction::St2084Pq, + } + } +} + +impl TryFrom for TransferFunction { + type Error = (); + + fn try_from(tf: mesa_color_manager_v1::TransferFunction) -> Result { + match tf { + mesa_color_manager_v1::TransferFunction::Linear => Ok(TransferFunction::Linear), + mesa_color_manager_v1::TransferFunction::Srgb => Ok(TransferFunction::Srgb), + mesa_color_manager_v1::TransferFunction::St2084Pq => Ok(TransferFunction::Pq), + _ => Err(()), + } + } +} + +impl From for mesa_color_manager_v1::Primaries { + fn from(p: Primaries) -> Self { + match p { + Primaries::Srgb => mesa_color_manager_v1::Primaries::Srgb, + Primaries::Bt2020 => mesa_color_manager_v1::Primaries::Bt2020, + } + } +} + +impl TryFrom for Primaries { + type Error = (); + + fn try_from(p: mesa_color_manager_v1::Primaries) -> Result { + match p { + mesa_color_manager_v1::Primaries::Srgb => Ok(Primaries::Srgb), + mesa_color_manager_v1::Primaries::Bt2020 => Ok(Primaries::Bt2020), + _ => Err(()), + } + } +} + +use super::State; + +mod protocol { + use smithay::reexports::wayland_server; + use smithay::reexports::wayland_server::protocol::*; + + mod __interfaces { + use smithay::reexports::wayland_server; + use wayland_server::backend as wayland_backend; + use wayland_server::protocol::__interfaces::*; + wayland_scanner::generate_interfaces!( + "src/compositor/protocol/mesa-color-management-v1.xml" + ); + } + + use __interfaces::*; + wayland_scanner::generate_server_code!("src/compositor/protocol/mesa-color-management-v1.xml"); +} + +const SUPPORTED_TFS: [mesa_color_manager_v1::TransferFunction; 3] = [ + mesa_color_manager_v1::TransferFunction::Linear, + mesa_color_manager_v1::TransferFunction::Srgb, + mesa_color_manager_v1::TransferFunction::St2084Pq, +]; + +const SUPPORTED_PRIMARIES: [mesa_color_manager_v1::Primaries; 2] = [ + mesa_color_manager_v1::Primaries::Srgb, + mesa_color_manager_v1::Primaries::Bt2020, +]; + +pub struct ColorManagementGlobal { + _global: GlobalId, +} + +impl ColorManagementGlobal { + pub fn new(display: &DisplayHandle) -> Self { + let global = display.create_global::(1, ()); + Self { _global: global } + } +} + +struct ColorManagementSurfaceUserData { + wl_surface: wl_surface::WlSurface, +} + +/// Represents a pending image description for a color-managed surface. +#[derive(Debug, Default, Clone)] +pub struct ColorManagementCachedState { + pub colorspace: Option, +} + +impl compositor::Cacheable for ColorManagementCachedState { + fn commit(&mut self, _dh: &DisplayHandle) -> Self { + self.clone() + } + + fn merge_into(self, into: &mut Self, _dh: &DisplayHandle) { + *into = self; + } +} + +#[derive(Debug, Default)] +struct ImageDescParams { + primaries: Option, + transfer_function: Option, +} + +impl GlobalDispatch for State { + fn bind( + _state: &mut State, + _handle: &DisplayHandle, + _client: &Client, + resource: New, + _global_data: &(), + data_init: &mut DataInit<'_, State>, + ) { + let color_manager = data_init.init(resource, ()); + + color_manager.supported_intent(RenderIntent::Perceptual); + color_manager.supported_feature(Feature::Parametric); + + for tf in SUPPORTED_TFS.iter().copied() { + color_manager.supported_tf_named(tf); + } + + for primaries in SUPPORTED_PRIMARIES.iter().copied() { + color_manager.supported_primaries_named(primaries); + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + _resource: &MesaColorManagerV1, + request: ::Request, + _data: &(), + _dhandle: &DisplayHandle, + data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_color_manager_v1::Request::GetOutput { .. } => { + todo!() + } + protocol::mesa_color_manager_v1::Request::GetSurface { id, surface } => { + data_init.init( + id, + ColorManagementSurfaceUserData { + wl_surface: surface, + }, + ); + } + protocol::mesa_color_manager_v1::Request::NewIccCreator { obj } => { + data_init.post_error( + obj, + protocol::mesa_color_manager_v1::Error::UnsupportedFeature, + "ICC profiles are not supported.", + ); + } + protocol::mesa_color_manager_v1::Request::NewParametricCreator { obj } => { + data_init.init(obj, Arc::new(Mutex::new(Default::default()))); + } + protocol::mesa_color_manager_v1::Request::Destroy => (), + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaColorManagementSurfaceV1, + request: ::Request, + data: &ColorManagementSurfaceUserData, + _dhandle: &DisplayHandle, + _data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_color_management_surface_v1::Request::SetImageDescription { + image_description, + render_intent, + } => { + if render_intent != WEnum::Value(RenderIntent::Perceptual) { + resource.post_error( + protocol::mesa_color_management_surface_v1::Error::RenderIntent as u32, + "Only perceptual render intent is supported", + ); + + return; + } + + if let Some(colorspace) = image_description.data::() { + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = Some(*colorspace); + }); + } + } + protocol::mesa_color_management_surface_v1::Request::UnsetImageDescription => { + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = None; + }); + } + protocol::mesa_color_management_surface_v1::Request::GetPreferred { .. } => todo!(), + protocol::mesa_color_management_surface_v1::Request::Destroy => { + // XXX: Why does the protocol allow multiple? + compositor::with_states(&data.wl_surface, |states| { + states + .cached_state + .pending::() + .colorspace = None; + }); + } + } + } +} + +impl Dispatch>> for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaImageDescriptionCreatorParamsV1, + request: ::Request, + data: &Arc>, + _dhandle: &DisplayHandle, + data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_image_description_creator_params_v1::Request::SetTfNamed { tf } => { + match tf { + WEnum::Value(tf) if SUPPORTED_TFS.contains(&tf) => { + data.lock().unwrap().transfer_function = Some(tf); + } + _ => { + resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidTf as u32, + "Unsupported transfer function", + ); + } + } + } + protocol::mesa_image_description_creator_params_v1::Request::SetPrimariesNamed { primaries } => { + match primaries { + WEnum::Value(p) if SUPPORTED_PRIMARIES.contains(&p) => { + data.lock().unwrap().primaries = Some(p); + }, + _ => { + resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidPrimaries as u32, + "Unsupported primaries", + ); + } + } + } + protocol::mesa_image_description_creator_params_v1::Request::Create { image_description } => { + let params = data.lock().unwrap(); + if params.transfer_function.is_none() || params.primaries.is_none() { + data_init.post_error( + image_description, + protocol::mesa_image_description_creator_params_v1::Error::IncompleteSet as u32, + "Primaries and transfer function must be set", + ); + + return; + } + + let colorspace = match (params.primaries.unwrap().try_into(), params.transfer_function.unwrap().try_into()) { + (Ok(primaries), Ok(transfer_function)) => ColorSpace::from_primaries_and_tf(primaries, transfer_function), + _ => None, + }; + + if let Some(colorspace) = colorspace { + let image_description = data_init.init(image_description, colorspace); + image_description.ready(colorspace as u32); + } else { + // We init and then immediately fail. + let image_description = data_init.init(image_description, ColorSpace::Srgb); + image_description.failed(Cause::Unsupported, "Unsupported combination of transfer function and primaries".to_string()); + } + } + protocol::mesa_image_description_creator_params_v1::Request::SetTfPower { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidTf as u32, + "set_tf_power not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetPrimaries { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidPrimaries as u32, + "set_primaries not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMasteringDisplayPrimaries {.. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidMastering as u32, + "set_mastering_display_primaries not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMasteringLuminance { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InvalidLuminance as u32, + "set_mastering_luminance not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMaxCll { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InconsistentSet as u32, + "set_max_cll not supported", + ), + protocol::mesa_image_description_creator_params_v1::Request::SetMaxFall { .. } => resource.post_error( + protocol::mesa_image_description_creator_params_v1::Error::InconsistentSet as u32, + "set_max_fall not supported", + ), + } + } +} + +impl Dispatch for State { + fn request( + _state: &mut State, + _client: &Client, + resource: &MesaImageDescriptionV1, + request: ::Request, + _data: &ColorSpace, + _dhandle: &DisplayHandle, + _data_init: &mut DataInit<'_, State>, + ) { + match request { + protocol::mesa_image_description_v1::Request::GetInformation { .. } => resource + .post_error( + protocol::mesa_image_description_v1::Error::NoInformation as u32, + "No information available", + ), + protocol::mesa_image_description_v1::Request::Destroy => (), + } + } +} diff --git a/mm-server/src/compositor/protocol/mesa-color-management-v1.xml b/mm-server/src/compositor/protocol/mesa-color-management-v1.xml new file mode 100644 index 0000000..b04a43d --- /dev/null +++ b/mm-server/src/compositor/protocol/mesa-color-management-v1.xml @@ -0,0 +1,1233 @@ + + + + Copyright 2019 Sebastian Wick + Copyright 2019 Erwin Burema + Copyright 2020 AMD + Copyright 2020, 2022, 2023 Collabora, Ltd. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice (including the next + paragraph) shall be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + The aim of the color management extension is to allow clients to know + the color properties of outputs, and to tell the compositor about the color + properties of their content on surfaces. Doing this enables a compositor + to perform automatic color management of content for different outputs + according to how content is intended to look like. + + The color properties are represented as an image description object which + is immutable after it has been created. A wl_output always has an + associated image description that clients can observe. A wl_surface + always has an associated preferred image description as a hint chosen by + the compositor that clients can also observe. Clients can set an image + description on a wl_surface to denote the color characteristics of the + surface contents. + + An image description includes SDR and HDR colorimetry and encoding, HDR + metadata, and viewing environment parameters. An image description does + not include the properties set through color-representation extension. + It is expected that the color-representation extension is used in + conjunction with the color management extension when necessary, + particularly with the YUV family of pixel formats. + + Recommendation ITU-T H.273 + "Coding-independent code points for video signal type identification" + shall be referred to as simply H.273 here. + + The color-and-hdr repository + (https://gitlab.freedesktop.org/pq/color-and-hdr) contains + background information on the protocol design and legacy color management. + It also contains a glossary, learning resources for digital color, tools, + samples and more. + + The terminology used in this protocol is based on common color science and + color encoding terminology where possible. The glossary in the color-and-hdr + repository shall be the authority on the definition of terms in this + protocol. + + + + + A global interface used for getting color management extensions for + wl_surface and wl_output objects, and for creating client defined image + description objects. The extension interfaces allow + getting the image description of outputs and setting the image + description of surfaces. + + + + + Destroy the mesa_color_manager_v1 object. This does not affect any other + objects in any way. + + + + + + + + + + See the ICC.1:2022 specification from the International Color Consortium + for more details about rendering intents. + + The principles of ICC defined rendering intents apply with all types + of image descriptions, not only those with ICC file profiles. + + Compositors must support the perceptual rendering intent. Other + rendering intents are optional. + + + + + + + + + + + + + + + + + + + The compositor supports set_mastering_display_primaries request + with a target color volume fully contained inside the primary + color volume. + + + + + The compositor additionally supports target color volumes that + extend outside of the primary color volume. + + This can only be advertised if feature set_mastering_display_primaries + is supported as well. + + + + + + + Named color primaries used to encode well-known sets of primaries. + H.273 is the authority, when it comes to the exact values of primaries and authoritative specifications, + where an equivalent code point exists. + Descriptions do list the specifications for convenience. + + + + Color primaries as defined by + - Rec. ITU-R BT.709-6 + - Rec. ITU-R BT.1361-0 conventional colour gamut system and extended colour gamut system (historical) + - IEC 61966-2-1 sRGB or sYCC + - IEC 61966-2-4 + - Society of Motion Picture and Television Engineers (SMPTE) RP 177 (1993) Annex B + Equivalent to H.273 ColourPrimaries code point 1. + + + + + Color primaries as defined by + - Rec. ITU-R BT.470-6 System M (historical) + - United States National Television System Committee 1953 Recommendation for transmission standards for color television + - United States Federal Communications Commission (2003) Title 47 Code of Federal Regulations 73.682 (a)(20) + Equivalent to H.273 ColourPrimaries code point 4. + + + + + Color primaries as defined by + - Rec. ITU-R BT.470-6 System B, G (historical) + - Rec. ITU-R BT.601-7 625 + - Rec. ITU-R BT.1358-0 625 (historical) + - Rec. ITU-R BT.1700-0 625 PAL and 625 SECAM + Equivalent to H.273 ColourPrimaries code point 5. + + + + + Color primaries as defined by + - Rec. ITU-R BT.601-7 525 + - Rec. ITU-R BT.1358-1 525 or 625 (historical) + - Rec. ITU-R BT.1700-0 NTSC + - SMPTE 170M (2004) + - SMPTE 240M (1999) (historical) + Equivalent to H.273 ColourPrimaries code point 6 and 7. + + + + + Color primaries as defined by H.273 for generic film. + Equivalent to H.273 ColourPrimaries code point 8. + + + + + Color primaries as defined by + - Rec. ITU-R BT.2020-2 + - Rec. ITU-R BT.2100-0 + Equivalent to H.273 ColourPrimaries code point 9. + + + + + Color primaries as defined as the maximum of the CIE 1931 XYZ color space by + - SMPTE ST 428-1 + - (CIE 1931 XYZ as in ISO 11664-1) + Equivalent to H.273 ColourPrimaries code point 10. + + + + + Color primaries as defined by Digital Cinema System and published in SMPTE RP 431-2 (2011). + Equivalent to H.273 ColourPrimaries code point 11. + + + + + Color primaries as defined by Digital Cinema System and published in SMPTE EG 432-1 (2010). + Equivalent to H.273 ColourPrimaries code point 12. + + + + + Color primaries as defined by Adobe as "Adobe RGB" and later published by ISO 12640-4 (2011). + + + + + + + Named Transfer Functions. + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.709-6 + - Rec. ITU-R BT.1361-0 conventional colour gamut system (historical) + Equivalent to H.273 TransferCharacteristics code point 1, 6, 14, 15. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.470-6 System M (historical) + - United States National Television System Committee 1953 Recommendation for transmission standards for color television + - United States Federal Communications Commission (2003) Title 47 Code of Federal Regulations 73.682 (a) (20) + - Rec. ITU-R BT.1700-0 625 PAL and 625 SECAM + Equivalent to H.273 TransferCharacteristics code point 4. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.470-6 System B, G (historical) + Equivalent to H.273 TransferCharacteristics code point 5. + + + + + Transfer characteristics as defined by + - SMPTE ST 240 (1999) + Equivalent to H.273 TransferCharacteristics code point 7. + + + + + Linear transfer characteristics. + Equivalent to H.273 TransferCharacteristics code point 8. + + + + + Logarithmic transfer characteristic (100:1 range). + Equivalent to H.273 TransferCharacteristics code point 9. + + + + + Logarithmic transfer characteristic (100 * Sqrt(10) : 1 range). + Equivalent to H.273 TransferCharacteristics code point 10. + + + + + Transfer characteristics as defined by + - IEC 61966-2-4 + Equivalent to H.273 TransferCharacteristics code point 11. + + + + + Transfer characteristics as defined by + - Rec. ITU-R BT.1361-0 extended colour gamut system (historical) + Equivalent to H.273 TransferCharacteristics code point 12. + + + + + Transfer characteristics as defined by + - IEC 61966-2-1 sRGB + Equivalent to H.273 TransferCharacteristics code point 13 with MatrixCoefficients set to 0. + + + + + Transfer characteristics as defined by + - IEC 61966-2-1 sYCC + Equivalent to H.273 TransferCharacteristics code point 13 with MatrixCoefficients set to anything but 0. + + + + + Transfer characteristics as defined by + - SMPTE ST 2084 (2014) for 10-, 12-, 14- and 16-bit systems + - Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system + Equivalent to H.273 TransferCharacteristics code point 16. + + + + + Transfer characteristics as defined by + - SMPTE ST 428-1 (2019) + Equivalent to H.273 TransferCharacteristics code point 17. + + + + + Transfer characteristics as defined by + - ARIB STD-B67 (2015) + - Rec. ITU-R BT.2100-2 hybrid log-gamma (HLG) system + Equivalent to H.273 TransferCharacteristics code point 18. + + + + + + + This creates a new mesa_color_management_output_v1 object for the + given wl_output. + + See the mesa_color_management_output_v1 interface for more details. + + + + + + + + + This creates a new color mesa_color_management_surface_v1 object for the + given wl_surface. + + See the mesa_color_management_surface_v1 interface for more details. + + + + + + + + + Makes a new ICC-based image description creator object with all + properties initially unset. The client can then use the object's + interface to define all the required properties for an image description + and finally create a mesa_image_description_v1 object. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.icc_v1_v4. + Otherwise this request raises the protocol error unsupported_feature. + + + + + + + + Makes a new parametric image description creator object with all + properties initially unset. The client can then use the object's + interface to define all the required properties for an image description + and finally create a mesa_image_description_v1 object. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.parametric. + Otherwise this request raises the protocol error unsupported_feature. + + + + + + + + When this object is created, it shall immediately send this event + once for each rendering intent the compositor supports. + + + + + + + + When this object is created, it shall immediately send this event + once for each compositor supported feature listed in the enumeration. + + + + + + + + When this object is created, it shall immediately send this event + once for each named transfer function the compositor + supports with the parametric image description creator. + + + + + + + + When this object is created, it shall immediately send this event + once for each named set of primaries the compositor + supports with the parametric image description creator. + + + + + + + + + A mesa_color_management_output_v1 describes the color properties of an + output. + + The mesa_color_management_output_v1 is associated with the wl_output global + underlying the wl_output object. Therefore the client destroying the + wl_output object has no impact, but the compositor removing the output + global makes the mesa_color_management_output_v1 object inert. + + + + + Destroy the color mesa_color_management_output_v1 object. This does not + affect any remaining protocol objects. + + + + + + This event is sent whenever the image description of the + output changed, followed by one wl_output.done event common to + output events across all extensions. + + If the client wants to use the updated image description, it needs + to do get_image_description again, because image description objects + are immutable. + + + + + + This creates a new mesa_image_description_v1 object for the current image description + of the output. There always is exactly one image description active for an + output so the client should destroy the image description created by earlier + invocations of this request. This request is usually sent as a reaction + to the image_description_changed event or when creating a + mesa_color_management_output_v1 object. + + The created mesa_image_description_v1 object preserves the image description + of the output from the time the object was created. + + The resulting image description object allows get_information request. + + If this protocol object is inert, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the no_output cause. + + If the interface version is inadequate for the output's image + description, meaning that the client does not support all the + events needed to deliver the crucial information, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the low_version cause. + + Otherwise the object shall immediately deliver the ready event. + + + + + + + + + A mesa_color_management_surface_v1 allows the client to set the color + space and HDR properties of a surface. + + If the wl_surface associated with the mesa_color_management_surface_v1 is + destroyed, the mesa_color_management_surface_v1 object becomes inert. + + + + + Destroy the mesa_color_management_surface_v1 object. + + When the last mesa_color_management_surface_v1 object for a wl_surface + is destroyed, it does the same as unset_image_description. + + + + + + + + + + + + + Set the image description of the underlying surface. The image + description and rendering intent are double-buffered state, see + wl_surface.commit. + + It is the client's responsibility to understand the image description + it sets on a surface, and to provide content that matches that image + description. + + A rendering intent provides the client's preference on how content + colors should be mapped to each output. The render_intent value must + be one advertised by the compositor with + mesa_color_manager_v1.render_intent event, otherwise the protocol error + render_intent is raised. + + By default, a surface does not have an associated image description + nor a rendering intent. The handling of color on such surfaces is + compositor implementation defined. Compositors should handle such + surfaces as sRGB but may handle them differently if they have specific + requirements. + + + + + + + + + This request removes any image description from the surface. See + set_image_description for how a compositor handles a surface without + an image description. This is double-buffered state, see + wl_surface.commit. + + + + + + The preferred image description is the one which likely has the most + performance and/or quality benefits for the compositor if used by the + client for its wl_surface contents. This event is sent whenever the + compositor changes the wl_surface's preferred image description. + + This is not an initial event. + + This event is merely a notification. When the client wants to know + what the preferred image description is, it shall use the get_preferred + request. + + The preferred image description is not automatically used for anything. + It is only a hint, and clients may set any valid image description with + set_image_description but there might be performance and color accuracy + improvements by providing the wl_surface contents in the preferred + image description. Therefore clients that can, should render according + to the preferred image description + + + + + + If this protocol object is inert, the protocol error inert is raised. + + This creates a new mesa_image_description_v1 object for the currently + preferred image description for the wl_surface. The client should + stop using and destroy the image descriptions created by earlier + invocations of this request for the associated wl_surface. + This request is usually sent as a reaction to the preferred_changed + event or when creating a mesa_color_management_surface_v1 object if + the client is capable of adapting to image descriptions. + + The created mesa_image_description_v1 object preserves the preferred image + description of the wl_surface from the time the object was created. + + The resulting image description object allows get_information request. + + If the interface version is inadequate for the preferred image + description, meaning that the client does not support all the + events needed to deliver the crucial information, the resulting image + description object shall immediately deliver the + mesa_image_description_v1.failed event with the low_version cause, + otherwise the object shall immediately deliver the ready event. + + + + + + + + + This type of object is used for collecting all the information required + to create a mesa_image_description_v1 object from an ICC file. A complete + set of required parameters consists of these properties: + - ICC file + + Each required property must be set exactly once if the client is to create + an image description. The set requests verify that a property was not + already set. The create request verifies that all required properties are + set. There may be several alternative requests for setting each property, + and in that case the client must choose one of them. + + Once all properties have been set, the create request must be used to + create the image description object, destroying the creator in the + process. + + + + + + + + + + + + + + + Create an image description object based on the ICC information + previously set on this object. A compositor must parse the ICC data in + some undefined but finite amount of time. + + The completeness of the parameter set is verified. If the set is not + complete, the protocol error incomplete_set is raised. For the + definition of a complete set, see the description of this interface. + + If the particular combination of the information is not supported + by the compositor, the resulting image description object shall + immediately deliver the mesa_image_description_v1.failed event with the + 'unsupported' cause. If a valid image description was created from the + information, the mesa_image_description_v1.ready event will eventually + be sent instead. + + This request destroys the mesa_image_description_creator_icc_v1 object. + + The resulting image description object does not allow get_information + request. + + + + + + + + Sets the ICC profile file to be used as the basis of the image + description. + + The data shall be found through the given fd at the given offset, + having the given length. The fd must seekable and readable. Violating + these requirements raises the bad_fd protocol error. + + If reading the data fails due to an error independent of the client, the + compositor shall send the mesa_image_description_v1.failed event on the + created mesa_image_description_v1 with the 'operating_system' cause. + + The maximum size of the ICC profile is 4 MB. If length is greater + than that or zero, the protocol error bad_size is raised. + If offset + length exceeds the file size, the protocol error + out_of_file is raised. + + A compositor may read the file at any time starting from this request + and only until whichever happens first: + - If create request was issued, the mesa_image_description_v1 object + delivers either failed or ready event; or + - if create request was not issued, this + mesa_image_description_creator_icc_v1 object is destroyed. + + A compositor shall not modify the contents of the file, and the fd may + be sealed for writes and size changes. The client must ensure to its + best ability that the data does not change while the compositor is + reading it. + + The data must represent a valid ICC profile. + The ICC profile version must be 2 or 4, it must be a 3 channel profile + and the class must be 'display'. + Violating these requirements will not result in a protocol error but + will eventually send the mesa_image_description_v1.failed event on the + created mesa_image_description_v1 with the 'unsupported' cause. + + See the International Color Consortium specification ICC.1:2022 for more + details about ICC profiles. + + If ICC file has already been set on this object, the protocol error + already_set is raised. + + + + + + + + + + + This type of object is used for collecting all the parameters required + to create a mesa_image_description_v1 object. A complete set of required + parameters consists of these properties: + - transfer characteristic function (tf) + - chromaticities of primaries and white point (primary color volume) + + The following properties are optional and have a well-defined default + if not explicitly set: + - mastering display primaries and white point (target color volume) + - mastering luminance range + - maximum content light level + - maximum frame-average light level + + Each required property must be set exactly once if the client is to create + an image description. The set requests verify that a property was not + already set. The create request verifies that all required properties are + set. There may be several alternative requests for setting each property, + and in that case the client must choose one of them. + + Once all properties have been set, the create request must be used to + create the image description object, destroying the creator in the + process. + + + + + + + + + + + + + + + + + Create an image description object based on the parameters previously + set on this object. + + The completeness of the parameter set is verified. If the set is not + complete, the protocol error incomplete_set is raised. For the + definition of a complete set, see the description of this interface. + + If the particular combination of the parameter set is not supported + by the compositor, the resulting image description object shall + immediately deliver the mesa_image_description_v1.failed event with the + 'unsupported' cause. If a valid image description was created from the + parameter set, the mesa_image_description_v1.ready event will eventually + be sent instead. + + This request destroys the mesa_image_description_creator_params_v1 + object. + + The resulting image description object does not allow get_information + request. + + + + + + + + Sets the transfer characteristic using explicitly enumerated named functions. + + Only names advertised with mesa_color_manager_v1 + event supported_tf_named are allowed. Other values shall raise the + protocol error invalid_tf. + + If transfer characteristic has already been set on this object, the + protocol error already_set is raised. + + + + + + + + Sets the color component transfer characteristic to a power curve + with the given exponent. This curve represents the conversion from + electrical to optical pixel or color values. + + The curve exponent shall be multiplied by 10000 to get the argument + eexp value to carry the precision of 4 decimals. + + The curve exponent must be at least 1.0 and at most 10.0. Otherwise + the protocol error invalid_tf is raised. + + If transfer characteristic has already been set on this object, the + protocol error already_set is raised. + + This request can be used when the compositor advertises + mesa_color_manager_v1.feature.set_tf_power. Otherwise this request raises + the protocol error invalid_tf. + + + + + + + + Sets the color primaries and white point using explicitly named sets. + This describes the primary color volume which is the basis + for color value encoding. + + Only names advertised with mesa_color_manager_v1 + event supported_primaries_named are allowed. Other values shall raise the + protocol error invalid_primaries. + + If primaries have already been set on this object, the protocol error + already_set is raised. + + + + + + + + Sets the color primaries and white point using CIE 1931 xy + chromaticity coordinates. This describes the primary color volume + which is the basis for color value encoding. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + If primaries have already been set on this object, the protocol error + already_set is raised. + + This request can be used if the compositor advertises + mesa_color_manager_v1.feature.set_primaries. Otherwise this request + raises the protocol error invalid_primaries. + + + + + + + + + + + + + + + Provides the color primaries and white point of the mastering display + using CIE 1931 xy chromaticity coordinates. This is compatible with the + SMPTE ST 2086 definition of HDR static metadata. + + The mastering display primaries define the target color volume. + + If mastering display primaries are not explicitly set, the target + color volume is assumed to be equal to the primary color volume. + + The target color volume is defined by all tristimulus values between 0.0 + and 1.0 (inclusive) of the color space defined by the given mastering + display primaries and white point. The colorimetry is identical between + the container color space and the mastering display color space, + including that no chromatic adaptation is applied even if the white + points differ. + + The target color volume can exceed the primary color volume to allow for + a greater color volume with an existing color space definition (for + example scRGB). It can be smaller than the primary color volume to + minimize gamut and tone mapping distances for big color spaces (HDR + metadata). + + To make use of the entire target color volume a suitable pixel format + has to be chosen (e.g. floating point to exceed the primary color + volume, or abusing limited quantization range as with xvYCC). + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + If mastering display primaries have already been set on this object, + the protocol error already_set is raised. + + This request can be used if the compositor advertises + mesa_color_manager_v1.feature.set_mastering_display_primaries. + Otherwise this request raises the protocol error invalid_mastering. + The advertisement implies support only for target color + volumes fully contained within the primary color volume. + + If a compositor additionally supports target color volume exceeding + the primary color volume, it must advertise + mesa_color_manager_v1.feature.extended_target_volume. + If a client uses target color volume exceeding the primary color volume + and the compositor does not support it, the result is implementation + defined. Compositors are recommended to detect this case and fail the + image description gracefully, but it may as well result in color + artifacts. + + + + + + + + + + + + + + + Sets the luminance range that was used during the content mastering + process as the minimum and maximum absolute luminance L. This is + compatible with the SMPTE ST 2086 definition of HDR static metadata. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + The mastering luminance range is undefined by default. + + If max L is less than or equal to min L, the protocol error + invalid_luminance is raised. + + Min L value is multiplied by 10000 to get the argument min_lum value + and carry precision of 4 decimals. Max L value is unscaled for max_lum. + + + + + + + + + Sets the maximum content light level (max_cll) as defined by CTA-861-H. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + max_cll is undefined by default. + + + + + + + + Sets the maximum frame-average light level (max_fall) as defined by + CTA-861-H. + + This can only be set when set_tf_cicp is used to set the transfer + characteristic to Rec. ITU-R BT.2100-2 perceptual quantization system. + Otherwise, 'create' request shall raise inconsistent_set protocol + error. + + max_fall is undefined by default. + + + + + + + + + An image description carries information about the color encoding used + on a surface when attached to a wl_surface via + mesa_color_management_surface_v1.set_image_description. A compositor can + use this information to decode pixel values into colorimetrically + meaningful quantities. + + Note, that the mesa_image_description_v1 object is not ready to be used + immediately after creation. The object eventually delivers either the + 'ready' or the 'failed' event, specified in all requests creating it. The + object is deemed "ready" after receiving the 'ready' event. + + An object which is not ready is illegal to use, it can only be destroyed. + Any other request in this interface shall result in the 'not_ready' + protocol error. Attempts to use an object which is not ready through other + interfaces shall raise protocol errors defined there. + + Once created and regardless of how it was created, a mesa_image_description_v1 + object always refers to one fixed image description. It cannot change + after creation. + + + + + Destroy this object. It is safe to destroy an object which is not ready. + + Destroying a mesa_image_description_v1 object has no side-effects, not + even if a mesa_color_management_surface_v1.set_image_description has + not yet been followed by a wl_surface.commit. + + + + + + + + + + + + + + + + + + + + + + If creating a mesa_image_description_v1 object fails for a reason that + is not defined as a protocol error, this event is sent. + The requests that create image description objects define whether + and when this can occur. Only such creation requests can trigger this + event. This event cannot be triggered after the image description was + successfully formed. + + Once this event has been sent, the mesa_image_description_v1 object will + never become ready and it can only be destroyed. + + + + + + + + + Once this event has been sent, the mesa_image_description_v1 object is + deemed "ready". Ready objects can be used to send requests and can be + used through other interfaces. + + Every ready mesa_image_description_v1 protocol object refers to an + underlying image description record in the compositor. Multiple protocol + objects may end up referring to the same record. Clients may identify + these "copies" by comparing their id numbers: if the numbers from two + protocol objects are identical, the protocol objects refer to the same + image description record. Two different image description records + cannot have the same id number simultaneously. The id number does not + change during the lifetime of the image description record. + + The id number is valid only as long as the protocol object is alive. + If all protocol objects referring to the same image description record + are destroyed, the id number may be recycled for a different image + description record. + + Image description id number is not a protocol object id. Zero is + reserved as an invalid id number. It shall not be possible for a + client to refer to an image description by its id number in protocol. + The id numbers might not be portable between Wayland connections. + + This identity allows clients to de-duplicate image description records + and avoid get_information request if they already have the image + description information. + + + + + + + + Creates a mesa_image_description_info_v1 object which delivers the + information that makes up the image description. + + Not all image description protocol objects allow get_information + request. Whether it is allowed or not is defined by the request that + created the object. If get_information is not allowed, the protocol + error no_information is raised. + + + + + + + + + Sends all matching events describing an image description object exactly + once and finally sends the 'done' event. + + Once a mesa_image_description_info_v1 object has delivered a 'done' event + it is automatically destroyed. + + Every mesa_image_description_info_v1 created from the same + mesa_image_description_v1 shall always return the exact same data. + + + + + Signals the end of information events and destroys the object. + + + + + + The icc argument provides a file descriptor to the client which may be + memory-mapped to provide the ICC profile matching the image description. + The fd is read-only, and if mapped then it must be mapped with + MAP_PRIVATE by the client. + + The ICC profile version and other details are determined by the + compositor. There is no provision for a client to ask for a specific + kind of a profile. + + + + + + + + + + Delivers the primary color volume primaries and white point + using CIE 1931 xy chromaticity coordinates. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + + + + + + + + + + + + + + Delivers the primary color volume primaries and white point using a + explicitly enumerated named set. + + + + + + + + The color component transfer characteristic of this image description + is a pure power curve. This event provides the exponent of the power + function. This curve represents the conversion from electrical to + optical pixel or color values. + + The curve exponent has been multiplied by 10000 to get the argument + eexp value to carry the precision of 4 decimals. + + + + + + + + Delivers the transfer characteristic using an explicitly enumerated + named function. + + + + + + + + Provides the color primaries and white point of the target + color volume using CIE 1931 xy chromaticity coordinates. This is + compatible with the SMPTE ST 2086 definition of HDR static metadata + for mastering displays. + + While primary color volume is about how color is encoded, the + target color volume is the actually displayable color volume. + If target color volume is equal to the primary color volume, + then this event is not sent. + + Each coordinate value is multiplied by 10000 to get the argument + value to carry precision of 4 decimals. + + + + + + + + + + + + + + + Provides the luminance range that the image description is targeting + as the minimum and maximum absolute luminance L. This is compatible + with the SMPTE ST 2086 definition of HDR static metadata. + + This luminance range is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + Min L value is multiplied by 10000 to get the argument min_lum value + and carry precision of 4 decimals. Max L value is unscaled for max_lum. + + + + + + + + + Provides the targeted max_cll of the image description. max_cll is + defined by CTA-861-H. + + This luminance is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + + + + + + + Provides the targeted max_fall of the image description. max_fall is + defined by CTA-861-H. + + This luminance is only theoretical and may not correspond to the + luminance of light emitted on an actual display. + + + + + + diff --git a/mm-server/src/compositor/video/color.slang b/mm-server/src/compositor/video/color.slang new file mode 100644 index 0000000..74a7ca2 --- /dev/null +++ b/mm-server/src/compositor/video/color.slang @@ -0,0 +1,193 @@ +module color; + +// A set of color primaries, defined in terms of a transformation to/from XYZ +// space. +public struct PrimariesTransform +{ + public float3x3 to_xyz; + public float3x3 from_xyz; +} + +// Named sets of color primaries. +namespace Primaries +{ +public static const PrimariesTransform BT709 = { + float3x3( + 0.4124564f, 0.3575761f, 0.1804375f, + 0.2126729f, 0.7151522f, 0.0721750f, + 0.0193339f, 0.1191920f, 0.9503041f), + float3x3( + 3.2404542f, -1.5371385f, -0.4985314f, + -0.9692660f, 1.8760108f, 0.0415560f, + 0.0556434f, -0.2040259f, 1.0572252f) +}; + +public static const PrimariesTransform BT2020 = { + float3x3( + 0.636958f, 0.1446169f, 0.1688810f, + 0.2627002f, 0.6779981f, 0.0593017f, + 0.0000000f, 0.0280727f, 1.0609851f), + float3x3( + 1.7166512, -0.3556708, -0.2533663, + -0.6666844, 1.6164812, 0.0157685, + 0.0176399, -0.0427706, 0.9421031), +}; +} + +// Named transfer functions. +public enum TransferFunction +{ + LINEAR, + SRGB, + BT709, + PQ, +} + +// Applies the EOTF of a transfer function to a color, linearizing the value. +// This is sometimes called removing gamma correction. +public float4 apply_eotf(float4 color, TransferFunction transferFunction) +{ + return float4( + apply_eotf(color.rgb, transferFunction), + color.a); +} + +// Applies the EOTF of a transfer function to a color, linearizing the value. +// This is sometimes called removing gamma correction. +public float3 apply_eotf(float3 color, TransferFunction transferFunction) +{ + return float3( + apply_eotf(color.r, transferFunction), + apply_eotf(color.g, transferFunction), + apply_eotf(color.b, transferFunction)); +} + +// Applies the EOTF of a transfer function to one channel of a color. This is +// sometimes called removing gamma correction. +public float apply_eotf(float channel, TransferFunction transferFunction) +{ + switch (transferFunction) + { + case TransferFunction::LINEAR: + return channel; + case TransferFunction::SRGB: + return channel > 0.04045 ? pow((channel + 0.055) / 1.055, 2.4) : channel / 12.92; + case TransferFunction::BT709: + return channel > 0.081 ? pow((channel + 0.099) / 1.099, 1.0 / 0.45) : channel / 4.5; + case TransferFunction::PQ: + return pq_eotf(channel); + } +} + +// Applies the inverse EOTF of a transfer function to a color, resulting in +// non-linear values. This is sometimes called gamma correction. +public float4 apply_inverse_eotf(float4 color, TransferFunction transferFunction) +{ + return float4( + apply_inverse_eotf(color.rgb, transferFunction), + color.a); +} + +// Applies the inverse EOTF of a transfer function to a color, resulting in +// non-linear values. This is sometimes called gamma correction. +public float3 apply_inverse_eotf(float3 color, TransferFunction transferFunction) +{ + return float3( + apply_inverse_eotf(color.r, transferFunction), + apply_inverse_eotf(color.g, transferFunction), + apply_inverse_eotf(color.b, transferFunction)); +} + +// Applies the inverse EOTF of a transfer function to one channel of a color, +// resulting in non-linear values. This is sometimes called gamma correction. +public float apply_inverse_eotf(float channel, TransferFunction transferFunction) +{ + switch (transferFunction) + { + case TransferFunction::LINEAR: + return channel; + case TransferFunction::SRGB: + return channel > 0.0031308 ? 1.055 * pow(channel, 1.0 / 2.4) - 0.055 : 12.92 * channel; + case TransferFunction::BT709: + return channel >= 0.018 ? 1.099 * pow(channel, 1.0 / 2.2) - 0.099 : 4.5 * channel; + case TransferFunction::PQ: + return pq_inverse_eotf(channel); + } +} + +static const float PQ_M1 = 0.1593017578125; +static const float PQ_M2 = 78.84375; +static const float PQ_C1 = 0.8359375; +static const float PQ_C2 = 18.8515625; +static const float PQ_C3 = 18.6875; + +public static const float SDR_REFERENCE_WHITE = 203.0; + +// Applies the Perceptual Quantizer EOTF to a color channel. The input should be +// in the range [0, 1], where 1 corresponds to the maximum 10,000 nits. +float pq_eotf(float channel) +{ + let c = pow(channel, 1.0 / PQ_M2); + return pow( + max(c - PQ_C1, 0.0) / (PQ_C2 - PQ_C3 * c), + 1.0 / PQ_M1); +} + +// Applies the inverse Perceptual Quantizer EOTF to a color channel. The output +// will be in the range [0, 1], where 1 corresponds to the maximum 10,000 nits. +float pq_inverse_eotf(float channel) +{ + let c = pow(channel, PQ_M1); + return pow( + (PQ_C1 + PQ_C2 * c) / (1.0 + PQ_C3 * c), + PQ_M2); +} + +// Applies a scaling function on the result of the PQ EOTF to map it to SDR +// space, based on a hypothetical maximum nits value for SDR. +public float4 scale_pq_to_sdr(float4 color, float max_nits) +{ + return float4( + scale_pq_to_sdr(color.rgb, max_nits), + color.a); +} + +// Applies a scaling function on the result of the PQ EOTF to map it to SDR +// space, based on a hypothetical maximum nits value for SDR. +public float3 scale_pq_to_sdr(float3 color, float max_nits) +{ + return color * (10000.0 / max_nits); +} + +// Applies a scaling function on the result of the PQ EOTF to map it to SDR +// space, based on a hypothetical maximum nits value for SDR. +public float scale_pq_to_sdr(float channel, float max_nits) +{ + return channel * (10000.0 / max_nits); +} + +// Transform a color from one set of primaries to another. The colors must be +// linear, that is, they must have already been linearized using the relevant +// OETF. +public float4 transform(float4 color, PrimariesTransform pa, PrimariesTransform pb) +{ + return float4( + transform(color.rgb, pa, pb), + color.a); +} + +// Transform a color from one set of primaries to another. The colors must be +// linear, that is, they must have already been linearized using the relevant +// inverse EOTF. +public float3 transform(float3 color, PrimariesTransform pa, PrimariesTransform pb) +{ + // XXX if it doesn't work, I got the row-major/column-major thing wrong here. + // let mat = mul(pa.to_xyz, pb.from_xyz); + // let xyz = mul(pa.to_xyz, color); + // return mul(pb.from_xyz, xyz); + let color = clamp(color, 0.0, 1.0); + let xyz = mul(pa.to_xyz, color); + return mul(pb.from_xyz, xyz); + // return mul(color, mat); + // TODO premul +} diff --git a/mm-server/src/compositor/video/composite.rs b/mm-server/src/compositor/video/composite.rs index 58695a1..5b580bb 100644 --- a/mm-server/src/compositor/video/composite.rs +++ b/mm-server/src/compositor/video/composite.rs @@ -8,12 +8,31 @@ use anyhow::Context; use ash::vk; use cstr::cstr; -use crate::vulkan::*; +use crate::{color::ColorSpace, vulkan::*}; use super::SurfaceTexture; pub const BLEND_FORMAT: vk::Format = vk::Format::R16G16B16A16_SFLOAT; +// Also defined in composite.slang. +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +enum SurfaceColorSpace { + Srgb = 1, + LinearExtSrgb = 2, + Hdr10 = 3, +} + +impl From for SurfaceColorSpace { + fn from(cs: ColorSpace) -> Self { + match cs { + ColorSpace::Srgb => SurfaceColorSpace::Srgb, + ColorSpace::LinearExtSrgb => SurfaceColorSpace::LinearExtSrgb, + ColorSpace::Hdr10 => SurfaceColorSpace::Hdr10, + } + } +} + #[derive(Copy, Clone, Debug)] #[repr(C)] #[allow(dead_code)] @@ -25,6 +44,7 @@ struct SurfacePC { // TODO: suck it up and use a matrix transform (mat3) to support rotations. dst_pos: glam::Vec2, dst_size: glam::Vec2, + color_space: SurfaceColorSpace, } /// Composites surfaces into a blend image. @@ -68,7 +88,7 @@ impl CompositePipeline { let pipeline_layout = { let ranges = [vk::PushConstantRange::default() - .stage_flags(vk::ShaderStageFlags::VERTEX) + .stage_flags(vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT) .offset(0) .size(std::mem::size_of::() as u32)]; let set_layouts = [descriptor_set_layout]; @@ -239,11 +259,17 @@ impl CompositePipeline { ) -> anyhow::Result<()> { let device = &self.vk.device; + let color_space = match tex { + SurfaceTexture::Uploaded { .. } => ColorSpace::Srgb, + SurfaceTexture::Imported { color_space, .. } => *color_space, + }; + let pc = SurfacePC { src_pos: glam::Vec2::ZERO, src_size: glam::Vec2::ONE, dst_pos, dst_size, + color_space: color_space.into(), }; // Push the texture. @@ -280,7 +306,7 @@ impl CompositePipeline { device.cmd_push_constants( cb, self.pipeline_layout, - vk::ShaderStageFlags::VERTEX, + vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT, 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, diff --git a/mm-server/src/compositor/video/composite.slang b/mm-server/src/compositor/video/composite.slang index 060b7d3..2fc5c9f 100644 --- a/mm-server/src/compositor/video/composite.slang +++ b/mm-server/src/compositor/video/composite.slang @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: BUSL-1.1 +import color; + const Sampler2D texture; struct VertOutput @@ -10,16 +12,26 @@ struct VertOutput float4 position : SV_Position; }; -struct VertPushConstants +// This must match the enum in composite.rs. +enum InputTextureColorSpace +{ + SRGB = 1, + LINEAR_EXTENDED_SRGB = 2, + HDR10 = 3, +} + +struct PushConstants { float2 src_pos; float2 src_size; float2 dst_pos; float2 dst_size; + + InputTextureColorSpace color_space; }; [[vk::push_constant]] -VertPushConstants vert_pc; +PushConstants pc; [shader("vertex")] VertOutput vert(uint vid: SV_VertexID) @@ -42,23 +54,11 @@ VertOutput vert(uint vid: SV_VertexID) } VertOutput output; - output.position = float4(vert_pc.dst_pos + vert_pc.dst_size * corner, 0.0, 1.0); - output.uv = vert_pc.src_pos + vert_pc.src_size * corner; + output.position = float4(pc.dst_pos + pc.dst_size * corner, 0.0, 1.0); + output.uv = pc.src_pos + pc.src_size * corner; return output; } -float srgb_linearize(float x) -{ - if (x > 0.04045) - { - return pow((x + 0.055) / 1.055, 2.4); - } - else - { - return x / 12.92; - } -} - [shader("fragment")] float4 frag(float2 uv: TextureCoord) : SV_Target @@ -66,30 +66,51 @@ float4 frag(float2 uv: TextureCoord) float4 color = texture.Sample(uv); // Wayland specifies that textures have premultiplied alpha. If we just - // import a dmabuf as sRGB, the colors are wrong, since vulkan expects sRGB - // textures to have not-premultiplied alpha. + // import a dmabuf as as an _SRGB format, the colors are wrong, since vulkan + // expects sRGB textures to have not-premultiplied alpha. // // Vulkan normally expects to do the sRGB -> linear conversion when sampling // in the shader. However, we're bypassing that operation here, by importing // the texture as UNORM (even though it's stored as sRGB) and then doing the // conversion manually. - // - // TODO: For imported textures with no alpha channel (XR24), we should skip - // this and use a sRGB view into the texture instead. if (color.a == 0) { return float4(0); } + else if (pc.color_space == InputTextureColorSpace::LINEAR_EXTENDED_SRGB) + { + // We're already in the right space for blending. + return color; + } color.rgb /= color.a; - color.rgb = float3( - srgb_linearize(color.r), - srgb_linearize(color.g), - srgb_linearize(color.b)); + switch (pc.color_space) + { + case InputTextureColorSpace::SRGB: + color.rgb = apply_eotf(color.rgb, TransferFunction::SRGB); + break; + case InputTextureColorSpace::HDR10: + float3 linear = apply_eotf(color.rgb, TransferFunction::PQ); + + // The resulting values have the range 0-1, where 1.0 corresponds 10,000 + // nits. In order to effectively blend with SDR textures, we need to + // scale based on our virtual display brightness, producing values where + // 1.0 matches the maximum brightness that SDR content would produce. We + // use the Rec. 2408 value of 203 nits for this. On this scale, a value + // of 300 nits would result in a scaled value of about 1.47, and 1.0 + // would result in about 49.26. Either value would be clipped unless we + // use a floating-point blend format (which we do). + // TODO: allow HDR metadata to override the scaling factor. This is called + // "nominal diffuse white level" or NDWL. + linear = scale_pq_to_sdr(linear, SDR_REFERENCE_WHITE); + + // Transform from the BT.2020 primaries to the BT.709 primaries. + color.rgb = transform(linear, Primaries::BT2020, Primaries::BT709); + break; + } color.rgb *= color.a; - color.a = 1; return color; } diff --git a/mm-server/src/compositor/video/convert.slang b/mm-server/src/compositor/video/convert.slang index 533c577..949240d 100644 --- a/mm-server/src/compositor/video/convert.slang +++ b/mm-server/src/compositor/video/convert.slang @@ -2,18 +2,16 @@ // // SPDX-License-Identifier: BUSL-1.1 +import color; + const Sampler2D blend_image; -[[vk::image_format("r8")]] const RWTexture2D luminance; #ifdef SEMIPLANAR -[[vk::image_format("rg8")]] const RWTexture2D chroma_uv; #else -[[vk::image_format("r8")]] const RWTexture2D chroma_u; -[[vk::image_format("r8")]] const RWTexture2D chroma_v; #endif @@ -27,7 +25,7 @@ PushConstants pc; float3 rgb_to_ycbcr(float3 color) { - let yuv = mul(color, transpose(pc.color_transform)); + let yuv = mul(pc.color_transform, color); // The matrix multiplication gives us Y in [0, 1] and Cb and Cr in [-0.5, 0.5]. // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. @@ -37,26 +35,6 @@ float3 rgb_to_ycbcr(float3 color) (224.0 * yuv.z + 128.0) / 256.0); } -float rgb709_unlinear(float s) -{ - if (s >= 0.018) - { - return 1.099 * pow(s, 1.0 / 2.2) - 0.099; - } - else - { - return 4.5 * s; - } -} - -float3 unlinearize(float3 color) -{ - return float3( - rgb709_unlinear(color.r), - rgb709_unlinear(color.g), - rgb709_unlinear(color.b)); -} - [shader("compute")] [numthreads(16, 16)] void main(uint2 self_id: SV_DispatchThreadID) @@ -72,8 +50,13 @@ void main(uint2 self_id: SV_DispatchThreadID) for (j = 0; j < 2; j += 1) { let texel_coords = coords + uint2(j, k); - let texel = blend_image.Load(uint3(texel_coords, 0)); - let yuv = rgb_to_ycbcr(unlinearize(texel.rgb)); + float4 texel = blend_image.Load(uint3(texel_coords, 0)); + + // Since we're using Rec. 709 primaries, but sampling scRGB, we need + // to clamp the values. + texel.rgb = clamp(texel.rgb, 0.0, 1.0); + + let yuv = rgb_to_ycbcr(apply_inverse_eotf(texel.rgb, TransferFunction::BT709)); luminance[texel_coords] = yuv.x; diff --git a/mm-server/src/compositor/video/textures.rs b/mm-server/src/compositor/video/textures.rs index f749eb2..7044375 100644 --- a/mm-server/src/compositor/video/textures.rs +++ b/mm-server/src/compositor/video/textures.rs @@ -18,7 +18,7 @@ use smithay::{ }; use tracing::{debug, error, trace, warn}; -use crate::vulkan::*; +use crate::{color::ColorSpace, vulkan::*}; use super::dmabuf::import_dma_texture; @@ -87,6 +87,7 @@ pub enum SurfaceTexture { buffer: wl_buffer::WlBuffer, image: Rc, semaphore: vk::Semaphore, + color_space: ColorSpace, }, } @@ -247,6 +248,7 @@ impl TextureManager { surface: &wl_surface::WlSurface, buffer: &wl_buffer::WlBuffer, dmabuf: dmabuf::Dmabuf, + color_space: ColorSpace, ) -> anyhow::Result<()> { let DmabufCacheEntry { image, semaphore, .. @@ -262,6 +264,7 @@ impl TextureManager { semaphore, buffer: buffer.clone(), image, + color_space, }, ); diff --git a/mm-server/src/main.rs b/mm-server/src/main.rs index 85110f0..80f6f28 100644 --- a/mm-server/src/main.rs +++ b/mm-server/src/main.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BUSL-1.1 mod codec; +mod color; mod compositor; mod config; mod pixel_scale; From 54a5990651f5bb506bd5d4930742c9e18ee1c551 Mon Sep 17 00:00:00 2001 From: Colin Marc Date: Wed, 27 Mar 2024 12:42:56 +0100 Subject: [PATCH 2/5] feat(protocol): allow attachments to be configured for HDR10 output --- mm-client/src/bin/mmclient.rs | 2 + mm-protocol/src/messages.proto | 48 ++++++++++++++++----- mm-server/src/color.rs | 32 ++++++++++++++ mm-server/src/compositor/control.rs | 2 + mm-server/src/server/handlers.rs | 2 + mm-server/src/server/handlers/validation.rs | 17 ++++++++ 6 files changed, 92 insertions(+), 11 deletions(-) diff --git a/mm-client/src/bin/mmclient.rs b/mm-client/src/bin/mmclient.rs index d46997f..9721109 100644 --- a/mm-client/src/bin/mmclient.rs +++ b/mm-client/src/bin/mmclient.rs @@ -621,6 +621,7 @@ impl App { session_id: self.session_id, streaming_resolution: self.remote_display_params.resolution.clone(), video_codec: self.configured_codec.into(), + video_profile: protocol::VideoProfile::Hd.into(), ..Default::default() }, None, @@ -937,6 +938,7 @@ fn main() -> Result<()> { session_id: session.session_id, streaming_resolution: Some(streaming_resolution), video_codec: configured_codec.into(), + video_profile: protocol::VideoProfile::Hd.into(), ..Default::default() }, None, diff --git a/mm-protocol/src/messages.proto b/mm-protocol/src/messages.proto index 0a4786f..ad9b10d 100644 --- a/mm-protocol/src/messages.proto +++ b/mm-protocol/src/messages.proto @@ -135,13 +135,13 @@ message PixelScale { // launch a session. message VirtualDisplayParameters { Size resolution = 1; // Required. - uint32 framerate_hz = 3; // Required. - PixelScale ui_scale = 2; // Required. + uint32 framerate_hz = 2; // Required. + PixelScale ui_scale = 3; // Required. } // ### Attachment type // -// This enum refers to the manner of attachment. +// This refers to the manner of attachment. enum AttachmentType { ATTACHMENT_TYPE_UNKNOWN = 0; ATTACHMENT_TYPE_OPERATOR = 1; @@ -150,7 +150,7 @@ enum AttachmentType { // ### Video codec // -// This enum refers to the codec used for a video stream. +// This refers to the codec used for a video stream. enum VideoCodec { VIDEO_CODEC_UNKNOWN = 0; VIDEO_CODEC_H264 = 1; @@ -158,9 +158,19 @@ enum VideoCodec { VIDEO_CODEC_AV1 = 3; } +// ### Video profile +// +// This refers to the profile used for a video stream. Profiles are fully +// defined in the output section, below. +enum VideoProfile { + VIDEO_PROFILE_UNKNOWN = 0; + VIDEO_PROFILE_HD = 1; + VIDEO_PROFILE_HDR10 = 2; +} + // ### Audio codec // -// This enum refers to the codec used for an audio stream. +// This refers to the codec used for an audio stream. enum AudioCodec { AUDIO_CODEC_UNKNOWN = 0; AUDIO_CODEC_OPUS = 1; @@ -448,6 +458,7 @@ message Attach { VideoCodec video_codec = 10; Size streaming_resolution = 11; + VideoProfile video_profile = 12; AudioCodec audio_codec = 15; AudioChannels channels = 16; @@ -466,8 +477,9 @@ message Attached { uint64 session_id = 1; // Required. uint64 attachment_id = 2; // Required. - VideoCodec video_codec = 10; // Required. - Size streaming_resolution = 11; // Required. + VideoCodec video_codec = 10; // Required. + Size streaming_resolution = 11; // Required. + VideoProfile video_profile = 12; // Required. AudioCodec audio_codec = 15; // Required. AudioChannels channels = 16; // Required. @@ -554,15 +566,29 @@ message Detach {} // // - The server must tag the video bitstream with resolution, framerate, and // YCbCr color space/range using whatever mechanism is supported by the codec -// (for example, PPS frames in H.264). Clients should use this information to -// verify that the parameters match the requested attachment parameters. +// (for example, PPS/VUI frames in H.264). Clients should use this +// information to verify that the parameters match the requested attachment +// parameters. // - The server must use YCbCr 4:2:0 chroma subsampling for the compressed // stream (this is sometimes called YUV420P, and is the default for most // implementations of H264, H265, and AV1). +// - For VIDEO_PROFILE_HD, a bit depth of 8, along with the Rec.709 color space +// and limited range must be used. For H.264, H.265, and AV1, this +// corresponds to `colour_primaries`, `transfer_characteristics`, and +// `matrix_coeffs` all equal to 1, and the `video_full_range_flag` set to 0 +// (named `color_range` for AV1). +// - For VIDEO_PROFILE_HDR10, a bit depth of 10, along with the Rec. 2100 color +// space and limited range must be used. For H.264, H.265, and AV1, this +// corresponds to `colour_primaries` and `matrix_coeffs` equal to 9, +// `transfer_characteristics` equal to 16, and the `video_full_range_flag` +// set to 0 (named `color_range` for AV1). The server should additionally use +// SEI headers (or metadata OBUs for AV1) to communicate HDR metadata such as +// mastering display color volume (MDCV) and content light level (CLL) +// information. // - The server may reuse an existing compression context for a new attachment, // but in this case the stream must be resumable by the client within a -// reasonable time frame. For H.265, for example, this means sending PPS -// frames with every keyframe, and keyframes every few seconds. +// reasonable time frame. For H.265, for example, this means sending headers +// with every keyframe, and keyframes every few seconds. // // ### Audio compression // diff --git a/mm-server/src/color.rs b/mm-server/src/color.rs index 0020e08..057579c 100644 --- a/mm-server/src/color.rs +++ b/mm-server/src/color.rs @@ -1,3 +1,5 @@ +use mm_protocol as protocol; + /// A combination of color primaries, white point, and transfer function. We /// generally ignore white point, since we deal only with colorspaces using the /// D65 white point. @@ -27,6 +29,36 @@ impl ColorSpace { } } +// A configuration for a compressed video bitstream. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum VideoProfile { + // Uses a bit depth of 8, BT.709 primaries and transfer function. + Hd, + // Uses a bit depth of 10, BT.2020 primaries and the ST2084 (PQ) transfer function. + Hdr10, +} + +impl TryFrom for VideoProfile { + type Error = String; + + fn try_from(profile: protocol::VideoProfile) -> Result { + match profile { + protocol::VideoProfile::Hd => Ok(VideoProfile::Hd), + protocol::VideoProfile::Hdr10 => Ok(VideoProfile::Hdr10), + _ => Err("invalid video profile".into()), + } + } +} + +impl From for protocol::VideoProfile { + fn from(profile: VideoProfile) -> Self { + match profile { + VideoProfile::Hd => protocol::VideoProfile::Hd, + VideoProfile::Hdr10 => protocol::VideoProfile::Hdr10, + } + } +} + #[derive(Debug, Clone, Copy)] pub enum TransferFunction { Linear, diff --git a/mm-server/src/compositor/control.rs b/mm-server/src/compositor/control.rs index 7e34712..3f6162e 100644 --- a/mm-server/src/compositor/control.rs +++ b/mm-server/src/compositor/control.rs @@ -6,6 +6,7 @@ use crossbeam_channel::Sender; use crate::{ codec::{AudioCodec, VideoCodec}, + color::VideoProfile, pixel_scale::PixelScale, }; @@ -22,6 +23,7 @@ pub struct VideoStreamParams { pub width: u32, pub height: u32, pub codec: VideoCodec, + pub profile: VideoProfile, } #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/mm-server/src/server/handlers.rs b/mm-server/src/server/handlers.rs index a9a2189..9b893d7 100644 --- a/mm-server/src/server/handlers.rs +++ b/mm-server/src/server/handlers.rs @@ -315,6 +315,7 @@ fn attach( }); let video_codec: protocol::VideoCodec = video_params.codec.into(); + let video_profile: protocol::VideoProfile = video_params.profile.into(); let audio_codec: protocol::AudioCodec = audio_params.codec.into(); let msg = protocol::Attached { session_id, @@ -325,6 +326,7 @@ fn attach( width: video_params.width, height: video_params.height, }), + video_profile: video_profile.into(), audio_codec: audio_codec.into(), sample_rate_hz: audio_params.sample_rate, diff --git a/mm-server/src/server/handlers/validation.rs b/mm-server/src/server/handlers/validation.rs index b5ab298..434cd90 100644 --- a/mm-server/src/server/handlers/validation.rs +++ b/mm-server/src/server/handlers/validation.rs @@ -7,6 +7,7 @@ use tracing::debug; use crate::{ codec::{AudioCodec, VideoCodec}, + color::VideoProfile, compositor::{AudioStreamParams, DisplayParams, VideoStreamParams}, pixel_scale::PixelScale, waking_sender::WakingSender, @@ -45,6 +46,7 @@ pub fn validate_attachment( ) -> Result<(VideoStreamParams, AudioStreamParams)> { let (width, height) = validate_resolution(params.streaming_resolution)?; let video_codec = validate_video_codec(params.video_codec)?; + let video_profile = validate_profile(params.video_profile)?; let sample_rate = validate_sample_rate(params.sample_rate_hz)?; let channels = validate_channels(params.channels)?; @@ -55,6 +57,7 @@ pub fn validate_attachment( width, height, codec: video_codec, + profile: video_profile, }, AudioStreamParams { sample_rate, @@ -91,6 +94,20 @@ pub fn validate_ui_scale(ui_scale: Option) -> Result Result { + let p: protocol::VideoProfile = match profile.try_into() { + Ok(p) => p, + Err(_) => return Err(ValidationError::Invalid("invalid video profile".into())), + }; + + match p.try_into() { + Ok(p) if p == VideoProfile::Hd => Ok(p), + _ => Err(ValidationError::NotSupported( + "unsupported video profile".into(), + )), + } +} + pub fn validate_video_codec(codec: i32) -> Result { let codec: protocol::VideoCodec = match codec.try_into() { Err(_) => return Err(ValidationError::Invalid("invalid video codec".into())), From b3b667456f22d6729f34a0ab00940cdce5d426bd Mon Sep 17 00:00:00 2001 From: Colin Marc Date: Sun, 31 Mar 2024 19:11:45 +0200 Subject: [PATCH 3/5] feat(server): support streaming in HDR --- mm-server/Cargo.toml | 2 +- mm-server/src/compositor/video.rs | 91 ++++++-- mm-server/src/compositor/video/color.slang | 214 ++++++++++-------- mm-server/src/compositor/video/composite.rs | 9 +- .../src/compositor/video/composite.slang | 67 +++--- mm-server/src/compositor/video/convert.rs | 64 ++++-- mm-server/src/compositor/video/convert.slang | 84 +++++-- .../src/compositor/video/vulkan_encode.rs | 8 + .../compositor/video/vulkan_encode/h264.rs | 40 ++-- .../compositor/video/vulkan_encode/h265.rs | 37 ++- mm-server/src/server/handlers/validation.rs | 2 +- 11 files changed, 416 insertions(+), 202 deletions(-) diff --git a/mm-server/Cargo.toml b/mm-server/Cargo.toml index b35bd31..0315c13 100644 --- a/mm-server/Cargo.toml +++ b/mm-server/Cargo.toml @@ -80,7 +80,7 @@ rev = "70ddb748f20ceecc20e963e571188124aeb30186" [dependencies.svt] git = "https://github.com/colinmarc/svt-rs" -rev = "ab9dbd872d8f01c2cb96d3f9fd880b26d37d1f0e" +rev = "747915fa4b2f7d0bd2c70ef5af108c75031efc53" optional = true features = ["av1", "hevc"] diff --git a/mm-server/src/compositor/video.rs b/mm-server/src/compositor/video.rs index d640986..23ae519 100644 --- a/mm-server/src/compositor/video.rs +++ b/mm-server/src/compositor/video.rs @@ -14,17 +14,21 @@ mod textures; mod timebase; mod vulkan_encode; -use cpu_encode::CpuEncoder; - +use anyhow::{anyhow, bail}; use smithay::{ reexports::wayland_server::{protocol::wl_surface, Resource}, wayland::compositor, }; use tracing::{error, instrument, trace, warn}; -use crate::{codec::VideoCodec, vulkan::*}; +use crate::{ + codec::VideoCodec, + color::{ColorSpace, VideoProfile}, + vulkan::*, +}; use super::{AttachedClients, DisplayParams, VideoStreamParams}; +use cpu_encode::CpuEncoder; pub use dmabuf::dmabuf_feedback; use textures::*; @@ -71,6 +75,10 @@ impl Encoder { } } + if params.profile != VideoProfile::Hd { + bail!("HDR requires vulkan encode") + } + Ok(Self::Cpu(CpuEncoder::new( vk, attachments, @@ -124,6 +132,7 @@ pub struct SwapFrame { pub struct EncodePipeline { display_params: DisplayParams, + streaming_params: VideoStreamParams, composite_pipeline: composite::CompositePipeline, convert_pipeline: convert::ConvertPipeline, @@ -142,10 +151,10 @@ impl EncodePipeline { stream_seq: u64, attachments: AttachedClients, display_params: DisplayParams, - stream_params: VideoStreamParams, + streaming_params: VideoStreamParams, ) -> anyhow::Result { - if stream_params.width != display_params.width - || stream_params.height != display_params.height + if streaming_params.width != display_params.width + || streaming_params.height != display_params.height { // Superres is not implemented yet. unimplemented!() @@ -155,7 +164,7 @@ impl EncodePipeline { vk.clone(), attachments.clone(), stream_seq, - stream_params, + streaming_params, display_params.framerate, )?; @@ -172,6 +181,7 @@ impl EncodePipeline { Ok(Self { display_params, + streaming_params, composite_pipeline, convert_pipeline, @@ -457,11 +467,16 @@ impl EncodePipeline { ); } + // We're converting the blend image, which is scRGB. + let input_color_space = ColorSpace::LinearExtSrgb; + self.convert_pipeline.cmd_convert( frame.render_cb, frame.blend_image.width, frame.blend_image.height, frame.convert_ds, + input_color_space, + self.streaming_params.profile, ); // Do a queue transfer for vulkan encode. @@ -630,16 +645,22 @@ fn new_swapframe( )?; let mut plane_views = Vec::new(); + let (single_plane_format, double_plane_format) = disjoint_plane_formats(encode_image.format) + .ok_or(anyhow!( + "couldn't find a disjoint plane formats for {:?}", + encode_image.format + ))?; + let disjoint_formats = if format_is_semiplanar(encode_image.format) { vec![ - vk::Format::R8_UNORM, // Y - vk::Format::R8G8_UNORM, // UV + single_plane_format, // Y + double_plane_format, // UV ] } else { vec![ - vk::Format::R8_UNORM, // Y - vk::Format::R8_UNORM, // U - vk::Format::R8_UNORM, // V + single_plane_format, // Y + single_plane_format, // U + single_plane_format, // V ] }; @@ -754,15 +775,55 @@ fn format_is_semiplanar(format: vk::Format) -> bool { format, vk::Format::G8_B8R8_2PLANE_420_UNORM | vk::Format::G8_B8R8_2PLANE_422_UNORM + | vk::Format::G8_B8R8_2PLANE_444_UNORM | vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 | vk::Format::G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 | vk::Format::G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16 | vk::Format::G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 | vk::Format::G16_B16R16_2PLANE_420_UNORM | vk::Format::G16_B16R16_2PLANE_422_UNORM - | vk::Format::G8_B8R8_2PLANE_444_UNORM - | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 - | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 | vk::Format::G16_B16R16_2PLANE_444_UNORM ) } + +fn disjoint_plane_formats(format: vk::Format) -> Option<(vk::Format, vk::Format)> { + match format { + vk::Format::G8_B8R8_2PLANE_420_UNORM + | vk::Format::G8_B8R8_2PLANE_422_UNORM + | vk::Format::G8_B8R8_2PLANE_444_UNORM + | vk::Format::G8_B8_R8_3PLANE_420_UNORM + | vk::Format::G8_B8_R8_3PLANE_422_UNORM + | vk::Format::G8_B8_R8_3PLANE_444_UNORM => { + Some((vk::Format::R8_UNORM, vk::Format::R8G8_UNORM)) + } + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16 + | vk::Format::G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16 => Some(( + vk::Format::R10X6_UNORM_PACK16, + vk::Format::R10X6G10X6_UNORM_2PACK16, + )), + vk::Format::G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16 + | vk::Format::G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16 => Some(( + vk::Format::R12X4_UNORM_PACK16, + vk::Format::R12X4G12X4_UNORM_2PACK16, + )), + vk::Format::G16_B16R16_2PLANE_420_UNORM + | vk::Format::G16_B16R16_2PLANE_422_UNORM + | vk::Format::G16_B16R16_2PLANE_444_UNORM + | vk::Format::G16_B16_R16_3PLANE_420_UNORM + | vk::Format::G16_B16_R16_3PLANE_422_UNORM + | vk::Format::G16_B16_R16_3PLANE_444_UNORM => { + Some((vk::Format::R16_UNORM, vk::Format::R16G16_UNORM)) + } + _ => None, + } +} diff --git a/mm-server/src/compositor/video/color.slang b/mm-server/src/compositor/video/color.slang index 74a7ca2..a09b3f7 100644 --- a/mm-server/src/compositor/video/color.slang +++ b/mm-server/src/compositor/video/color.slang @@ -34,85 +34,68 @@ public static const PrimariesTransform BT2020 = { }; } -// Named transfer functions. -public enum TransferFunction +// Applies the sRGB EOTF to a color, producing linear values. +public float3 srgb_eotf(float3 color) { - LINEAR, - SRGB, - BT709, - PQ, + return float3( + srgb_eotf(color.r), + srgb_eotf(color.g), + srgb_eotf(color.b)); } -// Applies the EOTF of a transfer function to a color, linearizing the value. -// This is sometimes called removing gamma correction. -public float4 apply_eotf(float4 color, TransferFunction transferFunction) +// Applies the sRGB EOTF to one channel of a color, producing a linear value. +public float srgb_eotf(float channel) { - return float4( - apply_eotf(color.rgb, transferFunction), - color.a); + return channel > 0.04045 ? pow((channel + 0.055) / 1.055, 2.4) : channel / 12.92; } -// Applies the EOTF of a transfer function to a color, linearizing the value. -// This is sometimes called removing gamma correction. -public float3 apply_eotf(float3 color, TransferFunction transferFunction) +// Applies the inverse sRGB EOTF to a color, producing non-linear values. This +// is sometimes called gamma correction. +public float3 srgb_inverse_eotf(float3 color) { return float3( - apply_eotf(color.r, transferFunction), - apply_eotf(color.g, transferFunction), - apply_eotf(color.b, transferFunction)); + srgb_inverse_eotf(color.r), + srgb_inverse_eotf(color.g), + srgb_inverse_eotf(color.b)); } -// Applies the EOTF of a transfer function to one channel of a color. This is -// sometimes called removing gamma correction. -public float apply_eotf(float channel, TransferFunction transferFunction) +// Applies the inverse sRGB EOTF to one channel of a color, producing non-linear +// values. This is sometimes called gamma correction. +public float srgb_inverse_eotf(float channel) { - switch (transferFunction) - { - case TransferFunction::LINEAR: - return channel; - case TransferFunction::SRGB: - return channel > 0.04045 ? pow((channel + 0.055) / 1.055, 2.4) : channel / 12.92; - case TransferFunction::BT709: - return channel > 0.081 ? pow((channel + 0.099) / 1.099, 1.0 / 0.45) : channel / 4.5; - case TransferFunction::PQ: - return pq_eotf(channel); - } + return channel > 0.0031308 ? 1.055 * pow(channel, 1.0 / 2.4) - 0.055 : 12.92 * channel; } -// Applies the inverse EOTF of a transfer function to a color, resulting in -// non-linear values. This is sometimes called gamma correction. -public float4 apply_inverse_eotf(float4 color, TransferFunction transferFunction) +// Applies the BT.709 EOTF to a color, producing linear values. +public float3 bt709_eotf(float3 color) { - return float4( - apply_inverse_eotf(color.rgb, transferFunction), - color.a); + return float3( + bt709_eotf(color.r), + bt709_eotf(color.g), + bt709_eotf(color.b)); +} + +// Applies the BT.709 EOTF to one channel of a color, producing a linear value. +public float bt709_eotf(float channel) +{ + return channel > 0.081 ? pow((channel + 0.099) / 1.099, 1.0 / 0.45) : channel / 4.5; } -// Applies the inverse EOTF of a transfer function to a color, resulting in -// non-linear values. This is sometimes called gamma correction. -public float3 apply_inverse_eotf(float3 color, TransferFunction transferFunction) +// Applies the inverse BT.709 EOTF to a color, producing non-linear values. This +// is sometimes called gamma correction. +public float3 bt709_inverse_eotf(float3 color) { return float3( - apply_inverse_eotf(color.r, transferFunction), - apply_inverse_eotf(color.g, transferFunction), - apply_inverse_eotf(color.b, transferFunction)); + bt709_inverse_eotf(color.r), + bt709_inverse_eotf(color.g), + bt709_inverse_eotf(color.b)); } -// Applies the inverse EOTF of a transfer function to one channel of a color, -// resulting in non-linear values. This is sometimes called gamma correction. -public float apply_inverse_eotf(float channel, TransferFunction transferFunction) +// Applies the inverse BT.709 EOTF to one channel of a color, producing non-linear +// values. This is sometimes called gamma correction. +public float bt709_inverse_eotf(float channel) { - switch (transferFunction) - { - case TransferFunction::LINEAR: - return channel; - case TransferFunction::SRGB: - return channel > 0.0031308 ? 1.055 * pow(channel, 1.0 / 2.4) - 0.055 : 12.92 * channel; - case TransferFunction::BT709: - return channel >= 0.018 ? 1.099 * pow(channel, 1.0 / 2.2) - 0.099 : 4.5 * channel; - case TransferFunction::PQ: - return pq_inverse_eotf(channel); - } + return channel >= 0.018 ? 1.099 * pow(channel, 1.0 / 2.2) - 0.099 : 4.5 * channel; } static const float PQ_M1 = 0.1593017578125; @@ -122,9 +105,22 @@ static const float PQ_C2 = 18.8515625; static const float PQ_C3 = 18.6875; public static const float SDR_REFERENCE_WHITE = 203.0; +public static const float PQ_MAX_WHITE = 10000.0; + +// Applies the Perceptual Quantizer EOTF to a color, producing linear values. +// The input should be in the range [0, 1], where 1 corresponds to the maximum +// 10,000 nits. +public float3 pq_eotf(float3 color) +{ + return float3( + pq_eotf(color.r), + pq_eotf(color.g), + pq_eotf(color.b)); +} -// Applies the Perceptual Quantizer EOTF to a color channel. The input should be -// in the range [0, 1], where 1 corresponds to the maximum 10,000 nits. +// Applies the Perceptual Quantizer EOTF to a color channel, producing linear +// values. The input should be in the range [0, 1], where 1 corresponds to the +// maximum 10,000 nits. float pq_eotf(float channel) { let c = pow(channel, 1.0 / PQ_M2); @@ -133,8 +129,20 @@ float pq_eotf(float channel) 1.0 / PQ_M1); } -// Applies the inverse Perceptual Quantizer EOTF to a color channel. The output -// will be in the range [0, 1], where 1 corresponds to the maximum 10,000 nits. +// Applies the inverse Perceptual Quantizer EOTF to a color, producing non-linear +// values. The output will be in the range [0, 1], where 1 corresponds to the +// maximum 10,000 nits. +public float3 pq_inverse_eotf(float3 color) +{ + return float3( + pq_inverse_eotf(color.r), + pq_inverse_eotf(color.g), + pq_inverse_eotf(color.b)); +} + +// Applies the inverse Perceptual Quantizer EOTF to a color channel, producing a +// non-linear value. The output will be in the range [0, 1], where 1 corresponds +// to the maximum 10,000 nits. float pq_inverse_eotf(float channel) { let c = pow(channel, PQ_M1); @@ -143,29 +151,6 @@ float pq_inverse_eotf(float channel) PQ_M2); } -// Applies a scaling function on the result of the PQ EOTF to map it to SDR -// space, based on a hypothetical maximum nits value for SDR. -public float4 scale_pq_to_sdr(float4 color, float max_nits) -{ - return float4( - scale_pq_to_sdr(color.rgb, max_nits), - color.a); -} - -// Applies a scaling function on the result of the PQ EOTF to map it to SDR -// space, based on a hypothetical maximum nits value for SDR. -public float3 scale_pq_to_sdr(float3 color, float max_nits) -{ - return color * (10000.0 / max_nits); -} - -// Applies a scaling function on the result of the PQ EOTF to map it to SDR -// space, based on a hypothetical maximum nits value for SDR. -public float scale_pq_to_sdr(float channel, float max_nits) -{ - return channel * (10000.0 / max_nits); -} - // Transform a color from one set of primaries to another. The colors must be // linear, that is, they must have already been linearized using the relevant // OETF. @@ -181,13 +166,54 @@ public float4 transform(float4 color, PrimariesTransform pa, PrimariesTransform // inverse EOTF. public float3 transform(float3 color, PrimariesTransform pa, PrimariesTransform pb) { - // XXX if it doesn't work, I got the row-major/column-major thing wrong here. - // let mat = mul(pa.to_xyz, pb.from_xyz); - // let xyz = mul(pa.to_xyz, color); - // return mul(pb.from_xyz, xyz); - let color = clamp(color, 0.0, 1.0); - let xyz = mul(pa.to_xyz, color); - return mul(pb.from_xyz, xyz); - // return mul(color, mat); - // TODO premul + let mat = mul(pb.from_xyz, pa.to_xyz); + return mul(mat, color); +} + +// Available conversions to and from YCbCr color space. +public enum YCbCrModel +{ + BT709, + BT2020, +} + +static const float3x3 YCBCR_709_MATRIX = float3x3( + 0.2126, 0.7152, 0.0722, + -0.114572, -0.385428, 0.5, + 0.5, -0.454153, -0.045847); + +static const float3x3 YCBCR_2020_MATRIX = float3x3( + 0.2627, 0.6780, 0.0593, + -0.139630, -0.360370, 0.5, + 0.5, -0.459786, -0.040214); + +// Encode a color in the YCbCr color system. The color should already be in +// nonlinear space. +public float3 encode_ycbcr(float3 color, YCbCrModel model, bool full_range) +{ + float3 ycbcr; + switch (model) + { + case YCbCrModel::BT709: + ycbcr = mul(YCBCR_709_MATRIX, color); + break; + case YCbCrModel::BT2020: + ycbcr = mul(YCBCR_2020_MATRIX, color); + break; + } + + // The matrix multiplication gives us Y in [0, 1] and Cb and Cr in [-0.5, 0.5]. + ycbcr.y += 0.5; + ycbcr.z += 0.5; + + if (!full_range) + { + // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. + ycbcr = float3( + (219.0 * ycbcr.x + 16.0) / 256.0, + (224.0 * ycbcr.y + 16.0) / 256.0, + (224.0 * ycbcr.z + 16.0) / 256.0); + } + + return clamp(ycbcr, 0.0, 1.0); } diff --git a/mm-server/src/compositor/video/composite.rs b/mm-server/src/compositor/video/composite.rs index 5b580bb..0a27437 100644 --- a/mm-server/src/compositor/video/composite.rs +++ b/mm-server/src/compositor/video/composite.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::Context; use ash::vk; use cstr::cstr; +use tracing::trace; use crate::{color::ColorSpace, vulkan::*}; @@ -18,9 +19,9 @@ pub const BLEND_FORMAT: vk::Format = vk::Format::R16G16B16A16_SFLOAT; #[repr(u32)] #[derive(Copy, Clone, Debug)] enum SurfaceColorSpace { - Srgb = 1, - LinearExtSrgb = 2, - Hdr10 = 3, + Srgb = 0, + LinearExtSrgb = 1, + Hdr10 = 2, } impl From for SurfaceColorSpace { @@ -264,6 +265,8 @@ impl CompositePipeline { SurfaceTexture::Imported { color_space, .. } => *color_space, }; + trace!(?color_space, ?dst_pos, ?dst_size, "compositing surface"); + let pc = SurfacePC { src_pos: glam::Vec2::ZERO, src_size: glam::Vec2::ONE, diff --git a/mm-server/src/compositor/video/composite.slang b/mm-server/src/compositor/video/composite.slang index 2fc5c9f..fe45ac4 100644 --- a/mm-server/src/compositor/video/composite.slang +++ b/mm-server/src/compositor/video/composite.slang @@ -15,9 +15,9 @@ struct VertOutput // This must match the enum in composite.rs. enum InputTextureColorSpace { - SRGB = 1, - LINEAR_EXTENDED_SRGB = 2, - HDR10 = 3, + SRGB = 0, + LINEAR_EXTENDED_SRGB = 1, + HDR10 = 2, } struct PushConstants @@ -59,6 +59,35 @@ VertOutput vert(uint vid: SV_VertexID) return output; } +float3 linearize(float3 color, InputTextureColorSpace color_space) +{ + switch (color_space) + { + case InputTextureColorSpace::SRGB: + return srgb_eotf(color); + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + return color; + case InputTextureColorSpace::HDR10: + float3 linear = pq_eotf(color); + + // The resulting values have the range 0-1, where 1.0 corresponds 10,000 + // nits. In order to effectively blend with SDR textures, we need to + // scale based on our virtual display brightness, producing values where + // 1.0 matches the maximum brightness that SDR content would produce. We + // use the Rec. 2408 value of 203 nits for this. On this scale, a value + // of 300 nits would result in a scaled value of about 1.47, and 1.0 + // would result in about 49.26. Either value would be clipped unless we + // use a floating-point blend format (which we do). + // TODO: allow HDR metadata to override the scaling factor. This is called + // "nominal diffuse white level" or NDWL. + linear *= PQ_MAX_WHITE / SDR_REFERENCE_WHITE; + + return transform(linear, Primaries::BT2020, Primaries::BT709); + default: + return srgb_eotf(color); + } +} + [shader("fragment")] float4 frag(float2 uv: TextureCoord) : SV_Target @@ -74,43 +103,15 @@ float4 frag(float2 uv: TextureCoord) // the texture as UNORM (even though it's stored as sRGB) and then doing the // conversion manually. if (color.a == 0) - { return float4(0); - } else if (pc.color_space == InputTextureColorSpace::LINEAR_EXTENDED_SRGB) - { // We're already in the right space for blending. return color; - } color.rgb /= color.a; - - switch (pc.color_space) - { - case InputTextureColorSpace::SRGB: - color.rgb = apply_eotf(color.rgb, TransferFunction::SRGB); - break; - case InputTextureColorSpace::HDR10: - float3 linear = apply_eotf(color.rgb, TransferFunction::PQ); - - // The resulting values have the range 0-1, where 1.0 corresponds 10,000 - // nits. In order to effectively blend with SDR textures, we need to - // scale based on our virtual display brightness, producing values where - // 1.0 matches the maximum brightness that SDR content would produce. We - // use the Rec. 2408 value of 203 nits for this. On this scale, a value - // of 300 nits would result in a scaled value of about 1.47, and 1.0 - // would result in about 49.26. Either value would be clipped unless we - // use a floating-point blend format (which we do). - // TODO: allow HDR metadata to override the scaling factor. This is called - // "nominal diffuse white level" or NDWL. - linear = scale_pq_to_sdr(linear, SDR_REFERENCE_WHITE); - - // Transform from the BT.2020 primaries to the BT.709 primaries. - color.rgb = transform(linear, Primaries::BT2020, Primaries::BT709); - break; - } - + color.rgb = linearize(color.rgb, pc.color_space); color.rgb *= color.a; + return color; } diff --git a/mm-server/src/compositor/video/convert.rs b/mm-server/src/compositor/video/convert.rs index d3148b4..6e0c1d8 100644 --- a/mm-server/src/compositor/video/convert.rs +++ b/mm-server/src/compositor/video/convert.rs @@ -7,21 +7,54 @@ use std::sync::Arc; use ash::vk; use tracing::instrument; -use crate::vulkan::*; +use crate::{ + color::{ColorSpace, VideoProfile}, + vulkan::*, +}; use super::VkPlaneView; -// GLSL requires vec4s for alignment. -const COLORSPACE_BT709: [[f32; 4]; 3] = [ - [0.2126, 0.7152, 0.0722, 0.0], - [-0.1146, -0.3854, 0.5, 0.0], - [0.5, -0.4542, -0.0458, 0.0], -]; +// Also defined in convert.slang. +#[repr(u32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum InputTextureColorSpace { + Srgb = 0, + LinearExtSrgb = 1, + Hdr10 = 2, +} + +impl From for InputTextureColorSpace { + fn from(cs: ColorSpace) -> Self { + match cs { + ColorSpace::Srgb => InputTextureColorSpace::Srgb, + ColorSpace::LinearExtSrgb => InputTextureColorSpace::LinearExtSrgb, + ColorSpace::Hdr10 => InputTextureColorSpace::Hdr10, + } + } +} + +// Also defined in convert.slang. +#[repr(u32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum OutputProfile { + Hd = 0, + Hdr10 = 1, +} + +impl From for OutputProfile { + fn from(profile: VideoProfile) -> Self { + match profile { + VideoProfile::Hd => OutputProfile::Hd, + VideoProfile::Hdr10 => OutputProfile::Hdr10, + } + } +} #[repr(C)] -#[derive(Copy, Clone, Debug)] -struct ColorspacePC { - color_space: [[f32; 4]; 3], +#[derive(Debug, Copy, Clone)] +struct ConvertPushConstants { + input_color_space: InputTextureColorSpace, + output_profile: OutputProfile, } pub struct ConvertPipeline { @@ -100,7 +133,7 @@ impl ConvertPipeline { let ranges = [vk::PushConstantRange::default() .stage_flags(vk::ShaderStageFlags::COMPUTE) .offset(0) - .size(std::mem::size_of::() as u32)]; + .size(std::mem::size_of::() as u32)]; let set_layouts = [descriptor_set_layout]; let create_info = vk::PipelineLayoutCreateInfo::default() @@ -150,6 +183,8 @@ impl ConvertPipeline { width: u32, height: u32, descriptor_set: vk::DescriptorSet, + input_color_space: ColorSpace, + video_profile: VideoProfile, ) { self.vk .device @@ -164,8 +199,9 @@ impl ConvertPipeline { &[], ); - let pc = ColorspacePC { - color_space: COLORSPACE_BT709, // TODO + let pc = ConvertPushConstants { + input_color_space: input_color_space.into(), + output_profile: video_profile.into(), }; self.vk.device.cmd_push_constants( @@ -175,7 +211,7 @@ impl ConvertPipeline { 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, - std::mem::size_of::(), + std::mem::size_of::(), ), ); diff --git a/mm-server/src/compositor/video/convert.slang b/mm-server/src/compositor/video/convert.slang index 949240d..f2167e1 100644 --- a/mm-server/src/compositor/video/convert.slang +++ b/mm-server/src/compositor/video/convert.slang @@ -15,24 +15,71 @@ const RWTexture2D chroma_u; const RWTexture2D chroma_v; #endif +// This must match the enum in convert.rs. +enum InputTextureColorSpace +{ + SRGB = 0, + LINEAR_EXTENDED_SRGB = 1, + HDR10 = 2, +} + +/// This must match the enum in convert.rs. +enum OutputProfile +{ + HD = 0, + HDR10 = 1, +} + struct PushConstants { - float3x3 color_transform; + InputTextureColorSpace input_color_space; + OutputProfile output_profile; } [[vk::push_constant]] PushConstants pc; -float3 rgb_to_ycbcr(float3 color) +float3 to_bt709(float3 rgb, InputTextureColorSpace color_space) +{ + float3 linear; + switch (color_space) + { + case InputTextureColorSpace::SRGB: + linear = srgb_eotf(rgb); + break; + case InputTextureColorSpace::HDR10: + { + // Treat 203 nits as 1.0, and clip everything above that. + linear = pq_eotf(rgb); + linear = clamp(linear * (PQ_MAX_WHITE / SDR_REFERENCE_WHITE), 0.0, 1.0); + break; + } + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + linear = clamp(rgb, 0.0, 1.0); + break; + } + + return bt709_inverse_eotf(linear); +} + +float3 to_bt2020_pq(float3 rgb, InputTextureColorSpace color_space) { - let yuv = mul(pc.color_transform, color); - - // The matrix multiplication gives us Y in [0, 1] and Cb and Cr in [-0.5, 0.5]. - // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. - return float3( - (219.0 * yuv.x + 16.0) / 256.0, - (224.0 * yuv.y + 128.0) / 256.0, - (224.0 * yuv.z + 128.0) / 256.0); + float3 bt2020_linear; + switch (color_space) + { + case InputTextureColorSpace::SRGB: + bt2020_linear = transform(srgb_eotf(rgb), Primaries::BT709, Primaries::BT2020); + break; + case InputTextureColorSpace::LINEAR_EXTENDED_SRGB: + bt2020_linear = transform(rgb, Primaries::BT709, Primaries::BT2020); + break; + case InputTextureColorSpace::HDR10: + // Happy identity path. + return rgb; + } + + // Tone-map 1.0 to 203 nits, then delinearize. + return clamp(pq_inverse_eotf(bt2020_linear * (SDR_REFERENCE_WHITE / PQ_MAX_WHITE)), 0.0, 1.0); } [shader("compute")] @@ -52,11 +99,18 @@ void main(uint2 self_id: SV_DispatchThreadID) let texel_coords = coords + uint2(j, k); float4 texel = blend_image.Load(uint3(texel_coords, 0)); - // Since we're using Rec. 709 primaries, but sampling scRGB, we need - // to clamp the values. - texel.rgb = clamp(texel.rgb, 0.0, 1.0); - - let yuv = rgb_to_ycbcr(apply_inverse_eotf(texel.rgb, TransferFunction::BT709)); + float3 yuv; + switch (pc.output_profile) + { + case OutputProfile::HD: + yuv = encode_ycbcr(to_bt709(texel.rgb, pc.input_color_space), + YCbCrModel::BT709, false); + break; + case OutputProfile::HDR10: + yuv = encode_ycbcr(to_bt2020_pq(texel.rgb, pc.input_color_space), + YCbCrModel::BT2020, false); + break; + } luminance[texel_coords] = yuv.x; diff --git a/mm-server/src/compositor/video/vulkan_encode.rs b/mm-server/src/compositor/video/vulkan_encode.rs index 764a1f0..3bbda3f 100644 --- a/mm-server/src/compositor/video/vulkan_encode.rs +++ b/mm-server/src/compositor/video/vulkan_encode.rs @@ -988,6 +988,14 @@ fn default_profile(op: vk::VideoCodecOperationFlagsKHR) -> vk::VideoProfileInfoK .luma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_8) } +fn default_hdr10_profile(op: vk::VideoCodecOperationFlagsKHR) -> vk::VideoProfileInfoKHR<'static> { + vk::VideoProfileInfoKHR::default() + .video_codec_operation(op) + .chroma_subsampling(vk::VideoChromaSubsamplingFlagsKHR::TYPE_420) + .chroma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_10) + .luma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_10) +} + fn default_encode_usage() -> vk::VideoEncodeUsageInfoKHR<'static> { vk::VideoEncodeUsageInfoKHR::default() .video_usage_hints(vk::VideoEncodeUsageFlagsKHR::STREAMING) diff --git a/mm-server/src/compositor/video/vulkan_encode/h264.rs b/mm-server/src/compositor/video/vulkan_encode/h264.rs index 316038d..c0958e0 100644 --- a/mm-server/src/compositor/video/vulkan_encode/h264.rs +++ b/mm-server/src/compositor/video/vulkan_encode/h264.rs @@ -14,6 +14,7 @@ use ash::vk::native::{ use bytes::Bytes; use tracing::trace; +use crate::color::VideoProfile; use crate::compositor::{AttachedClients, VideoStreamParams}; use crate::vulkan::*; @@ -70,16 +71,17 @@ impl H264Encoder { ) -> anyhow::Result { let (video_loader, encode_loader) = vk.video_apis.as_ref().unwrap(); - let profile_idc = 100; // HIGH + let op = vk::VideoCodecOperationFlagsKHR::ENCODE_H264_EXT; + let (profile, profile_idc) = match params.profile { + VideoProfile::Hd => (super::default_profile(op), 100), + VideoProfile::Hdr10 => (super::default_hdr10_profile(op), 110), + }; let h264_profile_info = vk::VideoEncodeH264ProfileInfoEXT::default().std_profile_idc(profile_idc); - let mut profile = H264EncodeProfile::new( - super::default_profile(vk::VideoCodecOperationFlagsKHR::ENCODE_H264_EXT), - super::default_encode_usage(), - h264_profile_info, - ); + let mut profile = + H264EncodeProfile::new(profile, super::default_encode_usage(), h264_profile_info); let mut caps = H264EncodeCapabilities::default(); @@ -95,10 +97,7 @@ impl H264Encoder { trace!("video capabilities: {:#?}", caps.video_caps); trace!("encode capabilities: {:#?}", caps.encode_caps); - trace!("h264 capabilities: {:#?}", caps.h264_caps); - - // let quality_level = caps.encode_caps.max_quality_levels - 1; - // let mut quality_props = H264QualityLevelProperties::default(); + trace!("h265 capabilities: {:#?}", caps.h264_caps); // unsafe { // let get_info = vk::PhysicalDeviceVideoEncodeQualityLevelInfoKHR::default() @@ -159,11 +158,16 @@ impl H264Encoder { trace!("crop right: {}, bottom: {}", crop_right, crop_bottom); + let (colour_primaries, transfer_characteristics, matrix_coefficients) = match params.profile + { + VideoProfile::Hd => (1, 1, 1), + VideoProfile::Hdr10 => (9, 16, 9), + }; + let mut vui = StdVideoH264SequenceParameterSetVui { - // BT.709. - colour_primaries: 1, - transfer_characteristics: 1, - matrix_coefficients: 1, + colour_primaries, + transfer_characteristics, + matrix_coefficients, // Unspecified. video_format: 5, ..unsafe { std::mem::zeroed() } @@ -179,11 +183,19 @@ impl H264Encoder { .ilog2() .saturating_sub(4) as u8; + let bit_depth = match params.profile { + VideoProfile::Hd => 8, + VideoProfile::Hdr10 => 10, + }; + let mut sps = StdVideoH264SequenceParameterSet { profile_idc, level_idc, chroma_format_idc: StdVideoH264ChromaFormatIdc_STD_VIDEO_H264_CHROMA_FORMAT_IDC_420, + bit_depth_chroma_minus8: bit_depth - 8, + bit_depth_luma_minus8: bit_depth - 8, + max_num_ref_frames: 1, pic_order_cnt_type: StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_0, log2_max_pic_order_cnt_lsb_minus4: log2_max_frame_num_minus4, diff --git a/mm-server/src/compositor/video/vulkan_encode/h265.rs b/mm-server/src/compositor/video/vulkan_encode/h265.rs index 9535573..2ef9acc 100644 --- a/mm-server/src/compositor/video/vulkan_encode/h265.rs +++ b/mm-server/src/compositor/video/vulkan_encode/h265.rs @@ -16,6 +16,7 @@ use ash::vk::native::{ use bytes::Bytes; use tracing::trace; +use crate::color::VideoProfile; use crate::compositor::{AttachedClients, VideoStreamParams}; use crate::vulkan::*; @@ -76,11 +77,14 @@ impl H265Encoder { let h265_profile_info = vk::VideoEncodeH265ProfileInfoEXT::default().std_profile_idc(profile_idc); - let mut profile = H265EncodeProfile::new( - super::default_profile(vk::VideoCodecOperationFlagsKHR::ENCODE_H265_EXT), - super::default_encode_usage(), - h265_profile_info, - ); + let op = vk::VideoCodecOperationFlagsKHR::ENCODE_H265_EXT; + let (profile, profile_idc) = match params.profile { + VideoProfile::Hd => (super::default_profile(op), 1), + VideoProfile::Hdr10 => (super::default_hdr10_profile(op), 2), + }; + + let mut profile = + H265EncodeProfile::new(profile, super::default_encode_usage(), h265_profile_info); let mut caps = H265EncodeCapabilities::default(); @@ -198,19 +202,23 @@ impl H265Encoder { trace!("crop right: {}, bottom: {}", crop_right, crop_bottom); + let (colour_primaries, transfer_characteristics, matrix_coeffs) = match params.profile { + VideoProfile::Hd => (1, 1, 1), + VideoProfile::Hdr10 => (9, 16, 9), + }; + let mut vui = StdVideoH265SequenceParameterSetVui { - // BT.709. - colour_primaries: 1, - transfer_characteristics: 1, - matrix_coeffs: 1, + colour_primaries, + transfer_characteristics, + matrix_coeffs, // Unspecified. video_format: 5, ..unsafe { std::mem::zeroed() } }; vui.flags.set_video_signal_type_present_flag(1); - vui.flags.set_video_full_range_flag(0); // Narrow range. vui.flags.set_colour_description_present_flag(1); + vui.flags.set_video_full_range_flag(0); // Narrow range. let ptl = StdVideoH265ProfileTierLevel { general_profile_idc: profile_idc, @@ -245,13 +253,18 @@ impl H265Encoder { let max_transform_hierarchy_depth = (max_ctb.ilog2() - min_tbs.ilog2()) as u8; + let bit_depth = match params.profile { + VideoProfile::Hd => 8, + VideoProfile::Hdr10 => 10, + }; + let mut sps = StdVideoH265SequenceParameterSet { chroma_format_idc: StdVideoH265ChromaFormatIdc_STD_VIDEO_H265_CHROMA_FORMAT_IDC_420, pic_width_in_luma_samples: aligned_width, pic_height_in_luma_samples: aligned_height, sps_max_sub_layers_minus1: layers_minus_1, - bit_depth_luma_minus8: 0, // TODO HDR - bit_depth_chroma_minus8: 0, // TODO HDR + bit_depth_luma_minus8: bit_depth - 8, + bit_depth_chroma_minus8: bit_depth - 8, log2_max_pic_order_cnt_lsb_minus4: 4, log2_min_luma_coding_block_size_minus3: (min_cb.ilog2() - 3) as u8, log2_diff_max_min_luma_coding_block_size: (max_cb.ilog2() - min_cb.ilog2()) as u8, diff --git a/mm-server/src/server/handlers/validation.rs b/mm-server/src/server/handlers/validation.rs index 434cd90..5c47e70 100644 --- a/mm-server/src/server/handlers/validation.rs +++ b/mm-server/src/server/handlers/validation.rs @@ -101,7 +101,7 @@ fn validate_profile(profile: i32) -> Result { }; match p.try_into() { - Ok(p) if p == VideoProfile::Hd => Ok(p), + Ok(p) => Ok(p), _ => Err(ValidationError::NotSupported( "unsupported video profile".into(), )), From e1530bee059319473de2c071e2c826a496281b90 Mon Sep 17 00:00:00 2001 From: Colin Marc Date: Tue, 30 Apr 2024 22:53:07 +0200 Subject: [PATCH 4/5] chore: move color.slang into a shared directory --- mm-client/build.rs | 1 + mm-server/build.rs | 1 + .../src/compositor/video => shader-common}/color.slang | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) rename {mm-server/src/compositor/video => shader-common}/color.slang (98%) diff --git a/mm-client/build.rs b/mm-client/build.rs index 45f50a1..2a18ce2 100644 --- a/mm-client/build.rs +++ b/mm-client/build.rs @@ -42,6 +42,7 @@ fn compile_shader( let mut compile_request = session.create_compile_request(); compile_request + .add_search_path("../shader-common") .set_codegen_target(slang::CompileTarget::Spirv) .set_optimization_level(slang::OptimizationLevel::Maximal) .set_target_profile(session.find_profile("glsl_460")); diff --git a/mm-server/build.rs b/mm-server/build.rs index 418436c..10278f5 100644 --- a/mm-server/build.rs +++ b/mm-server/build.rs @@ -71,6 +71,7 @@ fn compile_shader<'a>( let mut compile_request = session.create_compile_request(); compile_request + .add_search_path("../shader-common") .set_codegen_target(slang::CompileTarget::Spirv) .set_optimization_level(slang::OptimizationLevel::Maximal) .set_target_profile(session.find_profile("glsl_460")); diff --git a/mm-server/src/compositor/video/color.slang b/shader-common/color.slang similarity index 98% rename from mm-server/src/compositor/video/color.slang rename to shader-common/color.slang index a09b3f7..2c4c6bb 100644 --- a/mm-server/src/compositor/video/color.slang +++ b/shader-common/color.slang @@ -1,3 +1,7 @@ +// Copyright 2024 Colin Marc +// +// SPDX-License-Identifier: MIT + module color; // A set of color primaries, defined in terms of a transformation to/from XYZ @@ -207,13 +211,11 @@ public float3 encode_ycbcr(float3 color, YCbCrModel model, bool full_range) ycbcr.z += 0.5; if (!full_range) - { // This converts to "MPEG" or "Narrow" in the range [16, 235] and [16, 240]. ycbcr = float3( (219.0 * ycbcr.x + 16.0) / 256.0, (224.0 * ycbcr.y + 16.0) / 256.0, (224.0 * ycbcr.z + 16.0) / 256.0); - } return clamp(ycbcr, 0.0, 1.0); } From 5fd82bd8c847b481f6588dcd7cc10d9195101ee6 Mon Sep 17 00:00:00 2001 From: Colin Marc Date: Tue, 30 Apr 2024 22:54:05 +0200 Subject: [PATCH 5/5] feat(mmclient): support playing HDR streams --- mm-client/src/audio.rs | 1 + mm-client/src/bin/mmclient.rs | 21 ++- mm-client/src/render.rs | 145 ++++++++++++----- mm-client/src/render.slang | 87 ++++++++--- mm-client/src/video.rs | 287 +++++++++++++++++++++++----------- mm-client/src/vulkan.rs | 31 ++-- 6 files changed, 396 insertions(+), 176 deletions(-) diff --git a/mm-client/src/audio.rs b/mm-client/src/audio.rs index 599faf8..2795ab4 100644 --- a/mm-client/src/audio.rs +++ b/mm-client/src/audio.rs @@ -146,6 +146,7 @@ where if frames_remaining < frames_needed { out.fill(Default::default()); + trace!("audio buffer underrun"); return; } diff --git a/mm-client/src/bin/mmclient.rs b/mm-client/src/bin/mmclient.rs index 9721109..dd36ae1 100644 --- a/mm-client/src/bin/mmclient.rs +++ b/mm-client/src/bin/mmclient.rs @@ -96,9 +96,14 @@ struct Cli { /// resizes. #[arg(long, required = false, default_value = "auto")] resolution: Resolution, + /// Request 10-bit video output from the server. This will only work if + /// both your display and the application in question support rendering + /// HDR color. + #[arg(long, required = false)] + hdr: bool, /// The UI scale to communicate to the server. If not specified, this will /// be determined from the client-side window scale factor. - #[arg(long)] + #[arg(long, required = false)] ui_scale: Option, /// Video codec to use. #[arg(long, default_value = "h265")] @@ -121,6 +126,7 @@ struct MainLoop { struct App { configured_resolution: Resolution, configured_codec: protocol::VideoCodec, + configured_profile: protocol::VideoProfile, configured_framerate: u32, window: Arc, @@ -621,7 +627,7 @@ impl App { session_id: self.session_id, streaming_resolution: self.remote_display_params.resolution.clone(), video_codec: self.configured_codec.into(), - video_profile: protocol::VideoProfile::Hd.into(), + video_profile: self.configured_profile.into(), ..Default::default() }, None, @@ -816,6 +822,12 @@ fn main() -> Result<()> { Some(v) => bail!("invalid codec: {:?}", v), }; + let configured_profile = if args.hdr { + protocol::VideoProfile::Hdr10 + } else { + protocol::VideoProfile::Hd + }; + // TODO: anyhow errors are garbage for end-users. debug!("establishing connection to {:}", &args.host); let mut conn = Conn::new(&args.host).context("failed to establish connection")?; @@ -928,7 +940,7 @@ fn main() -> Result<()> { window.clone(), cfg!(debug_assertions), )?); - let renderer = Renderer::new(vk.clone(), window.clone())?; + let renderer = Renderer::new(vk.clone(), window.clone(), args.hdr)?; debug!("attaching session {:?}", session.session_id); let attachment_sid = conn.send( @@ -938,7 +950,7 @@ fn main() -> Result<()> { session_id: session.session_id, streaming_resolution: Some(streaming_resolution), video_codec: configured_codec.into(), - video_profile: protocol::VideoProfile::Hd.into(), + video_profile: configured_profile.into(), ..Default::default() }, None, @@ -966,6 +978,7 @@ fn main() -> Result<()> { configured_codec, configured_framerate: args.framerate, configured_resolution: args.resolution, + configured_profile, window, _proxy: proxy.clone(), diff --git a/mm-client/src/render.rs b/mm-client/src/render.rs index 8c7cedb..847c4c2 100644 --- a/mm-client/src/render.rs +++ b/mm-client/src/render.rs @@ -23,16 +23,36 @@ use crate::vulkan::*; const FONT_SIZE: f32 = 8.0; +// Matches the definition in render.slang. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +enum TextureColorSpace { + Bt709 = 0, + Bt2020Pq = 1, +} + +impl From for TextureColorSpace { + fn from(cs: crate::video::ColorSpace) -> Self { + match cs { + crate::video::ColorSpace::Bt709 => TextureColorSpace::Bt709, + crate::video::ColorSpace::Bt2020Pq => TextureColorSpace::Bt2020Pq, + } + } +} + #[derive(Copy, Clone, Debug)] #[repr(C)] struct PushConstants { aspect: glam::Vec2, + texture_color_space: TextureColorSpace, + output_color_space: vk::ColorSpaceKHR, } pub struct Renderer { width: u32, height: u32, scale_factor: f64, + hdr_mode: bool, imgui: imgui::Context, imgui_platform: imgui_winit_support::WinitPlatform, @@ -43,12 +63,18 @@ pub struct Renderer { swapchain: Option, swapchain_dirty: bool, - video_texture: Option, + new_video_texture: Option<(Arc, VideoStreamParams)>, vk: Arc, window: Arc, } +struct VideoTexture { + image: Arc, + view: vk::ImageView, + color_space: TextureColorSpace, +} + struct Swapchain { swapchain: vk::SwapchainKHR, frames: Vec, @@ -57,11 +83,13 @@ struct Swapchain { sampler_conversion: vk::SamplerYcbcrConversion, sampler: vk::Sampler, - bound_video_texture: Option<(Arc, vk::ImageView)>, + bound_video_texture: Option, + /// The normalized relationship between the output and the video texture, /// after scaling. For example, a 500x500 video texture in a 1000x500 /// swapchain would have the aspect (2.0, 1.0), as would a 250x250 texture. aspect: (f64, f64), + surface_format: vk::SurfaceFormatKHR, descriptor_set_layout: vk::DescriptorSetLayout, descriptor_pool: vk::DescriptorPool, pipeline_layout: vk::PipelineLayout, @@ -85,13 +113,12 @@ struct SwapImage { view: vk::ImageView, } -struct VideoTexture { - params: VideoStreamParams, - texture: Arc, -} - impl Renderer { - pub fn new(vk: Arc, window: Arc) -> Result { + pub fn new( + vk: Arc, + window: Arc, + hdr_mode: bool, + ) -> Result { let window_size = window.inner_size(); let scale_factor = window.scale_factor(); @@ -112,6 +139,7 @@ impl Renderer { width: window_size.width, height: window_size.height, scale_factor, + hdr_mode, window, imgui, imgui_platform, @@ -120,7 +148,7 @@ impl Renderer { imgui_time: time::Instant::now(), swapchain: None, swapchain_dirty: false, - video_texture: None, + new_video_texture: None, vk, }; @@ -134,8 +162,7 @@ impl Renderer { let start = time::Instant::now(); let device = &self.vk.device; - let surface_format = select_surface_format(self.vk.clone())?; - trace!(?surface_format, "surface format"); + let surface_format = select_surface_format(self.vk.clone(), self.hdr_mode)?; let surface_capabilities = self .vk @@ -219,12 +246,16 @@ impl Renderer { // We need to create a sampler, even if we don't have a video stream yet // and don't know what the fields should be. - let video_params = match self.video_texture.as_ref() { - Some(tex) => tex.params, - None => VideoStreamParams::default(), + let (video_texture_format, video_params) = match self.new_video_texture.as_ref() { + Some((tex, params)) => (tex.format, *params), + None => ( + vk::Format::G8_B8_R8_3PLANE_420_UNORM, + VideoStreamParams::default(), + ), }; - let sampler_conversion = sampler_conversion(device, &video_params)?; + let sampler_conversion = + create_ycbcr_sampler_conversion(device, video_texture_format, &video_params)?; let sampler = { let mut conversion_info = vk::SamplerYcbcrConversionInfo::builder() @@ -243,22 +274,26 @@ impl Renderer { unsafe { device.create_sampler(&create_info, None)? } }; - let bound_video_texture = if let Some(tex) = self.video_texture.as_ref() { + let bound_video_texture = if let Some((tex, params)) = self.new_video_texture.as_ref() { let view = create_image_view( &self.vk.device, - tex.texture.image, - tex.texture.format, + tex.image, + tex.format, Some(sampler_conversion), )?; // Increment the reference count on the texture. - Some((tex.texture.clone(), view)) + Some(VideoTexture { + image: tex.clone(), + view, + color_space: params.color_space.into(), + }) } else { None }; - let aspect = if let Some((tex, _)) = bound_video_texture.as_ref() { - calculate_aspect(self.width, self.height, tex.width, tex.height) + let aspect = if let Some(tex) = bound_video_texture.as_ref() { + calculate_aspect(self.width, self.height, tex.image.width, tex.image.height) } else { (1.0, 1.0) }; @@ -294,7 +329,7 @@ impl Renderer { let pipeline_layout = { let pc_ranges = [vk::PushConstantRange::builder() - .stage_flags(vk::ShaderStageFlags::VERTEX) + .stage_flags(vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT) .offset(0) .size(std::mem::size_of::() as u32) .build()]; @@ -435,10 +470,10 @@ impl Renderer { .unwrap(); // TODO: do the write in bind_video_texture? - if let Some((_, view)) = bound_video_texture.as_ref() { + if let Some(tex) = bound_video_texture.as_ref() { let info = vk::DescriptorImageInfo::builder() .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) - .image_view(*view); + .image_view(tex.view); let image_info = &[info.build()]; let sampler_write = vk::WriteDescriptorSet::builder() @@ -522,6 +557,7 @@ impl Renderer { sampler, bound_video_texture, aspect, + surface_format, pipeline_layout, pipeline, @@ -592,7 +628,7 @@ impl Renderer { params: VideoStreamParams, ) -> Result<()> { // TODO: no need to recreate the sampler if the params match. - self.video_texture = Some(VideoTexture { params, texture }); + self.new_video_texture = Some((texture, params)); self.swapchain_dirty = true; Ok(()) } @@ -603,7 +639,7 @@ impl Renderer { // 1.0). pub fn get_texture_aspect(&self) -> Option<(f64, f64)> { if let Some(Swapchain { - bound_video_texture: Some((_, _)), + bound_video_texture: Some(_), aspect, .. }) = self.swapchain.as_ref() @@ -738,21 +774,23 @@ impl Renderer { ); } - if self.video_texture.is_none() || swapchain.aspect != (1.0, 1.0) { + if self.new_video_texture.is_none() || swapchain.aspect != (1.0, 1.0) { // TODO Draw the background // https://www.toptal.com/designers/subtlepatterns/prism/ } // Draw the video texture. - if let Some((_texture, _)) = &swapchain.bound_video_texture { + if let Some(tex) = &swapchain.bound_video_texture { let pc = PushConstants { aspect: glam::Vec2::new(swapchain.aspect.0 as f32, swapchain.aspect.1 as f32), + texture_color_space: tex.color_space, + output_color_space: swapchain.surface_format.color_space, }; device.cmd_push_constants( frame.render_cb, swapchain.pipeline_layout, - vk::ShaderStageFlags::VERTEX, + vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT, 0, std::slice::from_raw_parts( &pc as *const _ as *const u8, @@ -899,8 +937,8 @@ impl Renderer { device.destroy_sampler(swapchain.sampler, None); device.destroy_sampler_ycbcr_conversion(swapchain.sampler_conversion, None); - if let Some((_img, view)) = swapchain.bound_video_texture.take() { - device.destroy_image_view(view, None); + if let Some(tex) = swapchain.bound_video_texture.take() { + device.destroy_image_view(tex.view, None); // We probably drop the last reference to the image here, which then // gets destroyed. } @@ -912,8 +950,11 @@ impl Renderer { } } -fn select_surface_format(vk: Arc) -> Result { - let surface_formats = unsafe { +fn select_surface_format( + vk: Arc, + hdr_mode: bool, +) -> Result { + let mut surface_formats = unsafe { vk.surface_loader .get_physical_device_surface_formats(vk.pdevice, vk.surface)? }; @@ -924,16 +965,36 @@ fn select_surface_format(vk: Arc) -> Result= 0.0031) - return 1.055 * pow(s, 1.0 / 2.4) - 0.055; - else - return s * 12.92; + if (vk_color_space == VK_COLOR_SPACE_BT709_NONLINEAR_EXT) + { + return color; + } + + let linear = bt709_eotf(color); + switch (vk_color_space) + { + case VK_COLOR_SPACE_SRGB_NONLINEAR_EXT: + return srgb_inverse_eotf(linear); + case VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT: + return linear; + // case VK_COLOR_SPACE_DISPLAY_P3_NONLINEAR_EXT: + // return srgb_inverse_eotf(transform(color, Primaries::BT709, Primaries::P3)) + case VK_COLOR_SPACE_HDR10_ST2084_EXT: + return pq_inverse_eotf(transform(linear, Primaries::BT709, Primaries::BT2020)); + default: + return srgb_inverse_eotf(linear); + } } -float bt709_linearize(float s) +float3 bt2020_pq_to_display(float3 color, int vk_color_space) { - if (s > 0.081) - return pow((s + 0.099) / 1.099, 1.0 / 0.45); - else - return s / 4.5; + if (vk_color_space == VK_COLOR_SPACE_HDR10_ST2084_EXT) + { + return color; + } + + let linear = transform(pq_eotf(color) * PQ_MAX_WHITE / SDR_REFERENCE_WHITE, Primaries::BT2020, Primaries::BT709); + switch (vk_color_space) + { + case VK_COLOR_SPACE_SRGB_NONLINEAR_EXT: + return srgb_inverse_eotf(linear); + case VK_COLOR_SPACE_BT709_NONLINEAR_EXT: + return bt709_inverse_eotf(clamp(linear, 0.0, 1.0)); + case VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT: + return linear; + // case VK_COLOR_SPACE_DISPLAY_P3_NONLINEAR_EXT: + // return srgb_inverse_eotf(transform(color, Primaries::BT2020, Primaries::P3)) + default: + return srgb_inverse_eotf(linear); + } } [shader("fragment")] float4 frag(float2 uv: TextureCoord) : SV_Target { - float4 color = texture.Sample(uv); + float4 color = clamp(texture.Sample(uv), 0.0, 1.0); // When sampling the video texture, vulkan does the matrix multiplication - // for us, but doesn't apply any transfer function. So we need to convert - // from the BT.709 transfer function to the sRGB one. - return float4( - srgb_unlinear(bt709_linearize(color.r)), - srgb_unlinear(bt709_linearize(color.g)), - srgb_unlinear(bt709_linearize(color.b)), - color.a); + // for us, but doesn't apply any transfer function, so the values are + // still nonlinear in either BT.709 or BT.2020/ST2048. + switch (pc.texture_color_space) + { + case TextureColorSpace::Bt709: + return float4(bt709_to_display(color.rgb, pc.vk_color_space), 1.0); + case TextureColorSpace::Bt2020Pq: + return float4(bt2020_pq_to_display(color.rgb, pc.vk_color_space), 1.0); + default: + return float4(0.0, 0.5, 1.0, 1.0); + } } diff --git a/mm-client/src/video.rs b/mm-client/src/video.rs index 1bc38f4..7015794 100644 --- a/mm-client/src/video.rs +++ b/mm-client/src/video.rs @@ -7,7 +7,7 @@ use std::{ time, }; -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, bail, Context}; use ash::vk; use bytes::{Bytes, BytesMut}; use ffmpeg_next as ffmpeg; @@ -31,20 +31,25 @@ pub struct FrameMetadata { pub pts: u64, } +#[derive(Debug, Clone)] struct YUVPicture { - y: Bytes, - u: Bytes, - v: Bytes, + planes: [Bytes; 3], + num_planes: usize, info: FrameMetadata, } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ColorSpace { + Bt709, + Bt2020Pq, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct VideoStreamParams { pub width: u32, pub height: u32, - pub pixel_format: ffmpeg::format::Pixel, - pub color_space: ffmpeg::color::Space, - pub color_range: ffmpeg::color::Range, + pub color_space: ColorSpace, + pub color_full_range: bool, } impl Default for VideoStreamParams { @@ -52,9 +57,8 @@ impl Default for VideoStreamParams { Self { width: 0, height: 0, - pixel_format: ffmpeg::format::Pixel::YUV420P, - color_space: ffmpeg::color::Space::BT709, - color_range: ffmpeg::color::Range::MPEG, + color_space: ColorSpace::Bt709, + color_full_range: false, } } } @@ -387,10 +391,29 @@ impl DecoderInit { match self.decoder.receive_frame(&mut frame) { Ok(()) => { self.first_frame = match frame.format() { - ffmpeg::format::Pixel::YUV420P => Some((frame, info)), ffmpeg::format::Pixel::VIDEOTOOLBOX => { + let sw_format = unsafe { + let ctx_ref = (*self.decoder.as_ptr()).hw_frames_ctx; + assert!(!ctx_ref.is_null()); + + let mut transfer_fmt_list = std::ptr::null_mut(); + if ffmpeg_sys::av_hwframe_transfer_get_formats( + ctx_ref, + ffmpeg_sys::AVHWFrameTransferDirection::AV_HWFRAME_TRANSFER_DIRECTION_FROM, + &mut transfer_fmt_list, + 0) < 0 + { + bail!("call to av_hwframe_transfer_get_formats failed"); + }; + + let transfer_formats = read_format_list(transfer_fmt_list); + assert!(!transfer_formats.is_empty()); + + transfer_formats[0] + }; + let mut sw_frame = ffmpeg::frame::Video::new( - ffmpeg::format::Pixel::YUV420P, + sw_format, self.decoder.width(), self.decoder.height(), ); @@ -409,7 +432,7 @@ impl DecoderInit { Some((sw_frame, info)) } } - f => return Err(anyhow!("unexpected stream format: {:?}", f)), + _ => Some((frame, info)), }; Ok(true) @@ -439,25 +462,31 @@ impl DecoderInit { None => return Err(anyhow!("no frames received yet")), }; - // debug_assert_eq!(self.decoder.color_space(), ffmpeg::color::Space::BT709); - - let output_format = first_frame.0.format(); - assert_eq!(output_format, ffmpeg::format::Pixel::YUV420P); - // If we're using VideoToolbox, create a "hardware" frame to use with // receive_frame. + let output_format = first_frame.0.format(); + let ((mut frame, info), mut hw_frame) = match decoder_format { - ffmpeg::format::Pixel::YUV420P => (first_frame, None), ffmpeg::format::Pixel::VIDEOTOOLBOX => { let hw_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::VIDEOTOOLBOX, width, height); (first_frame, Some(hw_frame)) } - _ => return Err(anyhow!("unexpected stream format: {:?}", decoder_format)), + _ => (first_frame, None), }; - let texture_format = vk::Format::G8_B8_R8_3PLANE_420_UNORM; + // For 10-bit textures, we need to end up in on the GPU in P010LE, + // because that's better supported. To make the copy easier, we'll use + // swscale to convert to a matching intermediate format. + let (intermediate_format, texture_format) = match output_format { + ffmpeg::format::Pixel::YUV420P => (None, vk::Format::G8_B8_R8_3PLANE_420_UNORM), + ffmpeg::format::Pixel::YUV420P10 | ffmpeg::format::Pixel::YUV420P10LE => ( + Some(ffmpeg::format::Pixel::P010LE), + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + ), + _ => return Err(anyhow!("unexpected pixel format: {:?}", output_format)), + }; debug_assert_eq!(frame.width(), width); debug_assert_eq!(frame.height(), height); @@ -472,22 +501,26 @@ impl DecoderInit { )); } - let y_stride = frame.stride(0); - let u_stride = frame.stride(1); - let v_stride = frame.stride(2); - - let y_len = y_stride * frame.plane_height(0) as usize; - let u_len = u_stride * frame.plane_height(1) as usize; - let v_len = v_stride * frame.plane_height(2) as usize; - - // This is vanishingly unlikely, and I have no idea how the pixels - // would be layed out in that case. - debug_assert_eq!(y_len % 4, 0); - - // Precalculate the offsets into the buffer for each plane. - let buffer_size = y_len + u_len + v_len; - let buffer_offsets = [0, y_len, y_len + u_len]; - let buffer_strides = [y_stride, u_stride, v_stride]; + let mut intermediate_frame = + intermediate_format.map(|fmt| ffmpeg::frame::Video::new(fmt, width, height)); + + // For the purposes of determining the size of and offsets into the + // staging buffer, we use the intermediate frame if it exists, otherwise + // the output frame. + let model_frame = intermediate_frame.as_ref().unwrap_or(&frame); + + // Precalculate the layout of the staging buffer. + let mut buffer_strides = [0; 3]; + let mut buffer_offsets = [0; 3]; + let mut buffer_size = 0; + for plane in 0..model_frame.planes() { + let stride = model_frame.stride(plane); + let len = stride * model_frame.plane_height(plane) as usize; + + buffer_strides[plane] = stride; + buffer_offsets[plane] = buffer_size; + buffer_size += len; + } let staging_buffer = create_host_buffer( &self.vk.device, @@ -496,21 +529,32 @@ impl DecoderInit { buffer_size, )?; - let color_space = match self.decoder.color_space() { - ffmpeg::color::Space::BT709 => ffmpeg::color::Space::BT709, - ffmpeg::color::Space::BT2020NCL => ffmpeg::color::Space::BT2020NCL, - cs => { - warn!("unexpected color space: {:?}", cs); - ffmpeg::color::Space::BT709 + let color_space = match ( + self.decoder.color_space(), + self.decoder.color_transfer_characteristic(), + ) { + (ffmpeg::color::Space::BT709, ffmpeg::color::TransferCharacteristic::BT709) => { + ColorSpace::Bt709 + } + (ffmpeg::color::Space::BT2020NCL, ffmpeg::color::TransferCharacteristic::SMPTE2084) => { + ColorSpace::Bt2020Pq } + ( + ffmpeg::color::Space::Unspecified, + ffmpeg::color::TransferCharacteristic::Unspecified, + ) => { + warn!("video stream has unspecified color primaries or transfer function"); + ColorSpace::Bt709 + } + (cs, ctrc) => bail!("unexpected color description: {:?} / {:?}", cs, ctrc), }; - let color_range = match self.decoder.color_range() { - ffmpeg::color::Range::MPEG => ffmpeg::color::Range::MPEG, - ffmpeg::color::Range::JPEG => ffmpeg::color::Range::JPEG, + let color_full_range = match self.decoder.color_range() { + ffmpeg::color::Range::MPEG => false, + ffmpeg::color::Range::JPEG => true, cr => { warn!("unexpected color range: {:?}", cr); - ffmpeg::color::Range::MPEG + false } }; @@ -536,7 +580,12 @@ impl DecoderInit { // Send the frame we have from before. decoded_send - .send(copy_frame(&mut frame, &mut BytesMut::new(), info)) + .send(copy_frame( + &mut frame, + intermediate_frame.as_mut(), + &mut BytesMut::new(), + info, + )) .unwrap(); // Spawn another thread that receives packets on one channel and sends @@ -580,11 +629,12 @@ impl DecoderInit { loop { match receive_frame(&mut decoder, &mut frame, hw_frame.as_mut()) { Ok(()) => { - let pic = copy_frame(&mut frame, &mut scratch, info); - - debug_assert_eq!(pic.y.len(), y_len); - debug_assert_eq!(pic.u.len(), u_len); - debug_assert_eq!(pic.v.len(), v_len); + let pic = copy_frame( + &mut frame, + intermediate_frame.as_mut(), + &mut scratch, + info, + ); let span = trace_span!("send"); let _guard = span.enter(); @@ -640,9 +690,8 @@ impl DecoderInit { let params = VideoStreamParams { width, height, - pixel_format: output_format, color_space, - color_range, + color_full_range, }; Ok((dec, video_texture, params)) @@ -715,14 +764,15 @@ impl CPUDecoder { // Copy data into the staging buffer. self.yuv_buffer_offsets .iter() - .zip([pic.y, pic.u, pic.v]) + .zip(pic.planes.iter()) + .take(pic.num_planes) .for_each(|(offset, src)| { let dst = std::slice::from_raw_parts_mut( (self.staging_buffer.access as *mut u8).add(*offset), src.len(), ); - dst.copy_from_slice(&src); + dst.copy_from_slice(src); }); // Trace the upload, including loading timestamps for the previous upload. @@ -793,6 +843,12 @@ impl CPUDecoder { // Upload from the staging buffer to the texture. { + let num_planes = match self.video_texture.format { + vk::Format::G8_B8_R8_3PLANE_420_UNORM => 3, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 => 2, + _ => unreachable!(), + }; + let regions = [ vk::ImageAspectFlags::PLANE_0, vk::ImageAspectFlags::PLANE_1, @@ -800,6 +856,7 @@ impl CPUDecoder { ] .into_iter() .enumerate() + .take(num_planes) .map(|(plane, plane_aspect_mask)| { // Vulkan considers the image width/height to be 1/2 the size // for the U and V planes. @@ -809,10 +866,21 @@ impl CPUDecoder { (self.texture_width / 2, self.texture_height / 2) }; + let texel_width = match self.video_texture.format { + vk::Format::G8_B8_R8_3PLANE_420_UNORM => 1, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 => { + if plane == 0 { + 2 + } else { + 4 + } + } + _ => unreachable!(), + }; + vk::BufferImageCopy::builder() .buffer_offset(self.yuv_buffer_offsets[plane] as u64) - // This is actually in texels, but each plane uses 1bpp. - .buffer_row_length(self.yuv_buffer_strides[plane] as u32) + .buffer_row_length((self.yuv_buffer_strides[plane] / texel_width) as u32) // In texels. .image_subresource(vk::ImageSubresourceLayers { aspect_mask: plane_aspect_mask, mip_level: 0, @@ -935,58 +1003,93 @@ fn copy_packet(pkt: &mut ffmpeg::Packet, buf: Undecoded) -> anyhow::Result<()> { #[instrument(skip_all)] fn copy_frame( frame: &mut ffmpeg::frame::Video, + intermediate_frame: Option<&mut ffmpeg::frame::Video>, scratch: &mut BytesMut, info: FrameMetadata, ) -> YUVPicture { - scratch.truncate(0); - - scratch.extend_from_slice(frame.data(0)); - let y = scratch.split().freeze(); + let transfer_src = if let Some(intermediate) = intermediate_frame { + // TODO reuse + let mut ctx = ffmpeg::software::scaling::Context::get( + frame.format(), + frame.width(), + frame.height(), + intermediate.format(), + intermediate.width(), + intermediate.height(), + ffmpeg::software::scaling::Flags::empty(), + ) + .expect("failed to create sws ctx"); + + ctx.run(frame, intermediate).expect("failed to convert"); + + intermediate + } else { + frame + }; - scratch.extend_from_slice(frame.data(1)); - let u = scratch.split().freeze(); + let mut pic = YUVPicture { + planes: [Bytes::new(), Bytes::new(), Bytes::new()], + num_planes: transfer_src.planes(), + info, + }; - scratch.extend_from_slice(frame.data(2)); - let v = scratch.split().freeze(); + scratch.truncate(0); + for plane in 0..transfer_src.planes() { + scratch.extend_from_slice(transfer_src.data(plane)); + pic.planes[plane] = scratch.split().freeze(); + } - YUVPicture { y, u, v, info } + pic } #[no_mangle] unsafe extern "C" fn get_hw_format_videotoolbox( ctx: *mut ffmpeg_sys::AVCodecContext, - mut formats: *const ffmpeg_sys::AVPixelFormat, + list: *const ffmpeg_sys::AVPixelFormat, ) -> ffmpeg_sys::AVPixelFormat { use ffmpeg_sys::AVPixelFormat::*; - while *formats != AV_PIX_FMT_NONE { - if *formats == AV_PIX_FMT_VIDEOTOOLBOX { - let frames_ctx_ref = ffmpeg_sys::av_hwframe_ctx_alloc((*ctx).hw_device_ctx); - if frames_ctx_ref.is_null() { - error!("call to av_hwframe_ctx_alloc failed"); - break; - } + let sw_pix_fmt = (*ctx).sw_pix_fmt; + let formats = read_format_list(list); - let frames_ctx = (*frames_ctx_ref).data as *mut ffmpeg_sys::AVHWFramesContext; - (*frames_ctx).width = (*ctx).width; - (*frames_ctx).height = (*ctx).height; - (*frames_ctx).format = AV_PIX_FMT_VIDEOTOOLBOX; - (*frames_ctx).sw_format = AV_PIX_FMT_YUV420P; + if formats.contains(&ffmpeg::format::Pixel::VIDEOTOOLBOX) { + let frames_ctx_ref = ffmpeg_sys::av_hwframe_ctx_alloc((*ctx).hw_device_ctx); + if frames_ctx_ref.is_null() { + error!("call to av_hwframe_ctx_alloc failed"); + return sw_pix_fmt; + } - let res = ffmpeg_sys::av_hwframe_ctx_init(frames_ctx_ref); - if res < 0 { - error!("call to av_hwframe_ctx_init failed"); - break; - } + debug!(?formats, sw_pix_fmt = ?sw_pix_fmt, "get_hw_format_videotoolbox"); - debug!("using VideoToolbox hardware encoder"); - (*ctx).hw_frames_ctx = frames_ctx_ref; - return *formats; + let frames_ctx = (*frames_ctx_ref).data as *mut ffmpeg_sys::AVHWFramesContext; + (*frames_ctx).width = (*ctx).width; + (*frames_ctx).height = (*ctx).height; + (*frames_ctx).format = AV_PIX_FMT_VIDEOTOOLBOX; + (*frames_ctx).sw_format = AV_PIX_FMT_YUV420P; + + let res = ffmpeg_sys::av_hwframe_ctx_init(frames_ctx_ref); + if res < 0 { + error!("call to av_hwframe_ctx_init failed"); + return sw_pix_fmt; } - formats = formats.add(1); + debug!("using VideoToolbox hardware encoder"); + (*ctx).hw_frames_ctx = frames_ctx_ref; + return AV_PIX_FMT_VIDEOTOOLBOX; + } + + warn!("unable to determine ffmpeg hw format"); + sw_pix_fmt +} + +unsafe fn read_format_list( + mut ptr: *const ffmpeg_sys::AVPixelFormat, +) -> Vec { + let mut formats = Vec::new(); + while !ptr.is_null() && *ptr != ffmpeg_sys::AVPixelFormat::AV_PIX_FMT_NONE { + formats.push((*ptr).into()); + ptr = ptr.add(1); } - warn!("VideoToolbox setup failed, falling back to CPU decoder"); - AV_PIX_FMT_YUV420P + formats } diff --git a/mm-client/src/vulkan.rs b/mm-client/src/vulkan.rs index 417e785..32a734c 100644 --- a/mm-client/src/vulkan.rs +++ b/mm-client/src/vulkan.rs @@ -21,10 +21,11 @@ use ash::{ vk, }; use cstr::cstr; -use ffmpeg_next as ffmpeg; use raw_window_handle::{HasRawDisplayHandle, HasRawWindowHandle}; use tracing::{debug, error, info, warn}; +use crate::video::ColorSpace; + pub struct VkDebugContext { debug: DebugUtils, messenger: vk::DebugUtilsMessengerEXT, @@ -957,34 +958,24 @@ pub fn load_shader(device: &ash::Device, bytes: &[u8]) -> anyhow::Result anyhow::Result { let ycbcr_model = match params.color_space { - ffmpeg::color::Space::BT709 => vk::SamplerYcbcrModelConversion::YCBCR_709, - ffmpeg::color::Space::BT2020NCL => vk::SamplerYcbcrModelConversion::YCBCR_2020, - _ => return Err(anyhow!("unsupported color space: {:?}", params.color_space)), - }; - - let ycbcr_range = match params.color_range { - ffmpeg::color::Range::MPEG => vk::SamplerYcbcrRange::ITU_NARROW, - ffmpeg::color::Range::JPEG => vk::SamplerYcbcrRange::ITU_FULL, - _ => return Err(anyhow!("unsupported color range: {:?}", params.color_range)), + ColorSpace::Bt709 => vk::SamplerYcbcrModelConversion::YCBCR_709, + ColorSpace::Bt2020Pq => vk::SamplerYcbcrModelConversion::YCBCR_2020, }; - let texture_format = match params.pixel_format { - ffmpeg::format::Pixel::YUV420P => vk::Format::G8_B8_R8_3PLANE_420_UNORM, - _ => { - return Err(anyhow!( - "unsupported pixel format: {:?}", - params.pixel_format - )) - } + let ycbcr_range = if params.color_full_range { + vk::SamplerYcbcrRange::ITU_FULL + } else { + vk::SamplerYcbcrRange::ITU_NARROW }; let create_info = vk::SamplerYcbcrConversionCreateInfo::builder() - .format(texture_format) + .format(format) .ycbcr_model(ycbcr_model) .ycbcr_range(ycbcr_range) .chroma_filter(vk::Filter::LINEAR)