feat(client): ✨ Add reprojection support for C API (alvr-org#2543)

* feat(client): ✨ Add reprojection support for C API

* Address comment

Author: zmerp

alvr/client_core/resources/lobby_quad.wgsl

@@ -1,26 +1,23 @@
-struct VertexOutput {
-    @builtin(position) position: vec4f,
-    @location(0) uv: vec2f,
-}
-
-@group(0) @binding(0) var hud_texture: texture_2d<f32>;
-@group(0) @binding(1) var hud_sampler: sampler;
-
 struct PushConstant {
     transform: mat4x4f,
     object_type: u32,
     floor_side: f32,
 }
 var<push_constant> pc: PushConstant;
 
+@group(0) @binding(0) var hud_texture: texture_2d<f32>;
+@group(0) @binding(1) var hud_sampler: sampler;
+
+struct VertexOutput {
+    @builtin(position) position: vec4f,
+    @location(0) uv: vec2f,
+}
+
 @vertex
 fn vertex_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput {
     var result: VertexOutput;
 
-    let norm_vert_a = f32(vertex_index & 1);
-    let norm_vert_b = f32(vertex_index >> 1);
-
-    result.uv = vec2f(norm_vert_a, norm_vert_b);
+    result.uv = vec2f(f32(vertex_index & 1), f32(vertex_index >> 1));
     result.position = pc.transform * vec4f(result.uv.x - 0.5, 0.5 - result.uv.y, 0.0, 1.0);
 
     return result;

alvr/client_core/resources/stream.wgsl

@@ -42,23 +42,26 @@ override C_RIGHT_Y: f32 = 0.;
 
 override COLOR_ALPHA: f32 = 1.0;
 
-struct VertexOutput {
-    @builtin(position) position: vec4f,
-    @location(0) uv: vec2f,
+struct PushConstant {
+    reprojection_transform: mat4x4f,
+    view_idx: u32,
 }
+var<push_constant> pc: PushConstant;
 
 @group(0) @binding(0) var stream_texture: texture_2d<f32>;
 @group(0) @binding(1) var stream_sampler: sampler;
 
-var<push_constant> view_idx: u32;
+struct VertexOutput {
+    @builtin(position) position: vec4f,
+    @location(0) uv: vec2f,
+}
 
 @vertex
 fn vertex_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput {
     var result: VertexOutput;
 
-    let screen_uv = vec2f(f32(vertex_index & 1), f32(vertex_index >> 1));
-    result.position = vec4f((screen_uv - 0.5) * 2.0, 0.0, 1.0);
-    result.uv = vec2f(screen_uv.x, screen_uv.y);
+    result.uv = vec2f(f32(vertex_index & 1), f32(vertex_index >> 1));
+    result.position = pc.reprojection_transform * vec4f(result.uv.x - 0.5, 0.5 - result.uv.y, 0.0, 1.0);
 
     return result;
 }
@@ -82,7 +85,7 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {
         let b_right = vec2f(B_RIGHT_X, B_RIGHT_Y);
         let c_right = vec2f(C_RIGHT_X, C_RIGHT_Y);
 
-        if view_idx == 1 {
+        if pc.view_idx == 1 {
             corrected_uv.x = 1.0 - corrected_uv.x;
         }
 
@@ -108,7 +111,7 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {
 
         corrected_uv = corrected_uv * view_size_ratio;
 
-        if view_idx == 1 {
+        if pc.view_idx == 1 {
             corrected_uv.x = 1.0 - corrected_uv.x;
         }
     }

alvr/client_core/src/c_api.rs

@@ -1,5 +1,5 @@
 use crate::{
-    graphics::{GraphicsContext, LobbyRenderer, RenderViewInput, StreamRenderer},
+    graphics::{GraphicsContext, LobbyRenderer, LobbyViewParams, StreamRenderer, StreamViewParams},
     storage,
     video_decoder::{self, VideoDecoderConfig, VideoDecoderSource},
     ClientCapabilities, ClientCoreContext, ClientCoreEvent,
@@ -159,8 +159,8 @@ pub fn to_capi_pose(pose: Pose) -> AlvrPose {
 
 #[repr(C)]
 pub struct AlvrViewParams {
-    pub pose: AlvrPose,
-    pub fov: AlvrFov,
+    pose: AlvrPose,
+    fov: AlvrFov,
 }
 
 #[repr(C)]
@@ -675,25 +675,32 @@ thread_local! {
 }
 
 #[repr(C)]
-pub struct AlvrViewInput {
+pub struct AlvrLobbyViewParams {
+    swapchain_index: u32,
     pose: AlvrPose,
     fov: AlvrFov,
+}
+
+#[repr(C)]
+pub struct AlvrStreamViewParams {
     swapchain_index: u32,
+    reprojection_rotation: AlvrQuat,
+    fov: AlvrFov,
 }
 
 #[repr(C)]
 pub struct AlvrStreamConfig {
-    pub view_resolution_width: u32,
-    pub view_resolution_height: u32,
-    pub swapchain_textures: *mut *const u32,
-    pub swapchain_length: u32,
-    pub enable_foveation: bool,
-    pub foveation_center_size_x: f32,
-    pub foveation_center_size_y: f32,
-    pub foveation_center_shift_x: f32,
-    pub foveation_center_shift_y: f32,
-    pub foveation_edge_ratio_x: f32,
-    pub foveation_edge_ratio_y: f32,
+    view_resolution_width: u32,
+    view_resolution_height: u32,
+    swapchain_textures: *mut *const u32,
+    swapchain_length: u32,
+    enable_foveation: bool,
+    foveation_center_size_x: f32,
+    foveation_center_size_y: f32,
+    foveation_center_shift_x: f32,
+    foveation_center_shift_y: f32,
+    foveation_edge_ratio_x: f32,
+    foveation_edge_ratio_y: f32,
 }
 
 #[no_mangle]
@@ -788,19 +795,19 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
 // todo: support hands
 #[no_mangle]
 pub unsafe extern "C" fn alvr_render_lobby_opengl(
-    view_inputs: *const AlvrViewInput,
+    view_inputs: *const AlvrLobbyViewParams,
     render_background: bool,
 ) {
     let view_inputs = [
-        RenderViewInput {
+        LobbyViewParams {
+            swapchain_index: (*view_inputs).swapchain_index,
             pose: from_capi_pose((*view_inputs).pose),
             fov: from_capi_fov((*view_inputs).fov),
-            swapchain_index: (*view_inputs).swapchain_index,
         },
-        RenderViewInput {
+        LobbyViewParams {
+            swapchain_index: (*view_inputs.offset(1)).swapchain_index,
             pose: from_capi_pose((*view_inputs.offset(1)).pose),
             fov: from_capi_fov((*view_inputs.offset(1)).fov),
-            swapchain_index: (*view_inputs.offset(1)).swapchain_index,
         },
     ];
 
@@ -816,16 +823,30 @@ pub unsafe extern "C" fn alvr_render_lobby_opengl(
     });
 }
 
+/// view_params: array of 2
 #[no_mangle]
 pub unsafe extern "C" fn alvr_render_stream_opengl(
     hardware_buffer: *mut c_void,
-    swapchain_indices: *const u32,
+    view_params: *const AlvrStreamViewParams,
 ) {
     STREAM_RENDERER.with_borrow(|renderer| {
         if let Some(renderer) = renderer {
+            let left_params = &*view_params;
+            let right_params = &*view_params.offset(1);
             renderer.render(
                 hardware_buffer,
-                [*swapchain_indices, *swapchain_indices.offset(1)],
+                [
+                    StreamViewParams {
+                        swapchain_index: left_params.swapchain_index,
+                        reprojection_rotation: from_capi_quat(left_params.reprojection_rotation),
+                        fov: from_capi_fov(left_params.fov),
+                    },
+                    StreamViewParams {
+                        swapchain_index: right_params.swapchain_index,
+                        reprojection_rotation: from_capi_quat(right_params.reprojection_rotation),
+                        fov: from_capi_fov(right_params.fov),
+                    },
+                ],
             );
         }
     });
@@ -967,3 +988,8 @@ pub extern "C" fn alvr_get_frame(
         false
     }
 }
+
+#[no_mangle]
+pub extern "C" fn alvr_rotation_delta(source: AlvrQuat, destination: AlvrQuat) -> AlvrQuat {
+    to_capi_quat(from_capi_quat(source).inverse() * from_capi_quat(destination))
+}

alvr/client_core/src/graphics/lobby.rs

@@ -1,6 +1,6 @@
 use super::{GraphicsContext, SDR_FORMAT};
 use alvr_common::{
-    glam::{IVec2, Mat4, Quat, UVec2, Vec3, Vec4},
+    glam::{IVec2, Mat4, Quat, UVec2, Vec3},
     Fov, Pose,
 };
 use glyph_brush_layout::{
@@ -103,29 +103,6 @@ const BODY_SKELETON_BONES_FB: [(usize, usize); 30] = [
     (82, 83),
 ];
 
-fn projection_from_fov(fov: Fov) -> Mat4 {
-    const NEAR: f32 = 0.1;
-
-    let tanl = f32::tan(fov.left);
-    let tanr = f32::tan(fov.right);
-    let tanu = f32::tan(fov.up);
-    let tand = f32::tan(fov.down);
-    let a = 2.0 / (tanr - tanl);
-    let b = 2.0 / (tanu - tand);
-    let c = (tanr + tanl) / (tanr - tanl);
-    let d = (tanu + tand) / (tanu - tand);
-
-    // note: for wgpu compatibility, the b and d components should be flipped. Maybe a bug in the
-    // viewport handling in wgpu?
-    Mat4::from_cols(
-        Vec4::new(a, 0.0, c, 0.0),
-        Vec4::new(0.0, -b, -d, 0.0),
-        Vec4::new(0.0, 0.0, -1.0, -NEAR),
-        Vec4::new(0.0, 0.0, -1.0, 0.0),
-    )
-    .transpose()
-}
-
 fn create_pipeline(
     device: &Device,
     label: &str,
@@ -183,10 +160,10 @@ fn create_pipeline(
     })
 }
 
-pub struct RenderViewInput {
+pub struct LobbyViewParams {
+    pub swapchain_index: u32,
     pub pose: Pose,
     pub fov: Fov,
-    pub swapchain_index: u32,
 }
 
 pub struct LobbyRenderer {
@@ -374,7 +351,7 @@ impl LobbyRenderer {
 
     pub fn render(
         &self,
-        view_inputs: [RenderViewInput; 2],
+        view_params: [LobbyViewParams; 2],
         hand_data: [(Option<Pose>, Option<[Pose; 26]>); 2],
         body_skeleton_fb: Option<Vec<Option<Pose>>>,
         render_background: bool,
@@ -386,13 +363,13 @@ impl LobbyRenderer {
                 label: Some("lobby_command_encoder"),
             });
 
-        for (view_idx, view_input) in view_inputs.iter().enumerate() {
+        for (view_idx, view_input) in view_params.iter().enumerate() {
             let view = Mat4::from_rotation_translation(
                 view_input.pose.orientation,
                 view_input.pose.position,
             )
             .inverse();
-            let view_proj = projection_from_fov(view_input.fov) * view;
+            let view_proj = super::projection_from_fov(view_input.fov) * view;
 
             let clear_color = if render_background {
                 Color {

alvr/client_core/src/graphics/mod.rs

@@ -5,7 +5,10 @@ mod stream;
 pub use lobby::*;
 pub use stream::*;
 
-use alvr_common::glam::UVec2;
+use alvr_common::{
+    glam::{Mat4, UVec2, Vec4},
+    Fov,
+};
 use glow::{self as gl, HasContext};
 use khronos_egl as egl;
 use std::{ffi::c_void, mem, num::NonZeroU32, ptr};
@@ -50,6 +53,29 @@ macro_rules! ck {
 }
 pub(crate) use ck;
 
+fn projection_from_fov(fov: Fov) -> Mat4 {
+    const NEAR: f32 = 0.1;
+
+    let tanl = f32::tan(fov.left);
+    let tanr = f32::tan(fov.right);
+    let tanu = f32::tan(fov.up);
+    let tand = f32::tan(fov.down);
+    let a = 2.0 / (tanr - tanl);
+    let b = 2.0 / (tanu - tand);
+    let c = (tanr + tanl) / (tanr - tanl);
+    let d = (tanu + tand) / (tanu - tand);
+
+    // note: for wgpu compatibility, the b and d components should be flipped. Maybe a bug in the
+    // viewport handling in wgpu?
+    Mat4::from_cols(
+        Vec4::new(a, 0.0, c, 0.0),
+        Vec4::new(0.0, -b, -d, 0.0),
+        Vec4::new(0.0, 0.0, -1.0, -NEAR),
+        Vec4::new(0.0, 0.0, -1.0, 0.0),
+    )
+    .transpose()
+}
+
 pub fn choose_swapchain_format(supported_formats: &[u32], enable_hdr: bool) -> u32 {
     // Priority-sorted list of swapchain formats we'll accept--
     let mut app_supported_swapchain_formats = vec![gl::SRGB8_ALPHA8, gl::RGBA8];