Merge pull request o3de#608 from galibzon/OpenXRFixes

This PR improves XrSpaces location calculation.

Previously, XrSpaces were being located each frame
during OpenXRVkInput::PollActions() which was called
before OpenXRVk::Device::BeginInternal(). The problem
with that approach is that xrLocateSpace needs
 a "predicted display time" for the CURRENT frame.
But the "predicted display time" is calculated during
  OpenXRVk::Device::BeginInternal(), this means that all XrPoses
  for the CURRENT frame were calculated with the "predicted
display time" of the PREVIOUS frame. This was causing the rendered
frames to look jittery.

In this PR, the XrSpaces are now located during
 OpenXRVk::Device::BeginInternal() this guarantees that
 when xrLocateSpace is called it is using the correct
 "predicted display time" for the CURRENT frame. This improves
 the jitter because the camera View Srg transforms now feed the correct
 transform value for each frame improving the stability of the rendered
 views.

Another improvement is that the Base Space used in xrLocateSpace
is the LOCAL space instead of the hard coded VIEW space. Also APIS were
added to change the Base Space for both Visualized spaces and Controller (Joysticks)
spaces.

Also the engine is now notified via AZ::RPI::XRSpaceNotificationBus::OnXRSpaceLocationsChanged
at the right time so the engine can update the ViewSrg for each eye
at the correct time.

Another improvement is that the AZ::RPI::PoseData reported to the engine
is now given according to the O3DE convention: X+ Right, Y+ forward, Z+ Up.
Also added equivalent APIs that report the data as AZ::Transform.

Fixed bug when the Xr Poses would be corrupted
if the Proximity Sensor was enabled.

Now the XrSpaces are reset each time a XR_SESSION_STATE_READY
event is received (typically because the Proximity Sensor is disabled)

Author: galibzon

Gems/OpenXRVk/Code/Include/OpenXRVk/OpenXRVkDevice.h

@@ -79,6 +79,5 @@ namespace OpenXRVk
         XrCompositionLayerProjection m_xrLayer{ XR_TYPE_COMPOSITION_LAYER_PROJECTION };
         AZStd::vector<XrCompositionLayerProjectionView> m_projectionLayerViews;
         AZStd::vector<XrView> m_views;
-        uint32_t m_viewCountOutput = 0;
     };
 }

Gems/OpenXRVk/Code/Include/OpenXRVk/OpenXRVkInput.h

@@ -16,6 +16,8 @@
 
 namespace OpenXRVk
 {
+    class Device;
+
     // Class that will help manage XrActionSet/XrAction
     class Input final
         : public XR::Input
@@ -24,10 +26,25 @@ namespace OpenXRVk
         AZ_CLASS_ALLOCATOR(Input, AZ::SystemAllocator);
         AZ_RTTI(Input, "{97ADD1FE-27DF-4F36-9F61-683F881F9477}", XR::Input);
 
+        static constexpr char LogName[] = "OpenXRVkInput";
+
         static XR::Ptr<Input> Create();
 
-        //! Sync all the actions and update controller
-        //! as well as various tracked space poses
+        //! Called by the session when the predicted display time has been updated (typically
+        //! the device updates the predicted display time during BeginFrame).
+        //! \param[in] device The device that emitted this event.
+        //! \param[in] predictedTime The predicted display time for the current frame.
+        //! \param[out] xrViews Vector where each Eye Pose will be stored. Eye poses are always relative to the VIEW space.
+        //!                     The VIEW pose typically represents the pose of the Head (The Head is typically centered
+        //!                     between both eyes). Subscript 0 is the left eye, while subscript 1 is the right eye.                   
+        //! Returns true if the number of Eye Poses matches the size of @xrViews.
+        bool UpdateXrSpaceLocations(const OpenXRVk::Device& device, XrTime predictedDisplayTime, AZStd::vector<XrView>& xrViews);
+
+        //! Sync all the actions and update controller.
+        //! REMARK: XrPoses are not updated in this function. Instead, poses are updated upon UpdateXrSpaceLocations().
+        //! Why? Because PollActions() is called on the main thread outside of the BeginFrame()/EndFrame() loop.
+        //! This means the if Poses are updated during PollActions(), those poses would be using the predicted display time
+        //! of the previous frame instead of the current frame.
         void PollActions();
 
         //! Initialize various actions/actions sets and add support for Oculus touch bindings
@@ -39,20 +56,30 @@ namespace OpenXRVk
         //! Attach action sets
         AZ::RHI::ResultCode InitializeActionSets(XrSession xrSession) const;
 
-        //! Update Controller space information
+        //! Updates a Controller/Joystick pose.
         void LocateControllerSpace(XrTime predictedDisplayTime, XrSpace baseSpace, AZ::u32 handIndex);
 
-        //! Update information for a specific tracked space type (i.e visualizedSpaceType)
+        //! Update pose information for the view. 
         void LocateVisualizedSpace(XrTime predictedDisplayTime, XrSpace space, XrSpace baseSpace, OpenXRVk::SpaceType visualizedSpaceType);
 
         //! Return Pose data for a controller attached to a hand index
-        AZ::RHI::ResultCode GetControllerPose(AZ::u32 handIndex, AZ::RPI::PoseData& outPoseData) const;
+        //! By default the pose data is converted per O3DE convention: Xright, Yfront, Zup.
+        //! You can read the raw XR Pose data by setting @convertToO3de to false (Not recommended, but useful for debugging).
+        AZ::RHI::ResultCode GetControllerPose(AZ::u32 handIndex, AZ::RPI::PoseData& outPoseData, bool convertToO3de = true) const;
+
+        //! Same as above but returns the pose data as an AZ::Transform. The AZ::Transform also includes the controller scale.
+        AZ::RHI::ResultCode GetControllerTransform(AZ::u32 handIndex, AZ::Transform& outTransform, bool convertToO3de = true) const;
 
-        //! Return scale for a controller attached to a hand index
+        //! Returns scale for a controller attached to a hand index
         float GetControllerScale(AZ::u32 handIndex) const;
 
-        //! Return Pose data for a tracked space type (i.e visualizedSpaceType)
-        AZ::RHI::ResultCode GetVisualizedSpacePose(OpenXRVk::SpaceType visualizedSpaceType, AZ::RPI::PoseData& outPoseData) const;
+        //! Return Pose data for a tracked space type (i.e visualizedSpaceType).
+        //! By default the pose data is converted per O3DE convention: Xright, Yfront, Zup.
+        //! You can read the raw XR Pose data by setting @convertToO3de to false (Not recommended, but useful for debugging).
+        AZ::RHI::ResultCode GetVisualizedSpacePose(OpenXRVk::SpaceType visualizedSpaceType, AZ::RPI::PoseData& outPoseData, bool convertToO3de = true) const;
+
+        //! Same as above but returns the pose data as an AZ::Transform
+        AZ::RHI::ResultCode GetVisualizedSpaceTransform(OpenXRVk::SpaceType visualizedSpaceType, AZ::Transform& outTransform, bool convertToO3de = true) const;
 
         //! Get the Squeeze action
         XrAction GetSqueezeAction(AZ::u32 handIndex) const;
@@ -107,6 +134,9 @@ namespace OpenXRVk
         //! Destroy native objects
         void ShutdownInternal() override;
 
+        //! Returns true if the number of Eye Poses matches the size of @xrViews.
+        bool LocateEyeViews(XrTime predictedDisplayTime, AZStd::vector<XrView>& xrViews);
+
         XrActionSet m_actionSet{ XR_NULL_HANDLE };
 
         XrAction m_hapticAction{};

Gems/OpenXRVk/Code/Include/OpenXRVk/OpenXRVkSession.h

@@ -40,6 +40,21 @@ namespace OpenXRVk
         //! Return the Xrspace related to the SpaceType enum
         XrSpace GetXrSpace(SpaceType spaceType) const;
 
+        ////////////////////////////////////////////////////////////////////////////////////////////
+        //! Called by a Device when the predicted display time has been updated (typically
+        //! the device updates the predicted display time during BeginFrame).
+        //! See OpenXRVkInput.h UpdateXrSpaceLocations(...) for more details.
+        void UpdateXrSpaceLocations(const OpenXRVk::Device& device, XrTime predictedDisplayTime, AZStd::vector<XrView>& xrViews);
+
+        //! Setters and Getters for the base spaces that will be used
+        //! when calling xrLocateSpace().
+        //! By default, the base space for visualization is SpaceType::Local
+        //! and the base space for Joysticks/controllers is SpaceType::View (aka the Head)
+        void SetBaseSpaceTypeForVisualization(SpaceType spaceType);
+        void SetBaseSpaceTypeForControllers(SpaceType spaceType);
+        SpaceType GetBaseSpaceTypeForVisualization() const;
+        SpaceType GetBaseSpaceTypeForControllers() const;
+
         //////////////////////////////////////////////////////////////////////////
         // XR::Session overrides
         AZ::RHI::ResultCode InitInternal(AZ::RHI::XRSessionDescriptor* descriptor) override;
@@ -50,6 +65,7 @@ namespace OpenXRVk
         void PollEvents() override;
         void LocateControllerSpace(AZ::u32 handIndex) override;
         AZ::RHI::ResultCode GetControllerPose(AZ::u32 handIndex, AZ::RPI::PoseData& outPoseData) const override;
+        AZ::RHI::ResultCode GetControllerTransform(AZ::u32 handIndex, AZ::Transform& outTransform) const override;
         AZ::RHI::ResultCode GetControllerStagePose(AZ::u32 handIndex, AZ::RPI::PoseData& outPoseData) const override;
         AZ::RHI::ResultCode GetViewFrontPose(AZ::RPI::PoseData& outPoseData) const override;
         AZ::RHI::ResultCode GetViewLocalPose(AZ::RPI::PoseData& outPoseData) const override;
@@ -68,12 +84,25 @@ namespace OpenXRVk
         void ShutdownInternal() override;
         void LogActionSourceName(XrAction action, const AZStd::string_view actionName) const;
         Input* GetNativeInput() const;
+        // Spaces are reset every time the proximity sensor turns off, or the user wears the headset
+        // when the proximity sensor is ON.
+        void ResetSpaces();
 
         XrSession m_session = XR_NULL_HANDLE;
         XrSessionState m_sessionState = XR_SESSION_STATE_UNKNOWN;
         XrEventDataBuffer m_eventDataBuffer;
         XrInstance m_xrInstance = XR_NULL_HANDLE;
         XrGraphicsBindingVulkan2KHR m_graphicsBinding{ XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR };
+        
+        // Application defined base space that will used to calculate
+        // the relative pose of all other spaces.
+        // Typically SpaceType::Local or SpaceType::Stage.
+        SpaceType m_baseSpaceTypeForVisualization = SpaceType::Local;
+
+        // Application defined base space that will use to calculate
+        // the relative pose of the joysticks (aka XR Controllers).
+        // Typically SpaceType::View, but could be SpaceType::Local or SpaceType::Stage.
+        SpaceType m_baseSpaceTypeForControllers = SpaceType::View;
 
         bool m_sessionRunning = false;
         bool m_exitRenderLoop = false;

Gems/OpenXRVk/Code/Include/OpenXRVk/OpenXRVkUtils.h

@@ -8,9 +8,11 @@
 
 #pragma once
 
+#include <AzCore/Math/Transform.h>
 #include <OpenXRVk_Platform.h>
 #include <XR/XRBase.h>
 
+
 // Macro to generate stringify functions for OpenXR enumerations based data provided in openxr_reflection.h
 #define ENUM_CASE_STR(name, val) case name: return #name;
 #define MAKE_XR_TO_STRING_FUNC(enumType)                  \
@@ -63,4 +65,10 @@ namespace OpenXRVk
     //! Iterate through the characters while caching the starting pointer to a string
     //! and every time ' ' is encountered replace it with '\0' to indicate the end of a string.
     AZStd::vector<const char*> ParseExtensionString(char* names);
+
+    AZ::Quaternion AzQuaternionFromXrPose(const XrPosef& pose, bool convertCoordinates = true);
+    AZ::Vector3 AzPositionFromXrPose(const XrPosef& pose, bool convertCoordinates = true);
+    AZ::Transform AzTransformFromXrPose(const XrPosef& pose, bool convertCoordinates = true);
+    XrPosef XrPoseFromAzTransform(const AZ::Transform& tm, bool convertCoordinates = true);
+
 }

Gems/OpenXRVk/Code/Source/OpenXRVkDevice.cpp

@@ -52,6 +52,11 @@ namespace OpenXRVk
         {
             WARN_IF_UNSUCCESSFUL(result);
         }
+
+        // Notify the input system that we have a new predicted display time.
+        // The new predicted display time will be used to calculate XrPoses for the current frame.
+        session->UpdateXrSpaceLocations(*this, m_frameState.predictedDisplayTime, m_views);
+
         //Always return true as we want EndFrame to always be called. 
         return true;
     }
@@ -112,36 +117,10 @@ namespace OpenXRVk
     {
         XR::SwapChain::View* baseSwapChainView = baseSwapChain->GetView(viewIndex);
         SwapChain::View* swapChainView = static_cast<SwapChain::View*>(baseSwapChainView);
-        Space* xrSpace = static_cast<Space*>(GetSession()->GetSpace());
-        Instance* instance = static_cast<Instance*>(GetDescriptor().m_instance.get());
-        Session* session = static_cast<Session*>(GetSession().get());
-        XrSession xrSession = session->GetXrSession();
         XrSwapchain swapChainHandle = swapChainView->GetSwapChainHandle();
 
-        XrViewState viewState{ XR_TYPE_VIEW_STATE };
-        uint32_t viewCapacityInput = aznumeric_cast<uint32_t>(m_views.size());
-
-        XrViewLocateInfo viewLocateInfo{ XR_TYPE_VIEW_LOCATE_INFO };
-        viewLocateInfo.viewConfigurationType = instance->GetViewConfigType(); 
-        viewLocateInfo.displayTime = m_frameState.predictedDisplayTime;
-        viewLocateInfo.space = xrSpace->GetXrSpace(OpenXRVk::SpaceType::View);
-
-        XrResult result = xrLocateViews(xrSession, &viewLocateInfo, &viewState, viewCapacityInput, &m_viewCountOutput, m_views.data());
-        ASSERT_IF_UNSUCCESSFUL(result);
-        
-        if ((viewState.viewStateFlags & XR_VIEW_STATE_POSITION_VALID_BIT) == 0 ||
-            (viewState.viewStateFlags & XR_VIEW_STATE_ORIENTATION_VALID_BIT) == 0)
-        {
-            //There is no valid tracking poses for the views
-            return false;
-        }
-
-        AZ_Assert(m_viewCountOutput == viewCapacityInput, "Size mismatch between xrLocateViews %i and xrEnumerateViewConfigurationViews %i", m_viewCountOutput, viewCapacityInput);
-        AZ_Assert(m_viewCountOutput == static_cast<SwapChain*>(baseSwapChain)->GetViewConfigs().size(), "Size mismatch between xrLocateViews %i and xrEnumerateViewConfigurationViews %i", m_viewCountOutput, static_cast<SwapChain*>(baseSwapChain)->GetViewConfigs().size());
-
-        m_projectionLayerViews.resize(m_viewCountOutput);
         XrSwapchainImageAcquireInfo acquireInfo{ XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO };
-        result = xrAcquireSwapchainImage(swapChainHandle, &acquireInfo, &baseSwapChainView->m_activeImageIndex);
+        auto result = xrAcquireSwapchainImage(swapChainHandle, &acquireInfo, &baseSwapChainView->m_activeImageIndex);
         baseSwapChainView->m_isImageAcquired = (result == XR_SUCCESS);
         WARN_IF_UNSUCCESSFUL(result);
 
@@ -150,6 +129,9 @@ namespace OpenXRVk
         result = xrWaitSwapchainImage(swapChainHandle, &waitInfo);
         ASSERT_IF_UNSUCCESSFUL(result);
 
+        // REMARK: The data in m_views was updated during BeginFrameInternal(), which
+        // calls session->UpdateXrSpaceLocations(...).
+        m_projectionLayerViews.resize(m_views.size());
         m_projectionLayerViews[viewIndex] = { XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW };
         m_projectionLayerViews[viewIndex].pose = m_views[viewIndex].pose;
         m_projectionLayerViews[viewIndex].fov = m_views[viewIndex].fov;
@@ -189,30 +171,23 @@ namespace OpenXRVk
 
     AZ::RHI::ResultCode Device::GetViewFov(AZ::u32 viewIndex, AZ::RPI::FovData& outFovData) const
     {
-        if(viewIndex < m_projectionLayerViews.size())
+        if(viewIndex < m_views.size())
         { 
-            outFovData.m_angleLeft = m_projectionLayerViews[viewIndex].fov.angleLeft;
-            outFovData.m_angleRight = m_projectionLayerViews[viewIndex].fov.angleRight;
-            outFovData.m_angleUp = m_projectionLayerViews[viewIndex].fov.angleUp;
-            outFovData.m_angleDown = m_projectionLayerViews[viewIndex].fov.angleDown;
+            outFovData.m_angleLeft = m_views[viewIndex].fov.angleLeft;
+            outFovData.m_angleRight = m_views[viewIndex].fov.angleRight;
+            outFovData.m_angleUp = m_views[viewIndex].fov.angleUp;
+            outFovData.m_angleDown = m_views[viewIndex].fov.angleDown;
             return AZ::RHI::ResultCode::Success;
         }
         return AZ::RHI::ResultCode::Fail;
     }
 
     AZ::RHI::ResultCode Device::GetViewPose(AZ::u32 viewIndex, AZ::RPI::PoseData& outPoseData) const
     { 
-        if (viewIndex < m_projectionLayerViews.size())
+        if (viewIndex < m_views.size())
         {
-            const XrQuaternionf& orientation = m_projectionLayerViews[viewIndex].pose.orientation;
-            const XrVector3f& position = m_projectionLayerViews[viewIndex].pose.position;
-            outPoseData.m_orientation.Set(orientation.x,
-                                          orientation.y, 
-                                          orientation.z, 
-                                          orientation.w);
-            outPoseData.m_position.Set(position.x,
-                                       position.y, 
-                                       position.z);
+            outPoseData.m_orientation = AzQuaternionFromXrPose(m_views[viewIndex].pose);
+            outPoseData.m_position = AzPositionFromXrPose(m_views[viewIndex].pose);
             return AZ::RHI::ResultCode::Success;
         }
         return AZ::RHI::ResultCode::Fail;
@@ -231,4 +206,5 @@ namespace OpenXRVk
         m_xrVkDevice = VK_NULL_HANDLE;
         m_xrVkPhysicalDevice = VK_NULL_HANDLE;
     }
+
 }