Merge pull request godotengine#112888 from dhoverb/meta_foveation_eye_tracked

Implement XR_META_foveation_eye_tracked

Author: Repiteo

drivers/vulkan/rendering_device_driver_vulkan.cpp

@@ -523,6 +523,7 @@ Error RenderingDeviceDriverVulkan::_initialize_device_extensions() {
 	_register_requested_device_extension(VK_KHR_MULTIVIEW_EXTENSION_NAME, false);
 	_register_requested_device_extension(VK_KHR_FRAGMENT_SHADING_RATE_EXTENSION_NAME, false);
 	_register_requested_device_extension(VK_EXT_FRAGMENT_DENSITY_MAP_EXTENSION_NAME, false);
+	_register_requested_device_extension(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME, false);
 	_register_requested_device_extension(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME, false);
 	_register_requested_device_extension(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME, false);
 	_register_requested_device_extension(VK_KHR_STORAGE_BUFFER_STORAGE_CLASS_EXTENSION_NAME, false);
@@ -760,6 +761,7 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 		VkPhysicalDeviceVulkanMemoryModelFeaturesKHR vulkan_memory_model_features = {};
 		VkPhysicalDeviceFragmentShadingRateFeaturesKHR fsr_features = {};
 		VkPhysicalDeviceFragmentDensityMapFeaturesEXT fdm_features = {};
+		VkPhysicalDeviceFragmentDensityMapOffsetFeaturesQCOM fdmo_features_qcom = {};
 		VkPhysicalDevice16BitStorageFeaturesKHR storage_feature = {};
 		VkPhysicalDeviceMultiviewFeatures multiview_features = {};
 		VkPhysicalDevicePipelineCreationCacheControlFeatures pipeline_cache_control_features = {};
@@ -799,6 +801,12 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 			next_features = &fdm_features;
 		}
 
+		if (enabled_device_extension_names.has(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME)) {
+			fdmo_features_qcom.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_DENSITY_MAP_OFFSET_FEATURES_QCOM;
+			fdmo_features_qcom.pNext = next_features;
+			next_features = &fdmo_features_qcom;
+		}
+
 		if (enabled_device_extension_names.has(VK_KHR_16BIT_STORAGE_EXTENSION_NAME)) {
 			storage_feature.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES_KHR;
 			storage_feature.pNext = next_features;
@@ -863,6 +871,10 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 			fdm_capabilities.non_subsampled_images_supported = fdm_features.fragmentDensityMapNonSubsampledImages;
 		}
 
+		if (enabled_device_extension_names.has(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME)) {
+			fdm_capabilities.offset_supported = fdmo_features_qcom.fragmentDensityMapOffset;
+		}
+
 		// Multiple VRS techniques can't co-exist during the existence of one device, so we must
 		// choose one at creation time and only report one of them as available.
 		_choose_vrs_capabilities();
@@ -898,6 +910,7 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 		void *next_properties = nullptr;
 		VkPhysicalDeviceFragmentShadingRatePropertiesKHR fsr_properties = {};
 		VkPhysicalDeviceFragmentDensityMapPropertiesEXT fdm_properties = {};
+		VkPhysicalDeviceFragmentDensityMapOffsetPropertiesQCOM fdmo_properties = {};
 		VkPhysicalDeviceMultiviewProperties multiview_properties = {};
 		VkPhysicalDeviceSubgroupProperties subgroup_properties = {};
 		VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control_properties = {};
@@ -935,6 +948,12 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 			next_properties = &fdm_properties;
 		}
 
+		if (fdm_capabilities.offset_supported) {
+			fdmo_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_DENSITY_MAP_OFFSET_PROPERTIES_QCOM;
+			fdmo_properties.pNext = next_properties;
+			next_properties = &fdmo_properties;
+		}
+
 		physical_device_properties_2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
 		physical_device_properties_2.pNext = next_properties;
 		functions.GetPhysicalDeviceProperties2(physical_device, &physical_device_properties_2);
@@ -1002,6 +1021,17 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 			print_verbose("- Vulkan Fragment Density Map not supported");
 		}
 
+		if (fdm_capabilities.offset_supported) {
+			print_verbose("- Vulkan Fragment Density Map Offset supported");
+
+			fdm_capabilities.offset_granularity.x = fdmo_properties.fragmentDensityOffsetGranularity.width;
+			fdm_capabilities.offset_granularity.y = fdmo_properties.fragmentDensityOffsetGranularity.height;
+
+			print_verbose(vformat("  Offset granularity: (%d, %d)", fdm_capabilities.offset_granularity.x, fdm_capabilities.offset_granularity.y));
+		} else {
+			print_verbose("- Vulkan Fragment Density Map Offset not supported");
+		}
+
 		if (multiview_capabilities.is_supported) {
 			multiview_capabilities.max_view_count = multiview_properties.maxMultiviewViewCount;
 			multiview_capabilities.max_instance_count = multiview_properties.maxMultiviewInstanceIndex;
@@ -3708,11 +3738,6 @@ RDD::FramebufferID RenderingDeviceDriverVulkan::framebuffer_create(RenderPassID
 	for (uint32_t i = 0; i < p_attachments.size(); i++) {
 		const TextureInfo *texture = (const TextureInfo *)p_attachments[i].id;
 		vk_img_views[i] = texture->vk_view;
-
-		if (render_pass->uses_fragment_density_map_offsets && (texture->vk_create_info.usage & VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT)) {
-			// If the render pass uses the FDM and the usage fits, we store the amount of layers to use it later on the render pass's end.
-			fragment_density_map_offsets_layers = texture->vk_create_info.arrayLayers;
-		}
 	}
 
 	VkFramebufferCreateInfo framebuffer_create_info = {};
@@ -4921,6 +4946,7 @@ RDD::RenderPassID RenderingDeviceDriverVulkan::render_pass_create(VectorView<Att
 
 	RenderPassInfo *render_pass = VersatileResource::allocate<RenderPassInfo>(resources_allocator);
 	render_pass->vk_render_pass = vk_render_pass;
+	render_pass->uses_fragment_density_map = uses_fragment_density_map;
 	return RenderPassID(render_pass);
 }
 
@@ -4982,7 +5008,25 @@ void RenderingDeviceDriverVulkan::command_end_render_pass(CommandBufferID p_cmd_
 	DEV_ASSERT(command_buffer->active_framebuffer != nullptr && "A framebuffer must be active.");
 	DEV_ASSERT(command_buffer->active_render_pass != nullptr && "A render pass must be active.");
 
-	vkCmdEndRenderPass(command_buffer->vk_command_buffer);
+	if (device_functions.EndRenderPass2KHR != nullptr && fdm_capabilities.offset_supported && command_buffer->active_render_pass->uses_fragment_density_map) {
+		LocalVector<VkOffset2D> fragment_density_offsets;
+		if (VulkanHooks::get_singleton() != nullptr) {
+			fragment_density_offsets = VulkanHooks::get_singleton()->get_fragment_density_offsets();
+		}
+
+		VkSubpassFragmentDensityMapOffsetEndInfoQCOM offset_info = {};
+		offset_info.sType = VK_STRUCTURE_TYPE_SUBPASS_FRAGMENT_DENSITY_MAP_OFFSET_END_INFO_QCOM;
+		offset_info.pFragmentDensityOffsets = fragment_density_offsets.is_empty() ? nullptr : fragment_density_offsets.ptr();
+		offset_info.fragmentDensityOffsetCount = fragment_density_offsets.size();
+
+		VkSubpassEndInfo subpass_end_info = {};
+		subpass_end_info.sType = VK_STRUCTURE_TYPE_SUBPASS_END_INFO;
+		subpass_end_info.pNext = &offset_info;
+
+		device_functions.EndRenderPass2KHR(command_buffer->vk_command_buffer, &subpass_end_info);
+	} else {
+		vkCmdEndRenderPass(command_buffer->vk_command_buffer);
+	}
 
 	command_buffer->active_render_pass = nullptr;
 	command_buffer->active_framebuffer = nullptr;

drivers/vulkan/rendering_device_driver_vulkan.h

@@ -575,7 +575,7 @@ class RenderingDeviceDriverVulkan : public RenderingDeviceDriver {
 
 	struct RenderPassInfo {
 		VkRenderPass vk_render_pass = VK_NULL_HANDLE;
-		bool uses_fragment_density_map_offsets = false;
+		bool uses_fragment_density_map = false;
 	};
 
 public:

drivers/vulkan/vulkan_hooks.h

@@ -30,6 +30,7 @@
 
 #pragma once
 
+#include "core/templates/local_vector.h"
 #include "drivers/vulkan/godot_vulkan.h"
 
 class VulkanHooks {
@@ -43,5 +44,6 @@ class VulkanHooks {
 	virtual bool get_physical_device(VkPhysicalDevice *r_device) = 0;
 	virtual bool create_vulkan_device(const VkDeviceCreateInfo *p_device_create_info, VkDevice *r_device) = 0;
 	virtual void set_direct_queue_family_and_index(uint32_t p_queue_family_index, uint32_t p_queue_index) = 0;
+	virtual LocalVector<VkOffset2D> get_fragment_density_offsets() = 0;
 	static VulkanHooks *get_singleton() { return singleton; }
 };

modules/openxr/extensions/openxr_fb_foveation_extension.cpp

@@ -30,6 +30,7 @@
 
 #include "openxr_fb_foveation_extension.h"
 #include "core/config/project_settings.h"
+#include "openxr_eye_gaze_interaction.h"
 
 #include "../openxr_platform_inc.h"
 
@@ -54,6 +55,14 @@ OpenXRFBFoveationExtension::OpenXRFBFoveationExtension(const String &p_rendering
 	swapchain_create_info_foveation_fb.next = nullptr;
 	swapchain_create_info_foveation_fb.flags = 0;
 
+	meta_foveation_eye_tracked_create_info.type = XR_TYPE_FOVEATION_EYE_TRACKED_PROFILE_CREATE_INFO_META;
+	meta_foveation_eye_tracked_create_info.next = nullptr;
+	meta_foveation_eye_tracked_create_info.flags = 0;
+
+	meta_foveation_eye_tracked_properties.type = XR_TYPE_SYSTEM_FOVEATION_EYE_TRACKED_PROPERTIES_META;
+	meta_foveation_eye_tracked_properties.next = nullptr;
+	meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked = XR_FALSE;
+
 	if (rendering_driver == "opengl3") {
 		swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB;
 	} else if (rendering_driver == "vulkan") {
@@ -75,6 +84,7 @@ HashMap<String, bool *> OpenXRFBFoveationExtension::get_requested_extensions() {
 #ifdef XR_USE_GRAPHICS_API_VULKAN
 	if (rendering_driver == "vulkan") {
 		request_extensions[XR_FB_FOVEATION_VULKAN_EXTENSION_NAME] = &fb_foveation_vulkan_ext;
+		request_extensions[XR_META_FOVEATION_EYE_TRACKED_EXTENSION_NAME] = &meta_foveation_eye_tracked;
 	}
 #endif // XR_USE_GRAPHICS_API_VULKAN
 
@@ -90,11 +100,16 @@ void OpenXRFBFoveationExtension::on_instance_created(const XrInstance p_instance
 	if (fb_foveation_configuration_ext) {
 		// nothing to register here...
 	}
+
+	if (meta_foveation_eye_tracked) {
+		EXT_INIT_XR_FUNC(xrGetFoveationEyeTrackedStateMETA);
+	}
 }
 
 void OpenXRFBFoveationExtension::on_instance_destroyed() {
 	fb_foveation_ext = false;
 	fb_foveation_configuration_ext = false;
+	meta_foveation_eye_tracked = false;
 }
 
 bool OpenXRFBFoveationExtension::is_enabled() const {
@@ -107,6 +122,16 @@ bool OpenXRFBFoveationExtension::is_enabled() const {
 	return enabled;
 }
 
+void *OpenXRFBFoveationExtension::set_system_properties_and_get_next_pointer(void *p_next_pointer) {
+#ifdef XR_USE_GRAPHICS_API_VULKAN
+	if (rendering_driver == "vulkan") {
+		meta_foveation_eye_tracked_properties.next = p_next_pointer;
+		return &meta_foveation_eye_tracked_properties;
+	}
+#endif
+	return p_next_pointer;
+}
+
 void *OpenXRFBFoveationExtension::set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) {
 	if (is_enabled()) {
 		swapchain_create_info_foveation_fb.next = p_next_pointer;
@@ -142,6 +167,37 @@ void OpenXRFBFoveationExtension::set_foveation_dynamic(XrFoveationDynamicFB p_fo
 	update_profile();
 }
 
+LocalVector<Vector2i> OpenXRFBFoveationExtension::get_fragment_density_offsets() {
+	LocalVector<Vector2i> ret;
+	if (!is_enabled() || !meta_foveation_eye_tracked || !meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked || !OpenXREyeGazeInteractionExtension::get_singleton()->is_available()) {
+		return ret;
+	}
+
+	OpenXRAPI *openxr_api = OpenXRAPI::get_singleton();
+	ERR_FAIL_NULL_V(openxr_api, ret);
+
+	XrFoveationEyeTrackedStateMETA state = {
+		XR_TYPE_FOVEATION_EYE_TRACKED_STATE_META, // type
+		nullptr, // next
+		{ XrVector2f{}, XrVector2f{} }, // foveationCenter[XR_FOVEATION_CENTER_SIZE_META];
+		0, // flags
+	};
+	XrResult result = xrGetFoveationEyeTrackedStateMETA(openxr_api->get_session(), &state);
+	if (XR_FAILED(result)) {
+		print_line("OpenXR: Unable to get foveation offsets [", openxr_api->get_error_string(result), "]");
+		return ret;
+	}
+
+	ret.reserve(XR_FOVEATION_CENTER_SIZE_META);
+	Size2 dims = openxr_api->get_recommended_target_size() * 0.5 / openxr_api->get_render_target_size_multiplier();
+	for (uint32_t i = 0; i < XR_FOVEATION_CENTER_SIZE_META; ++i) {
+		const XrVector2f &xr_offset = state.foveationCenter[i];
+		ret.push_back(Vector2i((int)(xr_offset.x * dims.x), (int)(xr_offset.y * dims.y)));
+	}
+
+	return ret;
+}
+
 void OpenXRFBFoveationExtension::_update_profile() {
 	// Must be called from rendering thread!
 	ERR_NOT_ON_RENDER_THREAD;
@@ -162,9 +218,15 @@ void OpenXRFBFoveationExtension::_update_profile() {
 		return;
 	}
 
+	void *next = nullptr;
+	if (fov_ext->meta_foveation_eye_tracked && fov_ext->meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked) {
+		fov_ext->meta_foveation_eye_tracked_create_info.next = next;
+		next = &fov_ext->meta_foveation_eye_tracked_create_info;
+	}
+
 	XrFoveationLevelProfileCreateInfoFB level_profile_create_info;
 	level_profile_create_info.type = XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB;
-	level_profile_create_info.next = nullptr;
+	level_profile_create_info.next = next;
 	level_profile_create_info.level = fov_ext->foveation_level;
 	level_profile_create_info.verticalOffset = 0.0f;
 	level_profile_create_info.dynamic = fov_ext->foveation_dynamic;

modules/openxr/extensions/openxr_fb_foveation_extension.h

@@ -57,6 +57,7 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
 	virtual void on_instance_created(const XrInstance p_instance) override;
 	virtual void on_instance_destroyed() override;
 
+	virtual void *set_system_properties_and_get_next_pointer(void *p_next_pointer) override;
 	virtual void *set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) override;
 
 	virtual void on_main_swapchains_created() override;
@@ -69,6 +70,8 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
 	XrFoveationDynamicFB get_foveation_dynamic() const;
 	void set_foveation_dynamic(XrFoveationDynamicFB p_foveation_dynamic);
 
+	LocalVector<Vector2i> get_fragment_density_offsets();
+
 private:
 	static OpenXRFBFoveationExtension *singleton;
 
@@ -77,6 +80,7 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
 	bool fb_foveation_ext = false;
 	bool fb_foveation_configuration_ext = false;
 	bool fb_foveation_vulkan_ext = false;
+	bool meta_foveation_eye_tracked = false;
 
 	// Configuration
 	XrFoveationLevelFB foveation_level = XR_FOVEATION_LEVEL_NONE_FB;
@@ -96,7 +100,12 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
 	XrSwapchainCreateInfoFoveationFB swapchain_create_info_foveation_fb;
 	OpenXRFBUpdateSwapchainExtension *swapchain_update_state_ext = nullptr;
 
+	// Enable eye tracked foveation
+	XrSystemFoveationEyeTrackedPropertiesMETA meta_foveation_eye_tracked_properties;
+	XrFoveationEyeTrackedProfileCreateInfoMETA meta_foveation_eye_tracked_create_info;
+
 	// OpenXR API call wrappers
 	EXT_PROTO_XRRESULT_FUNC3(xrCreateFoveationProfileFB, (XrSession), session, (const XrFoveationProfileCreateInfoFB *), create_info, (XrFoveationProfileFB *), profile);
 	EXT_PROTO_XRRESULT_FUNC1(xrDestroyFoveationProfileFB, (XrFoveationProfileFB), profile);
+	EXT_PROTO_XRRESULT_FUNC2(xrGetFoveationEyeTrackedStateMETA, (XrSession), session, (XrFoveationEyeTrackedStateMETA *), foveationState);
 };

modules/openxr/extensions/openxr_fb_update_swapchain_extension.cpp

@@ -105,17 +105,7 @@ void OpenXRFBUpdateSwapchainExtension::on_instance_destroyed() {
 }
 
 bool OpenXRFBUpdateSwapchainExtension::is_enabled() const {
-	if (rendering_driver == "vulkan") {
-		return fb_swapchain_update_state_ext && fb_swapchain_update_state_vulkan_ext;
-	} else if (rendering_driver == "opengl3") {
-#ifdef XR_USE_GRAPHICS_API_OPENGL_ES
-		return fb_swapchain_update_state_ext && fb_swapchain_update_state_opengles_ext;
-#else
-		return fb_swapchain_update_state_ext;
-#endif
-	}
-
-	return false;
+	return fb_swapchain_update_state_ext;
 }
 
 bool OpenXRFBUpdateSwapchainExtension::is_android_ext_enabled() const {

modules/openxr/extensions/platform/openxr_vulkan_extension.cpp

@@ -203,6 +203,23 @@ void OpenXRVulkanExtension::set_direct_queue_family_and_index(uint32_t p_queue_f
 	vulkan_queue_index = p_queue_index;
 }
 
+LocalVector<VkOffset2D> OpenXRVulkanExtension::get_fragment_density_offsets() {
+	LocalVector<VkOffset2D> ret;
+	OpenXRFBFoveationExtension *fb_foveation = OpenXRFBFoveationExtension::get_singleton();
+	if (fb_foveation == nullptr) {
+		return ret;
+	}
+
+	LocalVector<Vector2i> offsets = fb_foveation->get_fragment_density_offsets();
+
+	ret.reserve(offsets.size());
+	for (const Vector2i &offset : offsets) {
+		ret.push_back(VkOffset2D{ offset.x, offset.y });
+	}
+
+	return ret;
+}
+
 XrGraphicsBindingVulkanKHR OpenXRVulkanExtension::graphics_binding_vulkan;
 
 void *OpenXRVulkanExtension::set_session_create_and_get_next_pointer(void *p_next_pointer) {

modules/openxr/extensions/platform/openxr_vulkan_extension.h

@@ -54,6 +54,7 @@ class OpenXRVulkanExtension : public OpenXRGraphicsExtensionWrapper, VulkanHooks
 	virtual bool get_physical_device(VkPhysicalDevice *r_device) override final;
 	virtual bool create_vulkan_device(const VkDeviceCreateInfo *p_device_create_info, VkDevice *r_device) override final;
 	virtual void set_direct_queue_family_and_index(uint32_t p_queue_family_index, uint32_t p_queue_index) override final;
+	virtual LocalVector<VkOffset2D> get_fragment_density_offsets() override final;
 
 	virtual void get_usable_swapchain_formats(Vector<int64_t> &p_usable_swap_chains) override;
 	virtual void get_usable_depth_formats(Vector<int64_t> &p_usable_swap_chains) override;