Merge pull request #106166 from stuartcarnie/macos_embed_fixes

macOS: Embedded window fixes

Author: akien-mga

drivers/unix/os_unix.cpp

@@ -131,21 +131,28 @@ static void _setup_clock() {
 }
 #endif
 
+struct sigaction old_action;
+
 static void handle_interrupt(int sig) {
 	if (!EngineDebugger::is_active()) {
 		return;
 	}
 
 	EngineDebugger::get_script_debugger()->set_depth(-1);
 	EngineDebugger::get_script_debugger()->set_lines_left(1);
+
+	// Ensure we call the old action if it was configured.
+	if (old_action.sa_handler && old_action.sa_handler != SIG_IGN && old_action.sa_handler != SIG_DFL) {
+		old_action.sa_handler(sig);
+	}
 }
 
 void OS_Unix::initialize_debugging() {
 	if (EngineDebugger::is_active()) {
 		struct sigaction action;
 		memset(&action, 0, sizeof(action));
 		action.sa_handler = handle_interrupt;
-		sigaction(SIGINT, &action, nullptr);
+		sigaction(SIGINT, &action, &old_action);
 	}
 }
 

editor/plugins/game_view_plugin.cpp

@@ -864,7 +864,7 @@ void GameView::_update_arguments_for_instance(int p_idx, List<String> &r_argumen
 	N = r_arguments.insert_after(N, itos(DisplayServer::get_singleton()->window_get_native_handle(DisplayServer::WINDOW_HANDLE, get_window()->get_window_id())));
 
 #if MACOS_ENABLED
-	r_arguments.push_back("--embedded");
+	N = r_arguments.insert_after(N, "--embedded");
 #endif
 
 	// Be sure to have the correct window size in the embedded_process control.

main/main.cpp

@@ -1399,9 +1399,12 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 			display_driver = NULL_DISPLAY_DRIVER;
 
 		} else if (arg == "--embedded") { // Enable embedded mode.
-
+#ifdef MACOS_ENABLED
 			display_driver = EMBEDDED_DISPLAY_DRIVER;
-
+#else
+			OS::get_singleton()->print("--embedded is only supported on macOS, aborting.\n");
+			goto error;
+#endif
 		} else if (arg == "--log-file") { // write to log file
 
 			if (N) {

platform/macos/display_server_embedded.h

@@ -33,33 +33,27 @@
 #include "core/input/input.h"
 #include "servers/display_server.h"
 
-#if defined(GLES3_ENABLED)
-#include "embedded_gl_manager.h"
-#include "platform_gl.h"
-#endif // GLES3_ENABLED
-
-#if defined(RD_ENABLED)
-#include "servers/rendering/rendering_device.h"
-
-#if defined(VULKAN_ENABLED)
-#import "rendering_context_driver_vulkan_macos.h"
-#endif // VULKAN_ENABLED
-#if defined(METAL_ENABLED)
-#import "drivers/metal/rendering_context_driver_metal.h"
-#endif
-#endif // RD_ENABLED
-
 @class CAContext;
+@class CALayer;
+class GLManagerEmbedded;
+class RenderingContextDriver;
+class RenderingDevice;
+
+struct DisplayServerEmbeddedState {
+	/// Default to a scale of 2.0, which is the most common.
+	float screen_max_scale = 2.0f;
+	float screen_dpi = 96.0f;
+
+	void serialize(PackedByteArray &r_data);
+	Error deserialize(const PackedByteArray &p_data);
+};
 
 class DisplayServerEmbedded : public DisplayServer {
 	GDCLASS(DisplayServerEmbedded, DisplayServer)
 
 	_THREAD_SAFE_CLASS_
 
-	struct {
-		float screen_max_scale = 1.0f;
-		float screen_dpi = 96.0f;
-	} state;
+	DisplayServerEmbeddedState state;
 
 	NativeMenu *native_menu = nullptr;
 
@@ -70,13 +64,11 @@ class DisplayServerEmbedded : public DisplayServer {
 	HashMap<WindowID, Callable> input_event_callbacks;
 	HashMap<WindowID, Callable> input_text_callbacks;
 
-	float content_scale = 1.0f;
-
 	WindowID window_id_counter = MAIN_WINDOW_ID;
 
-	CAContext *ca_context = nil;
+	CAContext *ca_context = nullptr;
 	// Either be a CAMetalLayer or a CALayer depending on the rendering driver.
-	CALayer *layer = nil;
+	CALayer *layer = nullptr;
 #ifdef GLES3_ENABLED
 	GLManagerEmbedded *gl_manager = nullptr;
 #endif
@@ -222,8 +214,7 @@ class DisplayServerEmbedded : public DisplayServer {
 	virtual CursorShape cursor_get_shape() const override;
 	virtual void cursor_set_custom_image(const Ref<Resource> &p_cursor, CursorShape p_shape = CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()) override;
 
-	void update_state(const Dictionary &p_state);
-	void set_content_scale(float p_scale);
+	void set_state(const DisplayServerEmbeddedState &p_state);
 	virtual void swap_buffers() override;
 
 	DisplayServerEmbedded(const String &p_rendering_driver, DisplayServer::WindowMode p_mode, DisplayServer::VSyncMode p_vsync_mode, uint32_t p_flags, const Vector2i *p_position, const Vector2i &p_resolution, int p_screen, Context p_context, Error &r_error);

platform/macos/display_server_embedded.mm

@@ -30,19 +30,31 @@
 
 #import "display_server_embedded.h"
 
-#import "embedded_debugger.h"
-#import "macos_quartz_core_spi.h"
-
-#import "core/config/project_settings.h"
-#import "core/debugger/engine_debugger.h"
-
 #if defined(GLES3_ENABLED)
-#include "drivers/gles3/rasterizer_gles3.h"
+#import "embedded_gl_manager.h"
+#import "platform_gl.h"
+
+#import "drivers/gles3/rasterizer_gles3.h"
 #endif
 
 #if defined(RD_ENABLED)
 #import "servers/rendering/renderer_rd/renderer_compositor_rd.h"
+#import "servers/rendering/rendering_device.h"
+
+#if defined(VULKAN_ENABLED)
+#import "rendering_context_driver_vulkan_macos.h"
+#endif // VULKAN_ENABLED
+#if defined(METAL_ENABLED)
+#import "drivers/metal/rendering_context_driver_metal.h"
 #endif
+#endif // RD_ENABLED
+
+#import "embedded_debugger.h"
+#import "macos_quartz_core_spi.h"
+
+#import "core/config/project_settings.h"
+#import "core/debugger/engine_debugger.h"
+#import "core/io/marshalls.h"
 
 DisplayServerEmbedded::DisplayServerEmbedded(const String &p_rendering_driver, WindowMode p_mode, DisplayServer::VSyncMode p_vsync_mode, uint32_t p_flags, const Vector2i *p_position, const Vector2i &p_resolution, int p_screen, Context p_context, Error &r_error) {
 	EmbeddedDebugger::initialize(this);
@@ -176,15 +188,15 @@
 	}
 #endif
 
-	constexpr CGFloat CONTENT_SCALE = 2.0;
-	layer.contentsScale = CONTENT_SCALE;
+	CGFloat scale = screen_get_max_scale();
+	layer.contentsScale = scale;
 	layer.magnificationFilter = kCAFilterNearest;
 	layer.minificationFilter = kCAFilterNearest;
-	layer.opaque = NO; // Never opaque when embedded.
+	layer.opaque = YES; // Always opaque when embedded.
 	layer.actions = @{ @"contents" : [NSNull null] }; // Disable implicit animations for contents.
 	// AppKit frames, bounds and positions are always in points.
 	CGRect bounds = CGRectMake(0, 0, p_resolution.width, p_resolution.height);
-	bounds = CGRectApplyAffineTransform(bounds, CGAffineTransformMakeScale(1.0 / CONTENT_SCALE, 1.0 / CONTENT_SCALE));
+	bounds = CGRectApplyAffineTransform(bounds, CGAffineTransformInvert(CGAffineTransformMakeScale(scale, scale)));
 	layer.bounds = bounds;
 
 	CGSConnectionID connection_id = CGSMainConnectionID();
@@ -582,11 +594,11 @@
 	[CATransaction begin];
 	[CATransaction setDisableActions:YES];
 
-	// TODO(sgc): Pass scale as argument from parent process.
-	constexpr CGFloat CONTENT_SCALE = 2.0;
+	CGFloat scale = screen_get_max_scale();
 	CGRect bounds = CGRectMake(0, 0, p_size.width, p_size.height);
-	bounds = CGRectApplyAffineTransform(bounds, CGAffineTransformMakeScale(1.0 / CONTENT_SCALE, 1.0 / CONTENT_SCALE));
+	bounds = CGRectApplyAffineTransform(bounds, CGAffineTransformInvert(CGAffineTransformMakeScale(scale, scale)));
 	layer.bounds = bounds;
+	layer.contentsScale = scale;
 
 #if defined(RD_ENABLED)
 	if (rendering_context) {
@@ -685,12 +697,8 @@
 	ime_last_position = p_pos;
 }
 
-void DisplayServerEmbedded::update_state(const Dictionary &p_state) {
-	state.screen_max_scale = p_state["screen_get_max_scale"];
-}
-
-void DisplayServerEmbedded::set_content_scale(float p_scale) {
-	content_scale = p_scale;
+void DisplayServerEmbedded::set_state(const DisplayServerEmbeddedState &p_state) {
+	state = p_state;
 }
 
 void DisplayServerEmbedded::window_set_vsync_mode(DisplayServer::VSyncMode p_vsync_mode, WindowID p_window) {
@@ -742,3 +750,24 @@
 	}
 #endif
 }
+
+void DisplayServerEmbeddedState::serialize(PackedByteArray &r_data) {
+	r_data.resize(8);
+
+	uint8_t *data = r_data.ptrw();
+	data += encode_float(screen_max_scale, data);
+	data += encode_float(screen_dpi, data);
+
+	// Assert we had enough space.
+	DEV_ASSERT(data - r_data.ptrw() >= r_data.size());
+}
+
+Error DisplayServerEmbeddedState::deserialize(const PackedByteArray &p_data) {
+	const uint8_t *data = p_data.ptr();
+
+	screen_max_scale = decode_float(data);
+	data += sizeof(float);
+	screen_dpi = decode_float(data);
+
+	return OK;
+}

platform/macos/display_server_macos.mm

@@ -3297,6 +3297,7 @@
 	[CATransaction setDisableActions:YES];
 
 	EmbeddedProcessData *ed = embedded_processes.getptr(p_pid);
+	CGFloat scale = screen_get_max_scale();
 	if (ed == nil) {
 		ed = &embedded_processes.insert(p_pid, EmbeddedProcessData())->value;
 
@@ -3305,15 +3306,15 @@
 		CALayerHost *host = [CALayerHost new];
 		uint32_t p_context_id = p_process->get_context_id();
 		host.contextId = static_cast<CAContextID>(p_context_id);
-		host.contentsScale = wd->window_object.backingScaleFactor;
+		host.contentsScale = scale;
 		host.contentsGravity = kCAGravityCenter;
 		ed->layer_host = host;
 		[wd->window_view.layer addSublayer:host];
 	}
 
 	Rect2i p_rect = p_process->get_screen_embedded_window_rect();
 	CGRect rect = CGRectMake(p_rect.position.x, p_rect.position.y, p_rect.size.x, p_rect.size.y);
-	rect = CGRectApplyAffineTransform(rect, CGAffineTransformMakeScale(0.5, 0.5));
+	rect = CGRectApplyAffineTransform(rect, CGAffineTransformInvert(CGAffineTransformMakeScale(scale, scale)));
 
 	CGFloat height = wd->window_view.frame.size.height;
 	CGFloat x = rect.origin.x;

platform/macos/editor/embedded_process_macos.h

@@ -102,12 +102,12 @@ class EmbeddedProcessMacOS final : public EmbeddedProcessBase {
 		return embedding_state == EmbeddingState::COMPLETED;
 	}
 
-	virtual bool is_process_focused() const override { return layer_host->has_focus(); }
-	virtual void embed_process(OS::ProcessID p_pid) override;
-	virtual int get_embedded_pid() const override { return current_process_id; }
-	virtual void reset() override;
-	virtual void request_close() override;
-	virtual void queue_update_embedded_process() override { update_embedded_process(); }
+	bool is_process_focused() const override { return layer_host->has_focus(); }
+	void embed_process(OS::ProcessID p_pid) override;
+	int get_embedded_pid() const override { return current_process_id; }
+	void reset() override;
+	void request_close() override;
+	void queue_update_embedded_process() override { update_embedded_process(); }
 
 	Rect2i get_adjusted_embedded_window_rect(const Rect2i &p_rect) const override;
 
@@ -117,4 +117,5 @@ class EmbeddedProcessMacOS final : public EmbeddedProcessBase {
 	_FORCE_INLINE_ DisplayServer::MouseMode get_mouse_mode() const { return mouse_mode; }
 
 	EmbeddedProcessMacOS();
+	~EmbeddedProcessMacOS() override;
 };

platform/macos/editor/embedded_process_macos.mm

@@ -30,6 +30,7 @@
 
 #include "embedded_process_macos.h"
 
+#include "platform/macos/display_server_embedded.h"
 #include "platform/macos/display_server_macos.h"
 
 #include "core/input/input_event_codec.h"
@@ -122,7 +123,7 @@
 
 void EmbeddedProcessMacOS::request_close() {
 	if (current_process_id != 0 && is_embedding_completed()) {
-		ds->request_close_embedded_process(current_process_id);
+		script_debugger->send_message("embed:win_event", { DisplayServer::WINDOW_EVENT_CLOSE_REQUEST });
 	}
 }
 
@@ -133,19 +134,22 @@
 
 	Error err = ds->embed_process_update(window->get_window_id(), this);
 	if (err == OK) {
+		// Replicate some of the DisplayServer state.
+		{
+			DisplayServerEmbeddedState state;
+			state.screen_max_scale = ds->screen_get_max_scale();
+			state.screen_dpi = ds->screen_get_dpi();
+			PackedByteArray data;
+			state.serialize(data);
+			script_debugger->send_message("embed:ds_state", { data });
+		}
+
 		Rect2i rect = get_screen_embedded_window_rect();
 		script_debugger->send_message("embed:window_size", { rect.size });
 		embedding_state = EmbeddingState::COMPLETED;
 		queue_redraw();
 		emit_signal(SNAME("embedding_completed"));
 
-		// Replicate some of the DisplayServer state.
-		{
-			Dictionary state;
-			state["screen_get_max_scale"] = ds->screen_get_max_scale();
-			// script_debugger->send_message("embed:ds_state", { state });
-		}
-
 		// Send initial joystick state.
 		{
 			Input *input = Input::get_singleton();
@@ -209,13 +213,21 @@
 	ED_SHORTCUT("game_view/release_mouse", TTRC("Release Mouse"), KeyModifierMask::ALT | Key::ESCAPE);
 }
 
+EmbeddedProcessMacOS::~EmbeddedProcessMacOS() {
+	if (current_process_id != 0) {
+		// Stop embedding the last process.
+		OS::get_singleton()->kill(current_process_id);
+		reset();
+	}
+}
+
 void LayerHost::_notification(int p_what) {
 	switch (p_what) {
 		case NOTIFICATION_FOCUS_ENTER: {
 			if (script_debugger) {
 				script_debugger->send_message("embed:win_event", { DisplayServer::WINDOW_EVENT_MOUSE_ENTER });
 			}
-			// Temporarily release mouse capture, so we can interact with the editor.
+			// Restore mouse capture, if necessary.
 			DisplayServer *ds = DisplayServer::get_singleton();
 			if (process->get_mouse_mode() != ds->mouse_get_mode()) {
 				// Restore embedded process mouse mode.

platform/macos/embedded_debugger.h

@@ -63,6 +63,7 @@ class EmbeddedDebugger {
 	Error _msg_ime_update(const Array &p_args);
 	Error _msg_joy_add(const Array &p_args);
 	Error _msg_joy_del(const Array &p_args);
+	Error _msg_ds_state(const Array &p_args);
 
 public:
 	static Error parse_message(void *p_user, const String &p_msg, const Array &p_args, bool &r_captured);