Desktop: Execute editor and node graph natively (#2955)

* Desktop: Execute editor and node graph natively

* Remove decouple execution feature

* Disable feature gate for native communication functions

* Avoid ininite message loop on an infinite canvas

* Add any lint exception

* Build evaluation loop

* Fix texture passing message

* Cleanup

* More cleanup

---------

Co-authored-by: Timon Schelling <[email protected]>

Author: TrueDoctor

Cargo.lock

@@ -9,17 +9,19 @@ edition = "2024"
 rust-version = "1.87"
 
 [features]
-# default = ["gpu"]
-# gpu = ["graphite-editor/gpu"]
+default = ["gpu"]
+gpu = ["graphite-editor/gpu"]
 
 [dependencies]
-# Local dependencies
-# graphite-editor = { path = "../editor", features = [
-# 	"gpu",
-# 	"ron",
-# 	"vello",
-# 	"decouple-execution",
-# ] }
+# # Local dependencies
+graphite-editor = { path = "../editor", features = [
+	"gpu",
+	"ron",
+	"vello",
+] }
+graph-craft = { workspace = true }
+wgpu-executor = { workspace = true }
+
 wgpu = { workspace = true }
 winit = { workspace = true, features = ["serde"] }
 thiserror = { workspace = true }
@@ -29,4 +31,6 @@ include_dir = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing = { workspace = true }
 dirs = { workspace = true }
+ron = { workspace = true}
 bytemuck = { workspace = true }
+glam = { workspace = true }

desktop/Cargo.toml

@@ -2,6 +2,9 @@ use crate::CustomEvent;
 use crate::WindowSize;
 use crate::render::GraphicsState;
 use crate::render::WgpuContext;
+use graph_craft::wasm_application_io::WasmApplicationIo;
+use graphite_editor::application::Editor;
+use graphite_editor::messages::prelude::*;
 use std::sync::Arc;
 use std::sync::mpsc::Sender;
 use std::time::Duration;
@@ -21,11 +24,10 @@ pub(crate) struct WinitApp {
 	pub(crate) cef_context: cef::Context<cef::Initialized>,
 	pub(crate) window: Option<Arc<Window>>,
 	cef_schedule: Option<Instant>,
-	_ui_frame_buffer: Option<wgpu::Texture>,
 	window_size_sender: Sender<WindowSize>,
-	_viewport_frame_buffer: Option<wgpu::Texture>,
 	graphics_state: Option<GraphicsState>,
 	wgpu_context: WgpuContext,
+	pub(crate) editor: Editor,
 }
 
 impl WinitApp {
@@ -34,13 +36,28 @@ impl WinitApp {
 			cef_context,
 			window: None,
 			cef_schedule: Some(Instant::now()),
-			_viewport_frame_buffer: None,
-			_ui_frame_buffer: None,
 			graphics_state: None,
 			window_size_sender,
 			wgpu_context,
+			editor: Editor::new(),
 		}
 	}
+
+	fn dispatch_message(&mut self, message: Message) {
+		let responses = self.editor.handle_message(message);
+		self.send_messages_to_editor(responses);
+	}
+
+	fn send_messages_to_editor(&mut self, responses: Vec<FrontendMessage>) {
+		if responses.is_empty() {
+			return;
+		}
+		let Ok(message) = ron::to_string(&responses) else {
+			tracing::error!("Failed to serialize Messages");
+			return;
+		};
+		self.cef_context.send_web_message(message.as_bytes());
+	}
 }
 
 impl ApplicationHandler<CustomEvent> for WinitApp {
@@ -49,16 +66,22 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 		let timeout = Instant::now() + Duration::from_millis(10);
 		let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout));
 		self.cef_context.work();
+
 		event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
 	}
 
-	fn new_events(&mut self, _event_loop: &ActiveEventLoop, _cause: StartCause) {
+	fn new_events(&mut self, _event_loop: &ActiveEventLoop, cause: StartCause) {
 		if let Some(schedule) = self.cef_schedule
 			&& schedule < Instant::now()
 		{
 			self.cef_schedule = None;
 			self.cef_context.work();
 		}
+		if let StartCause::ResumeTimeReached { .. } = cause {
+			if let Some(window) = &self.window {
+				window.request_redraw();
+			}
+		}
 	}
 
 	fn resumed(&mut self, event_loop: &ActiveEventLoop) {
@@ -77,6 +100,10 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 		self.graphics_state = Some(graphics_state);
 
 		tracing::info!("Winit window created and ready");
+
+		let application_io = WasmApplicationIo::new_with_context(self.wgpu_context.clone());
+
+		futures::executor::block_on(graphite_editor::node_graph_executor::replace_application_io(application_io));
 	}
 
 	fn user_event(&mut self, _: &ActiveEventLoop, event: CustomEvent) {
@@ -97,6 +124,46 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 					self.cef_schedule = Some(instant);
 				}
 			}
+			CustomEvent::MessageReceived { message } => {
+				if let Message::InputPreprocessor(ipp_message) = &message {
+					if let Some(window) = &self.window {
+						window.request_redraw();
+					}
+				}
+				if let Message::InputPreprocessor(InputPreprocessorMessage::BoundsOfViewports { bounds_of_viewports }) = &message {
+					if let Some(graphic_state) = &mut self.graphics_state {
+						let window_size = self.window.as_ref().unwrap().inner_size();
+						let window_size = glam::Vec2::new(window_size.width as f32, window_size.height as f32);
+						let top_left = bounds_of_viewports[0].top_left.as_vec2() / window_size;
+						let bottom_right = bounds_of_viewports[0].bottom_right.as_vec2() / window_size;
+						let offset = top_left.to_array();
+						let scale = (bottom_right - top_left).recip();
+						graphic_state.set_viewport_offset(offset);
+						graphic_state.set_viewport_scale(scale.to_array());
+					} else {
+						panic!("graphics state not intialized, viewport offset might be lost");
+					}
+				}
+				self.dispatch_message(message);
+			}
+			CustomEvent::NodeGraphRan { texture } => {
+				if let Some(texture) = texture
+					&& let Some(graphics_state) = &mut self.graphics_state
+				{
+					graphics_state.bind_viewport_texture(&texture);
+				}
+				let mut responses = VecDeque::new();
+				let err = self.editor.poll_node_graph_evaluation(&mut responses);
+				if let Err(e) = err {
+					if e != "No active document" {
+						tracing::error!("Error poling node graph: {}", e);
+					}
+				}
+
+				for message in responses {
+					self.dispatch_message(message);
+				}
+			}
 		}
 	}
 

desktop/src/app.rs

@@ -120,5 +120,15 @@ impl CefEventHandler for CefHandler {
 		let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(scheduled_time));
 	}
 
-	fn receive_web_message(&self, message: &[u8]) {}
+	fn receive_web_message(&self, message: &[u8]) {
+		let str = std::str::from_utf8(message).unwrap();
+		match ron::from_str(str) {
+			Ok(message) => {
+				let _ = self.event_loop_proxy.send_event(CustomEvent::MessageReceived { message });
+			}
+			Err(e) => {
+				tracing::error!("Failed to deserialize message {:?}", e)
+			}
+		}
+	}
 }

desktop/src/cef.rs

@@ -1,9 +1,6 @@
 use cef::rc::{ConvertReturnValue, Rc, RcImpl};
 use cef::sys::{_cef_render_process_handler_t, cef_base_ref_counted_t, cef_render_process_handler_t, cef_v8_propertyattribute_t, cef_v8_value_create_array_buffer_with_copy};
-use cef::{
-	CefString, ImplFrame, ImplRenderProcessHandler, ImplV8Context, ImplV8Value, V8Handler, V8Propertyattribute, V8Value, WrapRenderProcessHandler, v8_context_get_entered_context,
-	v8_value_create_function,
-};
+use cef::{CefString, ImplFrame, ImplRenderProcessHandler, ImplV8Context, ImplV8Value, V8Handler, V8Propertyattribute, V8Value, WrapRenderProcessHandler, v8_value_create_function};
 
 use crate::cef::ipc::{MessageType, UnpackMessage, UnpackedMessage};
 
@@ -61,7 +58,9 @@ impl ImplRenderProcessHandler for RenderProcessHandlerImpl {
 					cef_v8_propertyattribute_t::V8_PROPERTY_ATTRIBUTE_READONLY.wrap_result(),
 				);
 
-				frame.execute_java_script(Some(&CefString::from(function_call.as_str())), None, 0);
+				if global.value_bykey(Some(&CefString::from(function_name))).is_some() {
+					frame.execute_java_script(Some(&CefString::from(function_call.as_str())), None, 0);
+				}
 
 				if context.exit() == 0 {
 					tracing::error!("Failed to exit V8 context");

desktop/src/cef/internal/render_process_handler.rs

@@ -1,4 +1,4 @@
-use cef::{CefString, Frame, ImplBinaryValue, ImplBrowser, ImplFrame, ImplListValue, ImplProcessMessage, ImplV8Context, ProcessId, V8Context, rc::ConvertParam, sys::cef_process_id_t};
+use cef::{CefString, Frame, ImplBinaryValue, ImplBrowser, ImplFrame, ImplListValue, ImplProcessMessage, ImplV8Context, ProcessId, V8Context, sys::cef_process_id_t};
 
 use super::{Context, Initialized};
 

desktop/src/cef/ipc.rs

@@ -1,7 +1,8 @@
-use std::fmt::Debug;
 use std::process::exit;
 use std::time::Instant;
+use std::{fmt::Debug, time::Duration};
 
+use graphite_editor::messages::prelude::Message;
 use tracing_subscriber::EnvFilter;
 use winit::event_loop::EventLoop;
 
@@ -20,6 +21,8 @@ mod dirs;
 pub(crate) enum CustomEvent {
 	UiUpdate(wgpu::Texture),
 	ScheduleBrowserWork(Instant),
+	MessageReceived { message: Message },
+	NodeGraphRan { texture: Option<wgpu::Texture> },
 }
 
 fn main() {
@@ -38,7 +41,7 @@ fn main() {
 
 	let (window_size_sender, window_size_receiver) = std::sync::mpsc::channel();
 
-	let wgpu_context = futures::executor::block_on(WgpuContext::new());
+	let wgpu_context = futures::executor::block_on(WgpuContext::new()).unwrap();
 	let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy(), wgpu_context.clone())) {
 		Ok(c) => c,
 		Err(cef::InitError::AlreadyRunning) => {
@@ -51,6 +54,25 @@ fn main() {
 		}
 	};
 
+	tracing::info!("Cef initialized successfully");
+
+	let rendering_loop_proxy = event_loop.create_proxy();
+	let target_fps = 60;
+	std::thread::spawn(move || {
+		loop {
+			let last_render = Instant::now();
+			let (has_run, texture) = futures::executor::block_on(graphite_editor::node_graph_executor::run_node_graph());
+			if has_run {
+				let _ = rendering_loop_proxy.send_event(CustomEvent::NodeGraphRan {
+					texture: texture.map(|t| (*t.texture).clone()),
+				});
+			}
+			let frame_time = Duration::from_secs_f32((target_fps as f32).recip());
+			let sleep = last_render + frame_time - Instant::now();
+			std::thread::sleep(sleep);
+		}
+	});
+
 	let mut winit_app = WinitApp::new(cef_context, window_size_sender, wgpu_context);
 
 	event_loop.run_app(&mut winit_app).unwrap();

desktop/src/main.rs

@@ -56,46 +56,7 @@ pub(crate) enum FrameBufferError {
 	InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize },
 }
 
-#[derive(Debug, Clone)]
-pub(crate) struct WgpuContext {
-	pub(crate) device: wgpu::Device,
-	pub(crate) queue: wgpu::Queue,
-	adapter: wgpu::Adapter,
-	instance: wgpu::Instance,
-}
-
-impl WgpuContext {
-	pub(crate) async fn new() -> Self {
-		let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
-			backends: wgpu::Backends::PRIMARY,
-			..Default::default()
-		});
-
-		let adapter = instance
-			.request_adapter(&wgpu::RequestAdapterOptions {
-				power_preference: wgpu::PowerPreference::default(),
-				compatible_surface: None,
-				force_fallback_adapter: false,
-			})
-			.await
-			.unwrap();
-
-		let required_limits = adapter.limits();
-
-		let (device, queue) = adapter
-			.request_device(&wgpu::DeviceDescriptor {
-				label: None,
-				required_features: wgpu::Features::PUSH_CONSTANTS,
-				required_limits,
-				memory_hints: Default::default(),
-				trace: wgpu::Trace::Off,
-			})
-			.await
-			.unwrap();
-
-		Self { device, queue, adapter, instance }
-	}
-}
+pub use wgpu_executor::Context as WgpuContext;
 
 #[derive(Debug)]
 pub(crate) struct GraphicsState {

desktop/src/render.rs

@@ -14,7 +14,6 @@ license = "Apache-2.0"
 default = ["wasm"]
 wasm = ["wasm-bindgen", "graphene-std/wasm", "wasm-bindgen-futures"]
 gpu = ["interpreted-executor/gpu", "wgpu-executor"]
-decouple-execution = []
 resvg = ["graphene-std/resvg"]
 vello = ["graphene-std/vello", "resvg"]
 ron = ["dep:ron"]

editor/Cargo.toml

@@ -206,11 +206,14 @@ impl Dispatcher {
 					self.message_handlers.preferences_message_handler.process_message(message, &mut queue, ());
 				}
 				Message::Tool(message) => {
-					let document_id = self.message_handlers.portfolio_message_handler.active_document_id().unwrap();
-					let Some(document) = self.message_handlers.portfolio_message_handler.documents.get_mut(&document_id) else {
+					let Some(document_id) = self.message_handlers.portfolio_message_handler.active_document_id() else {
 						warn!("Called ToolMessage without an active document.\nGot {message:?}");
 						return;
 					};
+					let Some(document) = self.message_handlers.portfolio_message_handler.documents.get_mut(&document_id) else {
+						warn!("Called ToolMessage with an invalid active document.\nGot {message:?}");
+						return;
+					};
 
 					let context = ToolMessageContext {
 						document_id,