wasm-rquickjs integration (#289)

* JS templates for using wasm-rquickjs

* Published deps

* Generating d.ts

* TS

* Quick js and ts: up-to-date checks and template fixes

* Make worker oplog streaming

* wasm-rquickjs 0.0.4

* Fix

* Fix

* Fix

* StubGen fix

---------

Co-authored-by: Dávid István Bíró <[email protected]>

Author: vigoo

Cargo.lock

@@ -36,6 +36,7 @@ axum = { version = "0.7.9", features = ["multipart"] }
 base64 = "0.22.1"
 blake3 = "1.5.5"
 bytes = "1.10.1"
+camino = "1.1.10"
 cargo-component = "0.21.1"
 cargo-component-core = "0.21.1"
 cargo_metadata = "0.20.0"
@@ -121,6 +122,7 @@ wax = "0.6.0"
 wasm-metadata = { version = "0.228", features = ["oci"] }
 wasmparser = "0.228.0"
 wasm-encoder = "0.228.0"
+wasm-rquickjs = "0.0.4"
 wit-bindgen-rust = "0.40.0"
 wit-component = "0.228"
 wit-encoder = "0.228"

Cargo.toml

@@ -153,7 +153,7 @@ exec --fail-on-error cargo test --package golem-cli :tag:uses_cargo --test integ
 '''
 
 [tasks.template-integration-tests]
-dependencies = ["install-golem-cli-debug"]
+dependencies = ["build"]
 description = "Run template integration tests"
 script_runner = "@duckscript"
 script = '''

Makefile.toml

@@ -50,6 +50,7 @@ auditable-serde = { version = "0.8.0" }
 base64 = { workspace = true }
 blake3 = { workspace = true }
 bytes = { workspace = true }
+camino = { workspace = true }
 cargo-component = { workspace = true }
 cargo-component-core = { workspace = true }
 cargo_toml = { workspace = true }
@@ -112,6 +113,7 @@ walkdir = { workspace = true }
 wax = { workspace = true }
 wasmparser = { workspace = true }
 wasm-encoder = { workspace = true }
+wasm-rquickjs = { workspace = true }
 wit-bindgen-rust = { workspace = true }
 wit-component = { workspace = true }
 wit-encoder = { workspace = true }

golem-cli/Cargo.toml

@@ -52,8 +52,7 @@ pub fn clean_app(ctx: &ApplicationContext) -> anyhow::Result<()> {
 
                     for build_step in &properties.build {
                         let build_dir = build_step
-                            .dir
-                            .as_ref()
+                            .dir()
                             .map(|dir| {
                                 ctx.application
                                     .component_source_dir(component_name)
@@ -66,7 +65,7 @@ pub fn clean_app(ctx: &ApplicationContext) -> anyhow::Result<()> {
                             });
 
                         paths.extend(
-                            compile_and_collect_globs(&build_dir, &build_step.targets)?
+                            compile_and_collect_globs(&build_dir, &build_step.targets())?
                                 .into_iter()
                                 .map(|path| ("build output", path)),
                         );

golem-cli/src/app/build/clean.rs

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::app::build::task_result_marker::{ResolvedExternalCommandMarkerHash, TaskResultMarker};
+use crate::app::build::task_result_marker::{
+    GenerateQuickJSCrateCommandMarkerHash, GenerateQuickJSDTSCommandMarkerHash,
+    ResolvedExternalCommandMarkerHash, TaskResultMarker,
+};
 use crate::app::build::{delete_path_logged, is_up_to_date, valid_env_vars};
 use crate::app::context::ApplicationContext;
 use crate::app::error::CustomCommandError;
 use crate::fs::compile_and_collect_globs;
 use crate::log::{log_action, log_skipping_up_to_date, LogColorize, LogIndent};
 use crate::model::app_raw;
 use anyhow::{anyhow, Context};
+use camino::Utf8Path;
 use std::collections::HashMap;
 use std::path::Path;
 use std::process::Command;
@@ -90,6 +94,111 @@ pub fn execute_custom_command(
     Ok(())
 }
 
+pub fn execute_build_command(
+    ctx: &ApplicationContext,
+    base_build_dir: &Path,
+    command: &app_raw::BuildCommand,
+    additional_env_vars: HashMap<String, String>,
+) -> anyhow::Result<()> {
+    match command {
+        app_raw::BuildCommand::External(external_command) => {
+            execute_external_command(ctx, base_build_dir, external_command, additional_env_vars)
+        }
+        app_raw::BuildCommand::QuickJSCrate(command) => {
+            let base_build_dir = Utf8Path::from_path(base_build_dir).unwrap();
+            let wit = base_build_dir.join(&command.wit);
+            let js = base_build_dir.join(&command.js);
+            let generate_quickjs_crate = base_build_dir.join(&command.generate_quickjs_crate);
+
+            let task_result_marker = TaskResultMarker::new(
+                &ctx.application.task_result_marker_dir(),
+                GenerateQuickJSCrateCommandMarkerHash {
+                    build_dir: base_build_dir.as_std_path(),
+                    command,
+                },
+            )?;
+
+            let skip_up_to_date_checks =
+                ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date();
+
+            if is_up_to_date(
+                skip_up_to_date_checks,
+                || {
+                    vec![
+                        wit.clone().into_std_path_buf(),
+                        js.clone().into_std_path_buf(),
+                    ]
+                },
+                || vec![generate_quickjs_crate.clone().into_std_path_buf()],
+            ) {
+                log_skipping_up_to_date(format!(
+                    "executing WASM RQuickJS wrapper generator in directory {}",
+                    base_build_dir.log_color_highlight()
+                ));
+                return Ok(());
+            }
+
+            log_action(
+                "Executing",
+                format!(
+                    "WASM RQuickJS wrapper generator in directory {}",
+                    base_build_dir.log_color_highlight()
+                ),
+            );
+
+            task_result_marker.result({
+                wasm_rquickjs::generate_wrapper_crate(
+                    &wit,
+                    &js,
+                    &generate_quickjs_crate,
+                    command.world.as_deref(),
+                )
+            })
+        }
+        app_raw::BuildCommand::QuickJSDTS(command) => {
+            let base_build_dir = Utf8Path::from_path(base_build_dir).unwrap();
+            let wit = &base_build_dir.join(&command.wit);
+            let generate_quickjs_dts = &base_build_dir.join(&command.generate_quickjs_dts);
+
+            let task_result_marker = TaskResultMarker::new(
+                &ctx.application.task_result_marker_dir(),
+                GenerateQuickJSDTSCommandMarkerHash {
+                    build_dir: base_build_dir.as_std_path(),
+                    command,
+                },
+            )?;
+
+            let skip_up_to_date_checks =
+                ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date();
+
+            if is_up_to_date(
+                skip_up_to_date_checks,
+                || vec![wit.clone().into_std_path_buf()],
+                || vec![generate_quickjs_dts.clone().into_std_path_buf()],
+            ) {
+                log_skipping_up_to_date(format!(
+                    "executing WASM RQuickJS d.ts generator in directory {}",
+                    base_build_dir.log_color_highlight()
+                ));
+                return Ok(());
+            }
+
+            log_action(
+                "Executing",
+                format!(
+                    "WASM RQuickJS d.ts generator in directory {}",
+                    base_build_dir.log_color_highlight()
+                ),
+            );
+
+            task_result_marker.result({
+                wasm_rquickjs::generate_dts(wit, generate_quickjs_dts, command.world.as_deref())
+                    .context("Failed to generate QuickJS DTS")
+            })
+        }
+    }
+}
+
 pub fn execute_external_command(
     ctx: &ApplicationContext,
     base_build_dir: &Path,

…em-cli/src/app/build/external_command.rs golem-cli/src/app/build/command.rsgolem-cli/src/app/build/external_command.rs renamed to golem-cli/src/app/build/command.rs golem-cli/src/app/build/external_command.rs renamed to golem-cli/src/app/build/command.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::app::build::external_command::execute_external_command;
+use crate::app::build::command::execute_build_command;
 use crate::app::context::ApplicationContext;
 use crate::log::{log_action, log_warn_action, LogColorize, LogIndent};
 use crate::model::app::{AppComponentName, DependencyType};
@@ -52,7 +52,7 @@ pub fn componentize(ctx: &mut ApplicationContext) -> anyhow::Result<()> {
             .context("Failed to get env vars for build step")?;
 
         for build_step in &component_properties.build {
-            execute_external_command(
+            execute_build_command(
                 ctx,
                 ctx.application.component_source_dir(&component_name),
                 build_step,

golem-cli/src/app/build/componentize.rs

@@ -25,15 +25,16 @@ use chrono::{DateTime, Utc};
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::ffi::OsString;
+use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use std::time::SystemTime;
 use tracing::debug;
 use walkdir::WalkDir;
 
 pub mod add_metadata;
 pub mod clean;
+pub mod command;
 pub mod componentize;
-pub mod external_command;
 pub mod gen_rpc;
 pub mod link;
 pub mod task_result_marker;
@@ -114,8 +115,8 @@ fn delete_path_logged(context: &str, path: &Path) -> anyhow::Result<()> {
 
 fn is_up_to_date<S, T, FS, FT>(skip_check: bool, sources: FS, targets: FT) -> bool
 where
-    S: IntoIterator<Item = PathBuf>,
-    T: IntoIterator<Item = PathBuf>,
+    S: Debug + IntoIterator<Item = PathBuf>,
+    T: Debug + IntoIterator<Item = PathBuf>,
     FS: FnOnce() -> S,
     FT: FnOnce() -> T,
 {
@@ -167,6 +168,7 @@ where
     }
 
     let targets = targets();
+    debug!(targets=?targets, "collected targets");
 
     let max_target_modified = max_modified_short_circuit_on_missing(targets);
 
@@ -179,6 +181,7 @@ where
     };
 
     let sources = sources();
+    debug!(source=?sources, "collected sources");
 
     let max_source_modified = max_modified_short_circuit_on_missing(sources);
 

golem-cli/src/app/build/mod.rs

@@ -16,6 +16,7 @@ use crate::app::build::task_result_marker::TaskResultMarkerHashSourceKind::{Hash
 use crate::fs;
 use crate::log::log_warn_action;
 use crate::model::app::{AppComponentName, DependentComponent};
+use crate::model::app_raw::{GenerateQuickJSCrate, GenerateQuickJSDTS};
 use crate::model::ProjectId;
 use crate::model::{app_raw, ComponentName};
 use anyhow::{anyhow, bail, Context};
@@ -92,6 +93,46 @@ impl TaskResultMarkerHashSource for ResolvedExternalCommandMarkerHash<'_> {
     }
 }
 
+#[derive(Serialize)]
+pub struct GenerateQuickJSCrateCommandMarkerHash<'a> {
+    pub build_dir: &'a Path,
+    pub command: &'a GenerateQuickJSCrate,
+}
+
+impl TaskResultMarkerHashSource for GenerateQuickJSCrateCommandMarkerHash<'_> {
+    fn kind() -> &'static str {
+        "GenerateQuickJSCrateCommandMarkerHash"
+    }
+
+    fn id(&self) -> anyhow::Result<Option<String>> {
+        Ok(None)
+    }
+
+    fn source(&self) -> anyhow::Result<TaskResultMarkerHashSourceKind> {
+        Ok(HashFromString(serde_json::to_string(self)?))
+    }
+}
+
+#[derive(Serialize)]
+pub struct GenerateQuickJSDTSCommandMarkerHash<'a> {
+    pub build_dir: &'a Path,
+    pub command: &'a GenerateQuickJSDTS,
+}
+
+impl TaskResultMarkerHashSource for GenerateQuickJSDTSCommandMarkerHash<'_> {
+    fn kind() -> &'static str {
+        "GenerateQuickJSDTSCommandMarkerHash"
+    }
+
+    fn id(&self) -> anyhow::Result<Option<String>> {
+        Ok(None)
+    }
+
+    fn source(&self) -> anyhow::Result<TaskResultMarkerHashSourceKind> {
+        Ok(HashFromString(serde_json::to_string(self)?))
+    }
+}
+
 pub struct ComponentGeneratorMarkerHash<'a> {
     pub component_name: &'a AppComponentName,
     pub generator_kind: &'a str,

golem-cli/src/app/build/task_result_marker.rs

@@ -14,7 +14,7 @@
 
 use crate::app::build::build_app;
 use crate::app::build::clean::clean_app;
-use crate::app::build::external_command::execute_custom_command;
+use crate::app::build::command::execute_custom_command;
 use crate::app::error::{format_warns, AppValidationError, CustomCommandError};
 use crate::app::remote_components::RemoteComponents;
 use crate::config::ProfileName;