Merge branch 'main' into tm/return-type-async

Author: Sl1mb0

Cargo.lock

@@ -31,6 +31,7 @@ datafusion-udf-wasm-query = { path = "query", version = "0.1.0" }
 http = { version = "1.3.1", default-features = false }
 hyper = { version = "1.8", default-features = false }
 insta = { version = "1.43.2", "default-features" = false }
+log = { version = "0.4.28", default-features = false }
 pyo3 = { version = "0.27.1", default-features = false, features = ["macros"] }
 sqlparser = { version = "0.55.0", default-features = false, features = [
   "std",
@@ -54,14 +55,22 @@ wit-bindgen = { version = "0.47", default-features = false, features = [
 ] }
 
 [workspace.lints.clippy]
-allow_attributes = "warn"
+allow_attributes = "deny"
 clone_on_ref_ptr = "deny"
+comparison_chain = "deny"
 dbg_macro = "deny"
 explicit_iter_loop = "deny"
 future_not_send = "deny"
 missing_docs_in_private_items = "deny"
+range_minus_one = "deny"
+range_plus_one = "deny"
+semicolon_if_nothing_returned = "deny"
 todo = "deny"
 undocumented_unsafe_blocks = "deny"
+unnecessary_semicolon = "deny"
+unnecessary_wraps = "deny"
+unused_async = "deny"
+unused_self = "deny"
 use_self = "deny"
 
 [workspace.lints.rust]

Cargo.toml

@@ -3,12 +3,15 @@
 
 use std::{
     collections::HashMap,
+    fs::File,
+    io::Write,
     path::{Path, PathBuf},
     process::{Command, Stdio},
     str::FromStr,
 };
 
 fn main() {
+    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
     let profile: Profile = std::env::var("PROFILE").unwrap().parse().unwrap();
     let package_locations = package_locations();
 
@@ -28,10 +31,16 @@ fn main() {
         || std::env::var("DOCS_RS").is_ok()
         || std::env::var("JUSTCHECK").is_ok();
 
+    let mut gen_file = File::create(out_dir.join("gen.rs")).unwrap();
+
     for feature in FEATURES {
         println!("processing {}", feature.name);
-        feature.build_or_stub(stub, profile, &package_locations);
+        feature.build_or_stub(stub, profile, &package_locations, &out_dir, &mut gen_file);
     }
+
+    gen_file.flush().unwrap();
+
+    println!("cargo::rerun-if-changed=build.rs");
 }
 
 /// Get locations for all packages in the dependency tree.
@@ -151,6 +160,27 @@ impl std::fmt::Display for Profile {
     }
 }
 
+/// Artifact type.
+enum ArtifactType {
+    /// Library.
+    Lib,
+
+    /// Example.
+    Example(&'static str),
+}
+
+/// Just(file) command.
+struct JustCmd {
+    /// Artifact type.
+    artifact_type: ArtifactType,
+
+    /// Name of the resulting constant.
+    const_name: &'static str,
+
+    /// Documentation for the created constant.
+    doc: &'static str,
+}
+
 /// Feature description.
 struct Feature {
     /// Lowercase feature name.
@@ -159,15 +189,8 @@ struct Feature {
     /// Package that contains the feature code.
     package: &'static str,
 
-    /// `just` command prefix that compiles the feature.
-    ///
-    /// This will call `just prefix{profile}` within the package directory.
-    just_cmd_prefix: &'static str,
-
-    /// Path components to file in target directory.
-    ///
-    /// So `["foo", "bar.bin"]` will resolve to `CARGO_TARGET_DIR/wasm32-wasip2/foo/bar.bin`.
-    just_out_file: &'static [&'static str],
+    /// Just commands.
+    just_cmds: &'static [JustCmd],
 }
 
 impl Feature {
@@ -177,12 +200,13 @@ impl Feature {
         stub: bool,
         profile: Profile,
         package_locations: &HashMap<String, PathBuf>,
+        out_dir: &Path,
+        gen_file: &mut File,
     ) {
         let Self {
             name,
             package,
-            just_cmd_prefix,
-            just_out_file,
+            just_cmds,
         } = self;
 
         let name_upper = name.to_uppercase();
@@ -191,32 +215,61 @@ impl Feature {
             return;
         }
 
-        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-
-        let out_file = if stub {
-            let out_file = out_dir.join(format!("{name}.wasm"));
-            // write empty stub file
-            std::fs::write(&out_file, b"").unwrap();
-            out_file
-        } else {
-            let target_dir = out_dir.join(name);
+        let cwd = package_locations.get(*package).unwrap();
+        let target_dir = out_dir.join(name);
+
+        for just_cmd in *just_cmds {
+            let JustCmd {
+                artifact_type,
+                const_name,
+                doc,
+            } = just_cmd;
+            let out_file = if stub {
+                let out_file = out_dir.join(format!("{name}.wasm"));
+                // write empty stub file
+                std::fs::write(&out_file, b"").unwrap();
+                out_file
+            } else {
+                let mut just_cmd = "build-".to_owned();
+                match artifact_type {
+                    ArtifactType::Lib => {}
+                    ArtifactType::Example(example) => {
+                        just_cmd.push_str(example);
+                        just_cmd.push('-');
+                    }
+                }
+                just_cmd.push_str(profile.as_str());
+
+                just_build(cwd, &just_cmd, &target_dir);
+
+                let out = target_dir.join("wasm32-wasip2").join(profile.as_str());
+                match artifact_type {
+                    ArtifactType::Lib => out.join(format!("{}.wasm", package.replace("-", "_"))),
+                    ArtifactType::Example(example) => out
+                        .join("examples")
+                        .join(format!("{}.wasm", example.replace("-", "_"))),
+                }
+            };
 
-            just_build(
-                package_locations.get(*package).unwrap(),
-                &format!("{just_cmd_prefix}{profile}"),
-                &target_dir,
+            println!(
+                "cargo::rustc-env=BIN_PATH_{const_name}={}",
+                out_file.display(),
             );
 
-            just_out_file.iter().fold(
-                target_dir.join("wasm32-wasip2").join(profile.as_str()),
-                |path, part| path.join(part),
-            )
-        };
-
-        println!(
-            "cargo::rustc-env=BIN_PATH_{name_upper}={}",
-            out_file.display(),
-        );
+            writeln!(gen_file, "/// {doc}").unwrap();
+            writeln!(gen_file, r#"#[cfg(feature = "{name}")]"#).unwrap();
+            writeln!(gen_file, r#"pub static BIN_{const_name}: &[u8] = include_bytes!(env!("BIN_PATH_{const_name}"));"#).unwrap();
+
+            // we cannot really depend directly on examples, so we need to tell Cargo about it
+            if let ArtifactType::Example(example) = artifact_type {
+                println!(
+                    "cargo::rerun-if-changed={}",
+                    cwd.join("examples")
+                        .join(format!("{}.rs", example.replace("-", "_")))
+                        .display(),
+                );
+            }
+        }
     }
 }
 
@@ -232,18 +285,24 @@ fn just_build(cwd: &Path, just_cmd: &str, cargo_target_dir: &Path) {
 
 /// All supported features.
 ///
-/// This must be in-sync with the feature list in `Cargo.toml` and the imports in `src/lib.rs`.
+/// This must be in-sync with the feature list in `Cargo.toml`.
 const FEATURES: &[Feature] = &[
     Feature {
         name: "example",
         package: "datafusion-udf-wasm-guest",
-        just_cmd_prefix: "build-add-one-",
-        just_out_file: &["examples", "add_one.wasm"],
+        just_cmds: &[JustCmd {
+            artifact_type: ArtifactType::Example("add-one"),
+            const_name: "EXAMPLE",
+            doc: r#""add-one" example."#,
+        }],
     },
     Feature {
         name: "python",
         package: "datafusion-udf-wasm-python",
-        just_cmd_prefix: "",
-        just_out_file: &["datafusion_udf_wasm_python.wasm"],
+        just_cmds: &[JustCmd {
+            artifact_type: ArtifactType::Lib,
+            const_name: "PYTHON",
+            doc: "Python UDF.",
+        }],
     },
 ];

guests/bundle/build.rs

@@ -1,9 +1,3 @@
 //! Bundles guests as pre-compiled WASM bytecode.
 
-/// "add-one" example.
-#[cfg(feature = "example")]
-pub static BIN_EXAMPLE: &[u8] = include_bytes!(env!("BIN_PATH_EXAMPLE"));
-
-/// Python UDF.
-#[cfg(feature = "python")]
-pub static BIN_PYTHON: &[u8] = include_bytes!(env!("BIN_PATH_PYTHON"));
+include!(concat!(env!("OUT_DIR"), "/gen.rs"));

guests/bundle/src/lib.rs

@@ -163,13 +163,13 @@ build-lib profile: download-python-sdk download-wasi-sdk python-site-packages
 do profile: (build-lib profile)
 
 # create dev/debug build
-debug: (do "debug")
+build-debug: (do "debug")
 
 # create release build
-release: (do "release")
+build-release: (do "release")
 
 # checks build
-check-build: debug
+check-build: build-debug
 
 # clean build artifacts
 clean:

guests/python/Justfile

@@ -4,13 +4,13 @@
 Use:
 
 ```console
-just debug
+just build-debug
 ```
 
 or
 
 ```console
-just release
+just build-release
 ```
 
 ## Python Version
@@ -197,6 +197,9 @@ def compute(x: int) -> int:
 ## I/O
 All I/O operations go through the host, there is no direct interaction with the host operating system.
 
+### Environment Variables
+Hosts can pass environment variables to the guest if they want. By default, NO variables are available for the guest though (i.e. there is NO implicit pass-through). The standard Python library can read these environment variables, e.g. via [`os.environ`].
+
 ### Filesystem
 The [Python Standard Library] is mounted as a read-only filesystem. The host file system (incl. special paths like `/proc`) are NOT exposed to the guest.
 
@@ -234,6 +237,7 @@ There is NO other I/O available that escapes the sandbox.
 [`timedelta`]: https://docs.python.org/3/library/datetime.html#datetime.timedelta
 [`Duration`]: https://docs.rs/arrow/latest/arrow/datatypes/enum.DataType.html#variant.Duration
 [`Microsecond`]: https://docs.rs/arrow/latest/arrow/datatypes/enum.TimeUnit.html#variant.Microsecond
+[`os.environ`]: https://docs.python.org/3/library/os.html#os.environ
 [Python 3.14.0]: https://www.python.org/downloads/release/python-3140
 [Python Standard Library]: https://docs.python.org/3/library/index.html
 [`requests`]: https://pypi.org/project/requests/

guests/python/README.md

@@ -102,6 +102,7 @@ fn root() -> Option<Vec<u8>> {
 /// Returns our one example UDF.
 ///
 /// The passed `source` is ignored.
+#[expect(clippy::unnecessary_wraps, reason = "public API through export! macro")]
 fn udfs(_source: String) -> DataFusionResult<Vec<Arc<dyn ScalarUDFImpl>>> {
     Ok(vec![Arc::new(AddOne::default())])
 }

guests/rust/examples/add_one.rs

@@ -12,6 +12,7 @@ datafusion-expr.workspace = true
 datafusion-udf-wasm-arrow2bytes.workspace = true
 http.workspace = true
 hyper.workspace = true
+log.workspace = true
 rand = { version = "0.9" }
 siphasher = { version = "1", default-features = false }
 tar.workspace = true