diff --git a/Cargo.lock b/Cargo.lock index 6fbc873..14a69f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,6 +329,13 @@ dependencies = [ "num", ] +[[package]] +name = "nix-wasm-plugin-nix-make" +version = "0.1.0" +dependencies = [ + "nix-wasm-rust", +] + [[package]] name = "nix-wasm-plugin-quickjs" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bfb6449..34e30ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "nix-wasm-plugin-test", "nix-wasm-plugin-quickjs", "nix-wasm-plugin-grep", + "nix-wasm-plugin-nix-make", ] resolver = "2" diff --git a/flake.nix b/flake.nix index 29586fc..403a348 100644 --- a/flake.nix +++ b/flake.nix @@ -60,7 +60,7 @@ ''; workspaceVendor = rustPlatform.fetchCargoVendor { src = self; - hash = "sha256-vkTdv3StxslmBOKy8mFfz5afOiMjBujFd4IU6pkgqGc="; + hash = "sha256-7+qf/W+ZAPWWghAzF33RDBLwZUrA51USjkGujXBRF4U="; }; stdlibVendor = rustPlatform.fetchCargoVendor { src = rustPlatform.rustcSrc; diff --git a/nix-wasm-plugin-nix-make/Cargo.toml b/nix-wasm-plugin-nix-make/Cargo.toml new file mode 100644 index 0000000..394a1d3 --- /dev/null +++ b/nix-wasm-plugin-nix-make/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nix-wasm-plugin-nix-make" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +nix-wasm-rust = { path = "../nix-wasm-rust" } diff --git a/nix-wasm-plugin-nix-make/demo.nix b/nix-wasm-plugin-nix-make/demo.nix new file mode 100644 index 0000000..5576f1c --- /dev/null +++ b/nix-wasm-plugin-nix-make/demo.nix @@ -0,0 +1,180 @@ +with import {}; + +rec { + getDeps = builtins.wasm { + path = ../target/wasm32-unknown-unknown/release/nix_wasm_plugin_nix_make.wasm; + function = "getDeps"; + }; + + compileCpp = source: runCommandCC + "${builtins.baseNameOf source.path}.o" + { + __structuredAttrs = true; + includes = source.includes; + srcPath = source.path; + src = source.src; + buildInputs = map (dep: pkgs'.${dep}) source.deps; + inherit (source) deps; + } + '' + for name in "''${!includes[@]}"; do + mkdir -p "$(dirname "$name")" + ln -s "''${includes[$name]}" "$name" + done + + srcDir="$(dirname "$srcPath")" + mkdir -p "$srcDir" + ln -s "$src" "$srcPath" + + mkdir -p "$out/$srcDir" + # FIXME: figure out the -I flags automatically. + gcc -std=c++23 -O1 -c "$srcPath" -o "$out/$srcDir/$(basename "$srcPath").o" -I . -I include -I unix/include -I linux/include -I windows/include -I widecharwidth + ''; + + link = name: objects: runCommandCC + name + { + inherit objects; + buildInputs = map (dep: pkgs'.${dep}) (builtins.concatLists (map (obj: obj.deps) objects)); + } + '' + mkdir -p $out/lib + g++ -o $out/lib/$name.so \ + $(find $objects -name '*.o' -type f) \ + -lboost_context -lboost_iostreams -lboost_url -larchive -lcrypto -lsodium -lblake3 -lbrotlicommon -lbrotlienc -lbrotlidec -lcpuid -shared + ''; + + sources = getDeps { + inherit builtins; + dirs = [ + { root = /home/eelco/Dev/nix/src/libutil; + prefix = ""; + } + #{ root = /home/eelco/Dev/nix/src/libstore; + # prefix = ""; + #} + ]; + files = { + "nix/store/config.hh" = builtins.toFile "config.hh" + '' + #pragma once + #define NIX_LOCAL_SYSTEM "x86_64-linux" + #define NIX_SUPPORT_ACL 1 + #define NIX_WITH_AWS_AUTH 1 + ''; + "util-config-private.hh" = builtins.toFile "util-config-private.hh" + '' + #pragma once + #define HAVE_LIBCPUID 1 + #define HAVE_POSIX_FALLOCATE 1 + ''; + "store-config-private.hh" = pkgs.writeText "store-config-private.hh" + '' + #pragma once + #define CAN_LINK_SYMLINK 1 + #define DETERMINATE_NIX_VERSION "3.16.3" + #define HAVE_EMBEDDED_SANDBOX_SHELL 0 + #define HAVE_LCHOWN 1 + #define HAVE_POSIX_FALLOCATE 1 + #define HAVE_SECCOMP 1 + #define HAVE_STATVFS 1 + #undef IS_STATIC + #define LSOF "lsof" + #define NIX_CONF_DIR "/etc/nix" + #define NIX_DATA_DIR "/home/eelco/Dev/nix/outputs/out/share" + #define NIX_LOG_DIR "/nix/var/log/nix" + #define NIX_MAN_DIR "/home/eelco/Dev/nix/outputs/out/share/man" + #define NIX_PREFIX "/home/eelco/Dev/nix/outputs/out" + #define NIX_STATE_DIR "/nix/var/nix" + #define NIX_STORE_DIR "/nix/store" + #define NIX_USE_WASMTIME 1 + #define PACKAGE_VERSION "2.33.3" + #define SANDBOX_SHELL "${pkgs.busybox}/bin/busybox" + ''; + "util-unix-config-private.hh" = builtins.toFile "util-unix-config-private.hh" + '' + #pragma once + #define HAVE_CLOSE_RANGE 1 + #define HAVE_DECL_AT_SYMLINK_NOFOLLOW 1 + #define HAVE_LUTIMES 1 + #define HAVE_PIPE2 1 + #define HAVE_STRSIGNAL 1 + #define HAVE_SYSCONF 1 + #define HAVE_UTIMENSAT 1 + ''; + }; + }; + + allSources = getDeps { + inherit builtins; + dirs = [ + { root = /home/eelco/Dev/nix/src; + prefix = ""; + } + #{ root = /home/eelco/Dev/nix/src/libstore; + # prefix = ""; + #} + ]; + files = { + "nix/store/config.hh" = builtins.toFile "config.hh" + '' + #pragma once + #define NIX_LOCAL_SYSTEM "x86_64-linux" + #define NIX_SUPPORT_ACL 1 + #define NIX_WITH_AWS_AUTH 1 + ''; + "util-config-private.hh" = builtins.toFile "util-config-private.hh" + '' + #pragma once + #define HAVE_LIBCPUID 1 + #define HAVE_POSIX_FALLOCATE 1 + ''; + "store-config-private.hh" = pkgs.writeText "store-config-private.hh" + '' + #pragma once + #define CAN_LINK_SYMLINK 1 + #define DETERMINATE_NIX_VERSION "3.16.3" + #define HAVE_EMBEDDED_SANDBOX_SHELL 0 + #define HAVE_LCHOWN 1 + #define HAVE_POSIX_FALLOCATE 1 + #define HAVE_SECCOMP 1 + #define HAVE_STATVFS 1 + #undef IS_STATIC + #define LSOF "lsof" + #define NIX_CONF_DIR "/etc/nix" + #define NIX_DATA_DIR "/home/eelco/Dev/nix/outputs/out/share" + #define NIX_LOG_DIR "/nix/var/log/nix" + #define NIX_MAN_DIR "/home/eelco/Dev/nix/outputs/out/share/man" + #define NIX_PREFIX "/home/eelco/Dev/nix/outputs/out" + #define NIX_STATE_DIR "/nix/var/nix" + #define NIX_STORE_DIR "/nix/store" + #define NIX_USE_WASMTIME 1 + #define PACKAGE_VERSION "2.33.3" + #define SANDBOX_SHELL "${pkgs.busybox}/bin/busybox" + ''; + "util-unix-config-private.hh" = builtins.toFile "util-unix-config-private.hh" + '' + #pragma once + #define HAVE_CLOSE_RANGE 1 + #define HAVE_DECL_AT_SYMLINK_NOFOLLOW 1 + #define HAVE_LUTIMES 1 + #define HAVE_PIPE2 1 + #define HAVE_STRSIGNAL 1 + #define HAVE_SYSCONF 1 + #define HAVE_UTIMENSAT 1 + ''; + }; + }; + + + pkgs' = pkgs // { + libcpuid = pkgs.runCommand "libcpuid" { inherit (pkgs) libcpuid; } + '' + ln -s $libcpuid $out + ''; + }; + + all = map compileCpp sources; + + libutil = link "libnixutil.so" (map compileCpp sources); +} diff --git a/nix-wasm-plugin-nix-make/src/lib.rs b/nix-wasm-plugin-nix-make/src/lib.rs new file mode 100644 index 0000000..71613e4 --- /dev/null +++ b/nix-wasm-plugin-nix-make/src/lib.rs @@ -0,0 +1,225 @@ +use nix_wasm_rust::{warn, Value}; +use std::collections::{HashMap, HashSet}; + +struct FileInfo { + path: String, + sys_includes: Vec, + user_includes: Vec, +} + +#[no_mangle] +pub extern "C" fn getDeps(args: Value) -> Value { + let builtins = args + .get_attr("builtins") + .expect("missing 'builtins' argument"); + let dirs = args.get_attr("dirs").expect("missing 'dirs' argument"); + + let read_dir = builtins.get_attr("readDir").unwrap(); + + // First pass: scan all .cc and .hh files, recording their direct includes. + let mut all_files: HashMap = HashMap::new(); + for entry in dirs.get_list() { + let root = entry.get_attr("root").expect("missing 'root' attribute"); + let prefix = entry + .get_attr("prefix") + .expect("missing 'prefix' attribute") + .get_string(); + scan_files(&read_dir, &root, &prefix, &mut all_files); + } + + // Process explicit files: each key is a path (possibly with slashes), + // the value is the file's path value. + if let Some(files) = args.get_attr("files") { + for (name, file_val) in files.get_attrset() { + let contents = file_val.read_file(); + let file_info = extract_includes(name, &contents); + all_files.insert(file_val, file_info); + } + } + + // Build a suffix map for efficient include resolution. + // For a file "foo/bar/xyzzy/util.hh", this inserts: + // "util.hh" -> "foo/bar/xyzzy/util.hh" + // "xyzzy/util.hh" -> "foo/bar/xyzzy/util.hh" + // "bar/xyzzy/util.hh" -> "foo/bar/xyzzy/util.hh" + // "foo/bar/xyzzy/util.hh" -> "foo/bar/xyzzy/util.hh" + let mut suffix_map: HashMap = HashMap::new(); + for (value, file) in &all_files { + let rel_path = &file.path; + let parts: Vec<&str> = rel_path.split('/').collect(); + for i in 0..parts.len() { + let suffix = parts[i..].join("/"); + suffix_map.entry(suffix).or_insert_with(|| *value); + } + } + + // Second pass: for each .cc file, compute the transitive closure of includes. + let mut results = vec![]; + for (cc_file_val, cc_file) in all_files.iter() { + if !cc_file.path.ends_with(".cc") { + continue; + } + //warn!("processing {path}...", path = all_files[cc_file_val].path); + let mut all_includes: HashMap = HashMap::new(); + let mut all_deps: HashSet = HashSet::new(); + let mut visited = HashSet::new(); + collect_transitive_includes( + *cc_file_val, + &all_files, + &suffix_map, + &mut all_includes, + &mut all_deps, + &mut visited, + ); + + let include_attrs: Vec<(&str, Value)> = all_includes + .values() + .map(|path_val| { + let inc_file = &all_files[path_val]; + (inc_file.path.as_str(), *path_val) + }) + .collect(); + let entry = Value::make_attrset(&[ + ("src", *cc_file_val), + ("path", Value::make_string(&cc_file.path)), + ("includes", Value::make_attrset(&include_attrs)), + ( + "deps", + Value::make_list( + &all_deps + .iter() + .map(|s| Value::make_string(s)) + .collect::>(), + ), + ), + ]); + results.push(entry); + } + + Value::make_list(&results) +} + +fn scan_files( + read_dir: &Value, + path_val: &Value, + prefix: &str, + all_files: &mut HashMap, +) { + for (name, file_type) in read_dir.call(&[*path_val]).get_attrset() { + if name == "windows" { + continue; + } // FIXME: hack + let child = path_val.make_path(&name); + let path = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}/{name}") + }; + let file_type = file_type.get_string(); + match file_type.as_str() { + "regular" => { + if name.ends_with(".cc") + || name.ends_with(".hh") + || name.ends_with(".h") + || name.ends_with(".sb") + || name.ends_with(".md") + { + let contents = child.read_file(); + let file_info = extract_includes(path, &contents); + all_files.insert(child, file_info); + } + } + "directory" => { + scan_files(read_dir, &child, &path, all_files); + } + _ => {} + } + } +} + +fn collect_transitive_includes( + file: Value, + all_files: &HashMap, + suffix_map: &HashMap, + all_includes: &mut HashMap, + all_deps: &mut HashSet, + visited: &mut HashSet, +) { + if !visited.insert(file) { + return; + } + + let file = &all_files[&file]; + + for inc in &file.user_includes { + if let Some(resolved) = suffix_map.get(inc) { + all_includes.entry(inc.clone()).or_insert(*resolved); + collect_transitive_includes( + *resolved, + all_files, + suffix_map, + all_includes, + all_deps, + visited, + ); + } else { + // FIXME: hack + if !inc.contains("windows") { + warn!("{file}: included file not found: {inc}", file = file.path); + } + } + } + + for include in &file.sys_includes { + if include.starts_with("boost/") { + all_deps.insert("boost".to_string()); + } else if include.starts_with("brotli/") { + all_deps.insert("brotli".to_string()); + } else if include.starts_with("openssl/") { + all_deps.insert("openssl".to_string()); + } else if include == "archive.h" { + all_deps.insert("libarchive".to_string()); + } else if include == "sodium.h" { + all_deps.insert("libsodium".to_string()); + } else if include == "blake3.h" { + all_deps.insert("libblake3".to_string()); + } else if include.starts_with("nlohmann/json") { + all_deps.insert("nlohmann_json".to_string()); + } else if include.starts_with("libcpuid/") { + all_deps.insert("libcpuid".to_string()); + } + } +} + +fn extract_includes(path: String, contents: &[u8]) -> FileInfo { + let mut file_info = FileInfo { + path, + sys_includes: vec![], + user_includes: vec![], + }; + let Ok(text) = std::str::from_utf8(contents) else { + return file_info; + }; + // FIXME: process #ifdefs so we can skip #includes that don't apply + for line in text.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix('#') { + let rest = rest.trim_start(); + let Some(rest) = rest.strip_prefix("include") else { + continue; + }; + let rest = rest.trim(); + if let Some(path) = rest.strip_prefix('"') { + if let Some(path) = path.strip_suffix('"') { + file_info.user_includes.push(path.to_string()); + } + } + if let Some(path) = rest.strip_prefix('<') { + if let Some(path) = path.strip_suffix('>') { + file_info.sys_includes.push(path.to_string()); + } + } + } + } + file_info +} diff --git a/nix-wasm-rust/src/lib.rs b/nix-wasm-rust/src/lib.rs index 9d39be6..c0e0823 100644 --- a/nix-wasm-rust/src/lib.rs +++ b/nix-wasm-rust/src/lib.rs @@ -31,7 +31,7 @@ macro_rules! warn { // FIXME: use externref for Values? #[repr(transparent)] -#[derive(Clone, Debug, Copy)] +#[derive(Clone, Debug, Copy, Eq, PartialEq, Hash)] pub struct Value(ValueId); type ValueId = u32;