Integrate wit-component into rbwasm command

We no longer need `wasm-tools` cli

Author: kateinoigakukun

Cargo.lock

@@ -14,3 +14,4 @@ magnus = "0.6.2"
 wizer = "4.0.0"
 wasi-vfs-cli = { git = "https://github.com/kateinoigakukun/wasi-vfs/", tag = "0.5.2" }
 structopt = "0.3.26"
+wit-component = "0.202.0"

ext/ruby_wasm/Cargo.toml

@@ -6,8 +6,8 @@ use magnus::{
     value::{self, InnerValue},
     wrap, Error, ExceptionClass, RModule, Ruby,
 };
-use wizer::Wizer;
 use structopt::StructOpt;
+use wizer::Wizer;
 
 static RUBY_WASM: value::Lazy<RModule> =
     value::Lazy::new(|ruby| ruby.define_module("RubyWasmExt").unwrap());
@@ -64,6 +64,83 @@ impl WasiVfs {
     }
 }
 
+#[wrap(class = "RubyWasmExt::ComponentLink")]
+struct ComponentLink(std::cell::RefCell<Option<wit_component::Linker>>);
+
+impl ComponentLink {
+    fn new() -> Self {
+        Self(std::cell::RefCell::new(Some(wit_component::Linker::default())))
+    }
+    fn linker(&self, body: impl FnOnce(wit_component::Linker) -> Result<wit_component::Linker, Error>) -> Result<(), Error> {
+        let mut linker = self.0.take().ok_or_else(|| {
+            Error::new(
+                exception::standard_error(),
+                "linker is already consumed".to_string(),
+            )
+        })?;
+        linker = body(linker)?;
+        self.0.replace(Some(linker));
+        Ok(())
+    }
+
+    fn library(&self, name: String, module: Vec<u8>, dl_openable: bool) -> Result<(), Error> {
+        self.linker(|linker| {
+            linker.library(&name, &module, dl_openable).map_err(|e| {
+                Error::new(
+                    exception::standard_error(),
+                    format!("failed to link library: {}", e),
+                )
+            })
+        })
+    }
+    fn adapter(&self, name: String, module: Vec<u8>) -> Result<(), Error> {
+        self.linker(|linker| {
+            linker.adapter(&name, &module).map_err(|e| {
+                Error::new(
+                    exception::standard_error(),
+                    format!("failed to link adapter: {}", e),
+                )
+            })
+        })
+    }
+    fn validate(&self, validate: bool) -> Result<(), Error> {
+        self.linker(|linker| {
+            Ok(linker.validate(validate))
+        })
+    }
+    fn stack_size(&self, size: u32) -> Result<(), Error> {
+        self.linker(|linker| {
+            Ok(linker.stack_size(size))
+        })
+    }
+    fn stub_missing_functions(&self, stub: bool) -> Result<(), Error> {
+        self.linker(|linker| {
+            Ok(linker.stub_missing_functions(stub))
+        })
+    }
+    fn use_built_in_libdl(&self, use_libdl: bool) -> Result<(), Error> {
+        self.linker(|linker| {
+            Ok(linker.use_built_in_libdl(use_libdl))
+        })
+    }
+    fn encode(&self) -> Result<Vec<u8>, Error> {
+        // Take the linker out of the cell and consume it
+        let linker = self.0.borrow_mut().take().ok_or_else(|| {
+            Error::new(
+                exception::standard_error(),
+                "linker is already consumed".to_string(),
+            )
+        })?;
+        let encoded = linker.encode().map_err(|e| {
+            Error::new(
+                exception::standard_error(),
+                format!("failed to encode linker: {}", e),
+            )
+        })?;
+        Ok(encoded)
+    }
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = RUBY_WASM.get_inner_with(ruby);
@@ -76,5 +153,16 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     wasi_vfs.define_singleton_method("run_cli", function!(WasiVfs::run_cli, 1))?;
     wasi_vfs.define_method("map_dir", method!(WasiVfs::map_dir, 2))?;
     wasi_vfs.define_method("pack", method!(WasiVfs::pack, 1))?;
+
+    let component_link = module.define_class("ComponentLink", ruby.class_object())?;
+    component_link.define_singleton_method("new", function!(ComponentLink::new, 0))?;
+    component_link.define_method("library", method!(ComponentLink::library, 3))?;
+    component_link.define_method("adapter", method!(ComponentLink::adapter, 2))?;
+    component_link.define_method("validate", method!(ComponentLink::validate, 1))?;
+    component_link.define_method("stack_size", method!(ComponentLink::stack_size, 1))?;
+    component_link.define_method("stub_missing_functions", method!(ComponentLink::stub_missing_functions, 1))?;
+    component_link.define_method("use_built_in_libdl", method!(ComponentLink::use_built_in_libdl, 1))?;
+    component_link.define_method("encode", method!(ComponentLink::encode, 0))?;
+
     Ok(())
 }

ext/ruby_wasm/src/lib.rs

@@ -27,9 +27,7 @@ def package(executor, dest_dir, options)
 
     fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
     fs.package_ruby_root(tarball, executor)
-
-    ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
-    wasm_bytes = File.binread(ruby_wasm_bin).bytes
+    wasm_bytes = ruby_core.build_and_link_exts(executor)
 
     fs.package_gems
     fs.remove_non_runtime_files(executor)

lib/ruby_wasm/packager.rb

@@ -12,7 +12,7 @@ def build(executor, options)
 
   extend Forwardable
 
-  def_delegators :build_strategy, :cache_key, :artifact
+  def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
 
   private
 
@@ -37,7 +37,7 @@ def build(executor, options)
       raise NotImplementedError
     end
 
-    def build_and_link_exts(executor, output)
+    def build_and_link_exts(executor)
     end
 
     # Array of paths to extconf.rb files.
@@ -89,27 +89,23 @@ def build(executor, options)
         else
           do_build.call
         end
-      self.build_exts(executor, build)
       build.crossruby.artifact
     end
 
-    def build_and_link_exts(executor, output)
+    def build_and_link_exts(executor)
       build = derive_build
       self.build_exts(executor, build)
-      self.link_exts(executor, build, output)
+      self.link_exts(executor, build)
     end
 
-    def link_exts(executor, build, output)
-      wasm_tools = ENV["WASM_TOOLS"] || "wasm-tools"
+    def link_exts(executor, build)
       ruby_root = build.crossruby.dest_dir
 
-      link_args = %W[component link]
-      link_args << File.join(ruby_root, "usr", "local", "bin", "ruby")
+      libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
 
       # TODO: Should be computed from dyinfo of ruby binary
       wasi_libc_shared_libs = [
         "libc.so",
-        "libdl.so",
         "libwasi-emulated-getpid.so",
         "libwasi-emulated-mman.so",
         "libwasi-emulated-process-clocks.so",
@@ -118,18 +114,47 @@ def link_exts(executor, build, output)
 
       wasi_libc_shared_libs.each do |lib|
         wasi_sdk_path = build.toolchain.wasi_sdk_path
-        link_args << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
+        libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
       end
       wasi_adapter = ENV["WASI_COMPONENT_ADAPTER"] or raise "WASI_COMPONENT_ADAPTER is not set"
-      link_args.concat ["--adapt", wasi_adapter]
-      Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so")).each do |so|
-        link_args << "--dl-openable"
-        link_args << "#{so.delete_prefix(ruby_root)}=#{so}"
+      adapters = [wasi_adapter]
+      dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
+      linker = RubyWasmExt::ComponentLink.new
+      linker.use_built_in_libdl(true)
+      linker.stub_missing_functions(false)
+      linker.validate(true)
+
+      libraries.each do |lib|
+        # Non-DL openable libraries should be referenced as base name
+        lib_name = File.basename(lib)
+        module_bytes = File.binread(lib).unpack("C*")
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, false)
       end
-      link_args << "-o"
-      link_args << File.join(ruby_root, "usr", "local", "bin", "ruby.component.wasm")
 
-      executor.system(wasm_tools, *link_args)
+      filter_libs = [
+        "enc/encdb.so",
+        "stringio.so",
+        "monitor.so",
+        "pathname.so",
+        "strscan.so",
+      ]
+      dl_openable_libs.each do |lib|
+        # DL openable lib_name should be a relative path from ruby_root
+        lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
+        # next unless filter_libs.any? { |filter| lib_name.include?(filter) }
+        module_bytes = File.binread(lib).unpack("C*")
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, true)
+      end
+
+      adapters.each do |adapter|
+        adapter_name = File.basename(adapter)
+        # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
+        adapter_name = adapter_name.split(".")[0]
+        linker.adapter(adapter_name, File.binread(adapter).unpack("C*"))
+      end
+      return linker.encode()
     end
 
     def build_exts(executor, build)
@@ -275,6 +300,12 @@ def derive_build
       build
     end
 
+    def build_and_link_exts(executor)
+      build = derive_build
+      ruby_root = build.crossruby.dest_dir
+      File.binread(File.join(ruby_root, "usr", "local", "bin", "ruby"))
+    end
+
     def user_exts(build)
       @user_exts ||=
         specs_with_extensions.flat_map do |spec, exts|