Initial dynamic extension library support

Author: kateinoigakukun

lib/ruby_wasm/build/product/crossruby.rb

@@ -44,12 +44,14 @@ def build(executor, crossruby)
       objdir = product_build_dir crossruby
       executor.mkdir_p objdir
       do_extconf executor, crossruby
+
+      build_target = crossruby.target.pic? ? "install-so" : "static"
       executor.system "make",
                       "-j#{executor.process_count}",
                       "-C",
                       "#{objdir}",
                       *make_args(crossruby),
-                      "static"
+                      build_target
       # A ext can provide link args by link.filelist. It contains only built archive file by default.
       unless File.exist?(linklist(crossruby))
         executor.write(
@@ -62,37 +64,48 @@ def build(executor, crossruby)
     def do_extconf(executor, crossruby)
       objdir = product_build_dir crossruby
       source = crossruby.source
-      extconf_args = [
-        "-C", objdir,
-        "--disable=gems",
-        # HACK: top_srcdir is required to find ruby headers
-        "-e",
-        %Q($top_srcdir="#{source.src_dir}"),
-        # HACK: extout is required to find config.h
-        "-e",
-        %Q($extout="#{crossruby.build_dir}/.ext"),
-        # HACK: force static ext build by imitating extmk
-        "-e",
-        "$static = true; trace_var(:$static) {|v| $static = true }",
-        # HACK: $0 should be extconf.rb path due to mkmf source file detection
-        # and we want to insert some hacks before it. But -e and $0 cannot be
-        # used together, so we rewrite $0 in -e.
-        "-e",
-        %Q($0="#{@srcdir}/extconf.rb"),
-        "-e",
-        %Q(require_relative "#{@srcdir}/extconf.rb"),
-        # HACK: extract "$target" from extconf.rb to get a full target name
-        # like "cgi/escape" instead of "escape"
-        "-e",
-        %Q(require "json"; File.write("#{metadata_json(crossruby)}", JSON.dump({target: $target}))),
-        "-I#{crossruby.build_dir}"
-      ]
-      # Clear RUBYOPT to avoid loading unrelated bundle setup
-      executor.system crossruby.baseruby_path,
+      extconf_args = ["-C", objdir]
+      # @type var extconf_env: Hash[String, String]
+      extconf_env = {}
+
+      if crossruby.target.pic?
+        extconf_args.concat [
+          "#{@srcdir}/extconf.rb",
+        ]
+        rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+        raise "rbconfig.rb not found" unless rbconfig_rb
+        extconf_env["MKMF_TARGET_RBCONFIG"] = rbconfig_rb
+      else
+        extconf_args.concat [
+          "--disable=gems",
+          # HACK: top_srcdir is required to find ruby headers
+          "-e",
+          %Q($top_srcdir="#{source.src_dir}"),
+          # HACK: extout is required to find config.h
+          "-e",
+          %Q($extout="#{crossruby.build_dir}/.ext"),
+          # HACK: force static ext build by imitating extmk
+          "-e",
+          "$static = true; trace_var(:$static) {|v| $static = true }",
+          # HACK: $0 should be extconf.rb path due to mkmf source file detection
+          # and we want to insert some hacks before it. But -e and $0 cannot be
+          # used together, so we rewrite $0 in -e.
+          "-e",
+          %Q($0="#{@srcdir}/extconf.rb"),
+          "-e",
+          %Q(require_relative "#{@srcdir}/extconf.rb"),
+          # HACK: extract "$target" from extconf.rb to get a full target name
+          # like "cgi/escape" instead of "escape"
+          "-e",
+          %Q(require "json"; File.write("#{metadata_json(crossruby)}", JSON.dump({target: $target}))),
+          "-I#{crossruby.build_dir}"
+        ]
+        # Clear RUBYOPT to avoid loading unrelated bundle setup
+        extconf_env["RUBYOPT"] = ""
+      end
+      executor.system Gem.ruby,
                       *extconf_args,
-                      env: {
-                        "RUBYOPT" => ""
-                      }
+                      env: extconf_env
     end
 
     def do_install_rb(executor, crossruby)
@@ -114,7 +127,7 @@ def cache_key(digest)
   end
 
   class CrossRubyProduct < AutoconfProduct
-    attr_reader :source, :toolchain
+    attr_reader :target, :source, :toolchain
     attr_accessor :user_exts,
                   :wasmoptflags,
                   :cppflags,
@@ -155,6 +168,10 @@ def need_exts_build?
       @user_exts.any?
     end
 
+    def need_extinit_obj?
+      need_exts_build? && !@target.pic?
+    end
+
     def build_exts(executor)
       @user_exts.each do |prod|
         executor.begin_section prod.class, prod.name, "Building"
@@ -181,7 +198,7 @@ def build(executor, remake: false, reconfigure: false)
 
       executor.begin_section self.class, name, "Building"
 
-      if need_exts_build?
+      if need_extinit_obj?
         executor.mkdir_p File.dirname(extinit_obj)
         executor.system "ruby",
                         extinit_c_erb,
@@ -312,7 +329,7 @@ def configure_args(build_triple, toolchain)
 
       args.concat(self.tools_args)
       (@user_exts || []).each { |lib| xldflags << "@#{lib.linklist(self)}" }
-      xldflags << extinit_obj if need_exts_build?
+      xldflags << extinit_obj if need_extinit_obj?
 
       cflags = @cflags.dup
       xcflags = @xcflags.dup

lib/ruby_wasm/cli.rb

@@ -122,8 +122,15 @@ def build(args)
         end
         .parse!(args)
 
+      __skip__ = Bundler.settings.temporary(force_ruby_platform: true) do
+        do_build_with_force_ruby_platform(options)
+      end
+    end
+
+    def do_build_with_force_ruby_platform(options)
       verbose = RubyWasm.logger.level == :debug
       executor = RubyWasm::BuildExecutor.new(verbose: verbose)
+
       packager = self.derive_packager(options)
 
       if options[:print_ruby_cache_key]
@@ -281,7 +288,7 @@ def self.bundled_patches_path
     end
 
     def derive_packager(options)
-      definition = nil
+      __skip__ = definition = nil
       __skip__ = if defined?(Bundler) && !options[:disable_gems]
         begin
           # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output.
@@ -293,6 +300,7 @@ def derive_packager(options)
           Bundler.ui.level = old_level
         end
       end
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
       RubyWasm::Packager.new(root, build_config(options), definition)
     end
 

lib/ruby_wasm/packager.rb

@@ -58,7 +58,9 @@ def ruby_core_build
   # Retrieves the specs from the Bundler definition, excluding the excluded gems.
   def specs
     return [] unless @definition
-    @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs ||= @definition.resolve.materialize(@definition.requested_dependencies)
+      .reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs
   end
 
   # Checks if dynamic linking is supported.

lib/ruby_wasm/packager/core.rb

@@ -66,6 +66,8 @@ def build(executor, options)
       force_rebuild =
         options[:remake] || options[:clean] || options[:reconfigure]
       if File.exist?(build.crossruby.artifact) && !force_rebuild
+        # Always build extensions because they are usually not expensive to build
+        self.build_exts(executor, build)
         return build.crossruby.artifact
       end
       build.crossruby.clean(executor) if options[:clean]
@@ -85,11 +87,12 @@ def build(executor, options)
         else
           do_build.call
         end
+      self.build_exts(executor, build)
       build.crossruby.artifact
     end
 
-    def build_exts(build)
-      specs_with_extensions.flat_map do |spec, exts|
+    def build_exts(executor, build)
+      exts = specs_with_extensions.flat_map do |spec, exts|
         exts.map do |ext|
           ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
           ext_srcdir = File.join(spec.full_gem_path, ext_feature)
@@ -101,6 +104,12 @@ def build_exts(build)
           )
         end
       end
+
+      exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, build.crossruby)
+        executor.end_section prod.class, prod.name
+      end
     end
 
     def cache_key(digest)

lib/ruby_wasm/packager/file_system.rb

@@ -65,12 +65,14 @@ def setup_rb_content
   end
 
   def remove_non_runtime_files(executor)
-    %w[
-      **/*.so
+    patterns = %w[
       usr/local/lib/libruby-static.a
       usr/local/bin/ruby
       usr/local/include
-    ].each do |pattern|
+    ]
+
+    patterns << "**/*.so" unless @packager.support_dynamic_linking?
+    patterns.each do |pattern|
       Dir
         .glob(File.join(@dest_dir, pattern))
         .each do |entry|

packages/gems/js/ext/witapi/depend

@@ -8,4 +8,4 @@ witapi-core.o: $(srcdir)/bindgen/rb-abi-guest.h
 
 bindgen/%.o: $(srcdir)/bindgen/%.c
 	@mkdir -p "$(@D)"
-	$(CC) -c -I$(srcdir)/bindgen -o $@ $<
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$< -I$(srcdir)/bindgen

sig/ruby_wasm/build.rbs

@@ -184,6 +184,7 @@ module RubyWasm
     @openssl: OpenSSLProduct
     @wasi_vfs: WasiVfsProduct
 
+    attr_reader target: Target
     attr_reader source: BuildSource
     attr_reader toolchain: Toolchain
     attr_accessor user_exts: Array[CrossRubyExtProduct]
@@ -197,6 +198,7 @@ module RubyWasm
     def initialize: (BuildParams params, String build_dir, String rubies_dir, BaseRubyProduct baseruby, BuildSource source, Toolchain toolchain) -> void
     def configure: (BuildExecutor executor, ?reconfigure: bool) -> void
     def need_exts_build?: -> bool
+    def need_extinit_obj?: -> bool
     def build_exts: (BuildExecutor executor) -> void
     def build: (BuildExecutor executor, ?remake: bool, ?reconfigure: bool) -> void
     def clean: (BuildExecutor executor) -> void

sig/ruby_wasm/packager.rbs

@@ -68,6 +68,7 @@ class RubyWasm::Packager
     class DynamicLinking < RubyWasm::Packager::Core::BuildStrategy
       @build: RubyWasm::Build
       def derive_build: () -> RubyWasm::Build
+      def build_exts: (RubyWasm::BuildExecutor, RubyWasm::Build) -> void
       def name: () -> string
     end