Skip to content

Commit 2827490

Browse files
kateinoigakukunkateinoigakukun
authored
Merge pull request #467 from ruby/katei/dynamic-component
Use wasi-virt for Componentized build
2 parents acd7824 + 7fb033d commit 2827490

File tree

13 files changed

+956
-356
lines changed

13 files changed

+956
-356
lines changed

‎Cargo.lock

Lines changed: 778 additions & 314 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎builders/wasm32-unknown-emscripten/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ ENV PATH=/usr/bin:$PATH
2323
ENV RUSTUP_HOME=/usr/local/rustup \
2424
CARGO_HOME=/usr/local/cargo \
2525
PATH=/usr/local/cargo/bin:$PATH \
26-
RUST_VERSION=1.75
26+
RUST_VERSION=1.76.0
2727

2828
RUN set -eux pipefail; \
2929
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \

‎builders/wasm32-unknown-wasi/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ RUN set -eux pipefail; \
2727
ENV RUSTUP_HOME=/usr/local/rustup \
2828
CARGO_HOME=/usr/local/cargo \
2929
PATH=/usr/local/cargo/bin:$PATH \
30-
RUST_VERSION=1.75
30+
RUST_VERSION=1.76.0
3131

3232
RUN set -eux pipefail; \
3333
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \

‎ext/ruby_wasm/Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,6 @@ bytes = "1"
1515
wizer = "4.0.0"
1616
wasi-vfs-cli = { git = "https://github.com/kateinoigakukun/wasi-vfs/", tag = "0.5.2" }
1717
structopt = "0.3.26"
18-
wit-component = "0.203.0"
18+
wit-component = "0.212.0"
19+
wasm-compose = "0.212.0"
20+
wasi-virt = { git = "https://github.com/bytecodealliance/wasi-virt", rev = "02de7b495eba3f1bf786bf17cf2236b7277be7b0", default-features = false }

‎ext/ruby_wasm/src/lib.rs

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::{collections::HashMap, path::PathBuf};
1+
use std::{collections::HashMap, env, path::PathBuf, time::SystemTime};
22

33
use magnus::{
44
eval, exception, function, method,
@@ -8,6 +8,7 @@ use magnus::{
88
};
99
use structopt::StructOpt;
1010
use wizer::Wizer;
11+
use wasi_virt;
1112

1213
static RUBY_WASM: value::Lazy<RModule> =
1314
value::Lazy::new(|ruby| ruby.define_module("RubyWasmExt").unwrap());
@@ -222,6 +223,99 @@ impl ComponentEncode {
222223
}
223224
}
224225

226+
#[wrap(class = "RubyWasmExt::WasiVirt")]
227+
struct WasiVirt(std::cell::RefCell<Option<wasi_virt::WasiVirt>>);
228+
229+
impl WasiVirt {
230+
fn new() -> Self {
231+
Self(std::cell::RefCell::new(Some(wasi_virt::WasiVirt::new())))
232+
}
233+
234+
fn virt<R>(
235+
&self,
236+
body: impl FnOnce(&mut wasi_virt::WasiVirt) -> Result<R, Error>,
237+
) -> Result<R, Error> {
238+
let mut virt = self.0.take().ok_or_else(|| {
239+
Error::new(
240+
exception::standard_error(),
241+
"wasi virt is already consumed".to_string(),
242+
)
243+
})?;
244+
let result = body(&mut virt)?;
245+
self.0.replace(Some(virt));
246+
Ok(result)
247+
}
248+
249+
fn allow_all(&self) -> Result<(), Error> {
250+
self.virt(|virt| {
251+
virt.allow_all();
252+
// Disable sockets for now since `sockets/ip-name-lookup` is not
253+
// supported by @bytecodealliance/preview2-shim yet
254+
virt.sockets(false);
255+
Ok(())
256+
})
257+
}
258+
259+
fn map_dir(&self, guest_dir: String, host_dir: String) -> Result<(), Error> {
260+
self.virt(|virt| {
261+
virt.fs().virtual_preopen(guest_dir, host_dir);
262+
Ok(())
263+
})
264+
}
265+
266+
fn finish(&self) -> Result<bytes::Bytes, Error> {
267+
self.virt(|virt| {
268+
let result = virt.finish().map_err(|e| {
269+
Error::new(
270+
exception::standard_error(),
271+
format!("failed to generate virtualization adapter: {}", e),
272+
)
273+
})?;
274+
Ok(result.adapter.into())
275+
})
276+
}
277+
278+
fn compose(&self, component_bytes: bytes::Bytes) -> Result<bytes::Bytes, Error> {
279+
let virt_adapter = self.finish()?;
280+
let tmpdir = env::temp_dir();
281+
let tmp_virt = tmpdir.join(format!("virt{}.wasm", timestamp()));
282+
std::fs::write(&tmp_virt, &virt_adapter).map_err(|e| {
283+
Error::new(
284+
exception::standard_error(),
285+
format!("failed to write virt adapter: {}", e),
286+
)
287+
})?;
288+
let tmp_component = tmpdir.join(format!("component{}.wasm", timestamp()));
289+
std::fs::write(&tmp_component, &component_bytes).map_err(|e| {
290+
Error::new(
291+
exception::standard_error(),
292+
format!("failed to write component: {}", e),
293+
)
294+
})?;
295+
296+
use wasm_compose::{composer, config};
297+
let config = config::Config {
298+
definitions: vec![tmp_virt],
299+
..Default::default()
300+
};
301+
let composer = composer::ComponentComposer::new(&tmp_component, &config);
302+
let composed = composer.compose().map_err(|e| {
303+
Error::new(
304+
exception::standard_error(),
305+
format!("failed to compose component: {}", e),
306+
)
307+
})?;
308+
return Ok(composed.into());
309+
310+
fn timestamp() -> u64 {
311+
match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
312+
Ok(n) => n.as_secs(),
313+
Err(_) => panic!(),
314+
}
315+
}
316+
}
317+
}
318+
225319
#[magnus::init]
226320
fn init(ruby: &Ruby) -> Result<(), Error> {
227321
let module = RUBY_WASM.get_inner_with(ruby);
@@ -266,5 +360,12 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
266360
)?;
267361
component_encode.define_method("encode", method!(ComponentEncode::encode, 0))?;
268362

363+
let wasi_virt = module.define_class("WasiVirt", ruby.class_object())?;
364+
wasi_virt.define_singleton_method("new", function!(WasiVirt::new, 0))?;
365+
wasi_virt.define_method("allow_all", method!(WasiVirt::allow_all, 0))?;
366+
wasi_virt.define_method("map_dir", method!(WasiVirt::map_dir, 2))?;
367+
wasi_virt.define_method("finish", method!(WasiVirt::finish, 0))?;
368+
wasi_virt.define_method("compose", method!(WasiVirt::compose, 1))?;
369+
269370
Ok(())
270371
}

‎lib/ruby_wasm/build/product/crossruby.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ def do_extconf(executor, crossruby)
8080
"--target-rbconfig=#{rbconfig_rb}",
8181
]
8282
extconf_args << "--enable-component-model" if @features.support_component_model?
83-
executor.system Gem.ruby, *extconf_args
83+
executor.system crossruby.baseruby_path, *extconf_args
8484
end
8585

8686
def do_legacy_extconf(executor, crossruby)

‎lib/ruby_wasm/packager.rb

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def package(executor, dest_dir, options)
4444
fs.remove_stdlib(executor)
4545
end
4646

47-
if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_dynamic_linking?
47+
if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_component_model?
4848
# wasi-vfs supports only WASI target
4949
wasi_vfs = RubyWasmExt::WasiVfs.new
5050
wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -54,6 +54,20 @@ def package(executor, dest_dir, options)
5454
end
5555
wasm_bytes = ruby_core.link_gem_exts(executor, fs.ruby_root, fs.bundle_dir, wasm_bytes)
5656

57+
if features.support_component_model?
58+
wasi_virt = RubyWasmExt::WasiVirt.new
59+
wasi_virt.allow_all
60+
[
61+
{ guest: "/bundle", host: fs.bundle_dir },
62+
{ guest: "/usr", host: File.dirname(fs.ruby_root) }
63+
].each do |map|
64+
map => { guest:, host: }
65+
RubyWasm.logger.debug "Adding files into VFS: #{host} => #{guest}"
66+
wasi_virt.map_dir(guest, host)
67+
end
68+
wasm_bytes = wasi_virt.compose(wasm_bytes)
69+
end
70+
5771
wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
5872
wasm_bytes
5973
end

‎lib/ruby_wasm/packager/core.rb

Lines changed: 25 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ def wasi_exec_model
6767
use_js_gem ? "reactor" : "command"
6868
end
6969

70+
def with_unbundled_env(&block)
71+
__skip__ = if defined?(Bundler)
72+
Bundler.with_unbundled_env(&block)
73+
else
74+
block.call
75+
end
76+
end
77+
7078
def cache_key(digest)
7179
raise NotImplementedError
7280
end
@@ -87,27 +95,22 @@ def build(executor, options)
8795
end
8896
build.crossruby.clean(executor) if options[:clean]
8997

90-
do_build =
91-
proc do
92-
build.crossruby.build(
93-
executor,
94-
remake: options[:remake],
95-
reconfigure: options[:reconfigure]
96-
)
97-
end
98+
self.with_unbundled_env do
99+
build.crossruby.build(
100+
executor,
101+
remake: options[:remake],
102+
reconfigure: options[:reconfigure]
103+
)
104+
end
98105

99-
__skip__ =
100-
if defined?(Bundler)
101-
Bundler.with_unbundled_env(&do_build)
102-
else
103-
do_build.call
104-
end
105106
build.crossruby.artifact
106107
end
107108

108109
def build_gem_exts(executor, gem_home)
109110
build = derive_build
110-
self._build_gem_exts(executor, build, gem_home)
111+
self.with_unbundled_env do
112+
self._build_gem_exts(executor, build, gem_home)
113+
end
111114
end
112115

113116
def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
@@ -269,21 +272,14 @@ def build(executor, options)
269272
end
270273
build.crossruby.clean(executor) if options[:clean]
271274

272-
do_build =
273-
proc do
274-
build.crossruby.build(
275-
executor,
276-
remake: options[:remake],
277-
reconfigure: options[:reconfigure]
278-
)
279-
end
275+
self.with_unbundled_env do
276+
build.crossruby.build(
277+
executor,
278+
remake: options[:remake],
279+
reconfigure: options[:reconfigure]
280+
)
281+
end
280282

281-
__skip__ =
282-
if defined?(Bundler)
283-
Bundler.with_unbundled_env(&do_build)
284-
else
285-
do_build.call
286-
end
287283
build.crossruby.artifact
288284
end
289285

‎packages/npm-packages/ruby-wasm-wasi/test/init.js

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,13 @@ async function initComponentRubyVM({ suppressStderr } = { suppressStderr: false
7373
return module;
7474
}
7575
const vm = await RubyVM._instantiate(async (jsRuntime) => {
76-
const { cli, clocks, filesystem, io, random, sockets } = preview2Shim;
77-
filesystem._setPreopens({})
76+
const { cli, clocks, filesystem, io, random, sockets, http } = preview2Shim;
77+
if (process.env.RUBY_BUILD_ROOT) {
78+
filesystem._setPreopens({
79+
"/usr": path.join(process.env.RUBY_BUILD_ROOT, "usr"),
80+
"/bundle": path.join(process.env.RUBY_BUILD_ROOT, "bundle"),
81+
})
82+
}
7883
cli._setArgs(["ruby.wasm"].concat(process.argv.slice(2)));
7984
cli._setCwd("/")
8085
const root = await instantiate(getCoreModule, {
@@ -98,6 +103,9 @@ async function initComponentRubyVM({ suppressStderr } = { suppressStderr: false
98103
"wasi:io/streams": io.streams,
99104
"wasi:random/random": random.random,
100105
"wasi:sockets/tcp": sockets.tcp,
106+
"wasi:http/types": http.types,
107+
"wasi:http/incoming-handler": http.incomingHandler,
108+
"wasi:http/outgoing-handler": http.outgoingHandler,
101109
})
102110
return root.rubyRuntime;
103111
}, {})

‎packages/npm-packages/ruby-wasm-wasi/tools/run-test-unit.mjs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,14 @@ const instantiateComponent = async (rootTestFile) => {
3535
return WebAssembly.compile(buffer);
3636
}
3737
const vm = await RubyVM._instantiate(async (jsRuntime) => {
38-
const { cli, clocks, filesystem, io, random, sockets } = preview2Shim;
38+
const { cli, clocks, filesystem, io, random, sockets, http } = preview2Shim;
3939
const dirname = path.dirname(new URL(import.meta.url).pathname);
40-
filesystem._setPreopens({
41-
"/__root__": path.join(dirname, ".."),
42-
})
40+
const preopens = { "/__root__": path.join(dirname, "..") };
41+
if (process.env.RUBY_ROOT) {
42+
preopens["/usr"] = path.join(process.env.RUBY_ROOT, "usr");
43+
preopens["/bundle"] = path.join(process.env.RUBY_ROOT, "bundle");
44+
}
45+
filesystem._setPreopens(preopens);
4346
cli._setArgs(["ruby.wasm"].concat(process.argv.slice(2)));
4447
cli._setCwd("/")
4548
const root = await instantiate(getCoreModule, {
@@ -63,6 +66,9 @@ const instantiateComponent = async (rootTestFile) => {
6366
"wasi:io/streams": io.streams,
6467
"wasi:random/random": random.random,
6568
"wasi:sockets/tcp": sockets.tcp,
69+
"wasi:http/types": http.types,
70+
"wasi:http/incoming-handler": http.incomingHandler,
71+
"wasi:http/outgoing-handler": http.outgoingHandler,
6672
})
6773
return root.rubyRuntime;
6874
}, {

0 commit comments

Comments
 (0)