Skip to content

Commit 20874d5

Browse files
authored
add --use-built-in-libdl option to component link subcommand (#1478)
* add `--use-built-in-libdl` option to `component link` subcommand This tells `wit-component` to include a prebuilt libdl.so when linking. This library provides `dlopen`, `dlsym`, etc. using the lookup tables generated by the linker when one or more `--dl-openable` options are provided. Signed-off-by: Joel Dice <[email protected]> * mark `dl` crate unpublishable Signed-off-by: Joel Dice <[email protected]> * move libdl.so to hopefully make `./publish verify` happy Signed-off-by: Joel Dice <[email protected]> * update link-dl-openable-builtin-libdl/component.wat Signed-off-by: Joel Dice <[email protected]> * tweak error message set by `dlsym` Signed-off-by: Joel Dice <[email protected]> * update libdl.so and link-dl-openable-builtin-libdl/component.wat Signed-off-by: Joel Dice <[email protected]> * address review feedback - make libdl.so smaller - remove unnecessary newtype - check that libdl.so is up-to-date during CI - use CStr literal syntax - add `dl` crate to workspace Signed-off-by: Joel Dice <[email protected]> * CI fix Signed-off-by: Joel Dice <[email protected]> * use LTO and strip when building libdl.so This helps ensure the build is deterministic and not system-dependent. Signed-off-by: Joel Dice <[email protected]> * another CI fix Signed-off-by: Joel Dice <[email protected]> * exclude `dl` crate when testing on WebAssembly Signed-off-by: Joel Dice <[email protected]> * remove debug logging Signed-off-by: Joel Dice <[email protected]> * revert unneeded change in crates/wit-component/tests/components.rs Signed-off-by: Joel Dice <[email protected]> --------- Signed-off-by: Joel Dice <[email protected]>
1 parent cbf7c01 commit 20874d5

File tree

19 files changed

+962
-5
lines changed

19 files changed

+962
-5
lines changed

.github/workflows/main.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,13 @@ jobs:
7979
submodules: true
8080
- name: Install Rust (rustup)
8181
run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }}
82+
- run: |
83+
curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-linux.tar.gz
84+
tar xf wasi-sdk-21.0-linux.tar.gz
85+
export WASI_SDK_PATH=$(pwd)/wasi-sdk-21.0
86+
rustup target add wasm32-wasi
87+
cd crates/wit-component/dl && bash check.sh
88+
if: matrix.os == 'ubuntu-latest' && matrix.build == 'stable'
8289
- run: cargo test --locked --all
8390
- run: cargo test --locked -p wasmparser --benches
8491
- run: cargo test --locked -p wasm-encoder --all-features
@@ -109,7 +116,8 @@ jobs:
109116
cargo --locked test --workspace \
110117
--exclude fuzz-stats \
111118
--exclude wasm-tools-fuzz \
112-
--exclude wasm-mutate-stats
119+
--exclude wasm-mutate-stats \
120+
--exclude dl
113121
114122
rustfmt:
115123
name: Rustfmt

Cargo.lock

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ members = [
2222
'crates/wasm-mutate-stats',
2323
'fuzz',
2424
'crates/wit-parser/fuzz',
25+
'crates/wit-component/dl',
2526
]
2627

2728
[workspace.lints.rust]

crates/wit-component/dl/Cargo.toml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[package]
2+
name = "dl"
3+
version = "0.1.0"
4+
edition = "2021"
5+
publish = false
6+
7+
[lib]
8+
crate-type = [ "staticlib" ]
9+
10+
[dependencies]

crates/wit-component/dl/build.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# This script will rebuild libdl.so from source.
2+
#
3+
# To run this, you'll need to use `wasi-sdk` 21 or later, installed at
4+
# $WASI_SDK_PATH.
5+
#
6+
# Example: WASI_SDK_PATH=/opt/wasi-sdk bash build.sh ../libdl.so
7+
8+
CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C relocation-model=pic" cargo build --release --target=wasm32-wasi
9+
$WASI_SDK_PATH/bin/clang -shared -o $1 -Wl,--whole-archive ../../../target/wasm32-wasi/release/libdl.a -Wl,--no-whole-archive
10+
cargo run --manifest-path ../../../Cargo.toml -- strip -a $1 -o $1

crates/wit-component/dl/check.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
bash ./build.sh ../../../target/wasm32-wasi/release/tmp.so
2+
if diff ../../../target/wasm32-wasi/release/tmp.so ../libdl.so; then
3+
exit 0
4+
else
5+
echo "libdl.so is out-of-date; please run crates/wit-component/dl/build.sh to update it">&2
6+
exit 1
7+
fi

crates/wit-component/dl/src/lib.rs

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
#![no_std]
2+
3+
//! This library uses lookup tables generated by `wasm-tools component link` to
4+
//! provide runtime library and symbol resolution via `dlopen` and `dlsym`. The
5+
//! tables are provided via a call to `__wasm_set_libraries`, which `wasm-tools
6+
//! component link` arranges as part of component instantiation.
7+
8+
use core::{
9+
ffi::{c_char, c_int, c_void, CStr},
10+
ptr, slice,
11+
};
12+
13+
const RTLD_LAZY: c_int = 1;
14+
const RTLD_NOW: c_int = 2;
15+
16+
const RTLD_NEXT: isize = -1;
17+
const RTLD_DEFAULT: isize = 0;
18+
19+
#[repr(C)]
20+
pub struct Name {
21+
length: u32,
22+
data: *const u8,
23+
}
24+
25+
#[repr(C)]
26+
pub struct Symbol {
27+
name: Name,
28+
address: *const c_void,
29+
}
30+
31+
#[repr(C)]
32+
pub struct Symbols {
33+
count: u32,
34+
symbols: *const Symbol,
35+
}
36+
37+
#[repr(C)]
38+
pub struct Library {
39+
name: Name,
40+
symbols: Symbols,
41+
}
42+
43+
#[repr(C)]
44+
pub struct Libraries {
45+
count: u32,
46+
libraries: *const Library,
47+
}
48+
49+
static mut ERROR: *const c_char = ptr::null();
50+
static mut LIBRARIES: *const Libraries = ptr::null();
51+
52+
unsafe fn invalid_handle(library: *const c_void) -> bool {
53+
if LIBRARIES.is_null() {
54+
panic!(
55+
"`__wasm_set_libraries` should have been called during \
56+
instantiation with a non-NULL value"
57+
);
58+
}
59+
60+
let library = library as *const Library;
61+
if (0..(*LIBRARIES).count)
62+
.any(|index| (*LIBRARIES).libraries.add(usize::try_from(index).unwrap()) == library)
63+
{
64+
false
65+
} else {
66+
ERROR = c"invalid library handle".as_ptr();
67+
true
68+
}
69+
}
70+
71+
/// # Safety
72+
///
73+
/// `library` must be a valid, not-yet-closed library pointer returned by
74+
/// `dlopen`.
75+
#[no_mangle]
76+
pub unsafe extern "C" fn dlclose(library: *mut c_void) -> c_int {
77+
if invalid_handle(library) {
78+
-1
79+
} else {
80+
0
81+
}
82+
}
83+
84+
#[no_mangle]
85+
pub extern "C" fn dlerror() -> *const c_char {
86+
unsafe {
87+
let value = ERROR;
88+
ERROR = ptr::null();
89+
value
90+
}
91+
}
92+
93+
/// # Safety
94+
///
95+
/// `name` must be a valid pointer to a null-terminated string.
96+
#[no_mangle]
97+
pub unsafe extern "C" fn dlopen(name: *const c_char, flags: c_int) -> *const c_void {
98+
if LIBRARIES.is_null() {
99+
panic!(
100+
"`__wasm_set_libraries` should have been called during \
101+
instantiation with a non-NULL value"
102+
);
103+
}
104+
105+
if (flags & !(RTLD_LAZY | RTLD_NOW)) != 0 {
106+
// TODO
107+
ERROR = c"dlopen flags not yet supported".as_ptr();
108+
return ptr::null();
109+
}
110+
111+
let name = CStr::from_ptr(name);
112+
let name = name.to_bytes();
113+
let libraries = slice::from_raw_parts(
114+
(*LIBRARIES).libraries,
115+
usize::try_from((*LIBRARIES).count).unwrap(),
116+
);
117+
if let Ok(index) = libraries.binary_search_by(|library| {
118+
slice::from_raw_parts(
119+
library.name.data,
120+
usize::try_from(library.name.length).unwrap(),
121+
)
122+
.cmp(name)
123+
}) {
124+
&libraries[index] as *const _ as _
125+
} else {
126+
ERROR = c"library not found".as_ptr();
127+
ptr::null()
128+
}
129+
}
130+
131+
/// # Safety
132+
///
133+
/// `library` must be a valid, not-yet-closed library pointer returned by
134+
/// `dlopen`, and `name` must be a valid pointer to a null-terminated string.
135+
#[no_mangle]
136+
pub unsafe extern "C" fn dlsym(library: *const c_void, name: *const c_char) -> *const c_void {
137+
if library as isize == RTLD_NEXT || library as isize == RTLD_DEFAULT {
138+
// TODO
139+
ERROR = c"dlsym RTLD_NEXT and RTLD_DEFAULT not yet supported".as_ptr();
140+
return ptr::null();
141+
}
142+
143+
if invalid_handle(library) {
144+
return ptr::null();
145+
}
146+
147+
let library = library as *const Library;
148+
let name = CStr::from_ptr(name);
149+
let name = name.to_bytes();
150+
let symbols = slice::from_raw_parts(
151+
(*library).symbols.symbols,
152+
usize::try_from((*library).symbols.count).unwrap(),
153+
);
154+
if let Ok(index) = symbols.binary_search_by(|symbol| {
155+
slice::from_raw_parts(
156+
symbol.name.data,
157+
usize::try_from(symbol.name.length).unwrap(),
158+
)
159+
.cmp(name)
160+
}) {
161+
symbols[index].address
162+
} else {
163+
ERROR = c"symbol not found".as_ptr();
164+
ptr::null()
165+
}
166+
}
167+
168+
/// # Safety
169+
///
170+
/// `libraries` must be a valid pointer to a `Libraries` object, and this
171+
/// pointer must remain valid for the lifetime of the process.
172+
#[no_mangle]
173+
pub unsafe extern "C" fn __wasm_set_libraries(libraries: *const Libraries) {
174+
LIBRARIES = libraries;
175+
}
176+
177+
#[cfg(target_arch = "wasm32")]
178+
#[panic_handler]
179+
fn panic(_info: &core::panic::PanicInfo) -> ! {
180+
core::arch::wasm32::unreachable()
181+
}

crates/wit-component/libdl.so

1.31 KB
Binary file not shown.

crates/wit-component/src/linking.rs

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,7 @@ enum Address<'a> {
5757
/// Represents a `dlopen`/`dlsym` lookup table enabling runtime symbol resolution
5858
///
5959
/// The top level of this table is a sorted list of library names and offsets, each pointing to a sorted list of
60-
/// symbol names and offsets. See
61-
/// https://github.com/dicej/wasi-libc/blob/76c7e1e1cfdad577ecd7f61c67ead7a38d62a7c4/libc-top-half/musl/src/misc/dl.c
62-
/// for how this is used.
60+
/// symbol names and offsets. See ../dl/src/lib.rs for how this is used at runtime.
6361
struct DlOpenables<'a> {
6462
/// Offset into the main module's table where function references will be stored
6563
table_base: u32,
@@ -1202,6 +1200,9 @@ pub struct Linker {
12021200
/// Whether to generate trapping stubs for any unresolved imports
12031201
stub_missing_functions: bool,
12041202

1203+
/// Whether to use a built-in implementation of `dlopen`/`dlsym`.
1204+
use_built_in_libdl: bool,
1205+
12051206
/// Size of stack (in bytes) to allocate in the synthesized main module
12061207
///
12071208
/// If `None`, use `DEFAULT_STACK_SIZE_BYTES`.
@@ -1247,8 +1248,18 @@ impl Linker {
12471248
self
12481249
}
12491250

1251+
/// Specify whether to use a built-in implementation of `dlopen`/`dlsym`.
1252+
pub fn use_built_in_libdl(mut self, use_built_in_libdl: bool) -> Self {
1253+
self.use_built_in_libdl = use_built_in_libdl;
1254+
self
1255+
}
1256+
12501257
/// Encode the component and return the bytes
12511258
pub fn encode(mut self) -> Result<Vec<u8>> {
1259+
if self.use_built_in_libdl {
1260+
self = self.library("libdl.so", include_bytes!("../libdl.so"), false)?;
1261+
}
1262+
12521263
let adapter_names = self
12531264
.adapters
12541265
.iter()
@@ -1332,6 +1343,7 @@ impl Linker {
13321343
.all(|(_, export)| export.flags.contains(SymbolFlags::BINDING_WEAK)))
13331344
{
13341345
self.stub_missing_functions = false;
1346+
self.use_built_in_libdl = false;
13351347
self.libraries.push((
13361348
"wit-component:stubs".into(),
13371349
make_stubs_module(&missing),

crates/wit-component/tests/components.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ use wit_parser::{PackageId, Resolve, UnresolvedPackage};
3131
/// * [optional] `stub-missing-functions` - if linking libraries and this file
3232
/// exists, `Linker::stub_missing_functions` will be set to `true`. The
3333
/// contents of the file are ignored.
34+
/// * [optional] `use-built-in-libdl` - if linking libraries and this file
35+
/// exists, `Linker::use_built_in_libdl` will be set to `true`. The contents
36+
/// of the file are ignored.
3437
///
3538
/// And the output files are one of the following:
3639
///
@@ -94,6 +97,10 @@ fn component_encoding_via_flags() -> Result<()> {
9497
linker = linker.stub_missing_functions(true);
9598
}
9699

100+
if path.join("use-built-in-libdl").is_file() {
101+
linker = linker.use_built_in_libdl(true);
102+
}
103+
97104
let linker =
98105
libs.into_iter()
99106
.try_fold(linker, |linker, (prefix, path, dl_openable)| {

0 commit comments

Comments
 (0)