Skip to content

Commit ecacf1b

Browse files
mkroeningGelbpunkt
mkroening
and
Gelbpunkt
committed
fix(kernel_stack): don't transmute args via memory
Co-authored-by: Jens Reidel <[email protected]>
1 parent 0f4d234 commit ecacf1b

File tree

3 files changed

+130
-172
lines changed

3 files changed

+130
-172
lines changed

‎hermit-macro/src/system.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ fn emit_func(func: ItemFn, sig: &ParsedSig, errno: bool) -> Result<ItemFn> {
179179
feature = "kernel-stack",
180180
not(any(target_arch = "riscv64", feature = "common-os"))
181181
))] {
182-
unsafe { crate::arch::kernel::kernel_stack::#kernel_function_ident(#kernel_ident, #(#args),*) }
182+
unsafe { crate::arch::kernel::kernel_stack::#kernel_function_ident(#(#args,)* #kernel_ident) }
183183
} else {
184184
#unsafety { #kernel_ident(#(#args),*) }
185185
}
@@ -259,7 +259,7 @@ mod tests {
259259
feature = "kernel-stack",
260260
not(any(target_arch = "riscv64", feature = "common-os"))
261261
))] {
262-
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(_sys_test, a, b) }
262+
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(a, b, _sys_test) }
263263
} else {
264264
{ _sys_test(a, b) }
265265
}
@@ -327,7 +327,7 @@ mod tests {
327327
feature = "kernel-stack",
328328
not(any(target_arch = "riscv64", feature = "common-os"))
329329
))] {
330-
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(_sys_test, a, b) }
330+
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(a, b, _sys_test) }
331331
} else {
332332
unsafe { _sys_test(a, b) }
333333
}
@@ -397,7 +397,7 @@ mod tests {
397397
feature = "kernel-stack",
398398
not(any(target_arch = "riscv64", feature = "common-os"))
399399
))] {
400-
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(_sys_test, a, b) }
400+
unsafe { crate::arch::kernel::kernel_stack::kernel_function2(a, b, _sys_test) }
401401
} else {
402402
{ _sys_test(a, b) }
403403
}

‎src/arch/aarch64/kernel/kernel_stack.rs‎

Lines changed: 64 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,79 @@
1-
use core::arch::asm;
2-
use core::{mem, ptr};
1+
use core::mem;
2+
3+
type Reg = core::mem::MaybeUninit<usize>;
4+
5+
#[unsafe(naked)]
6+
pub unsafe extern "C" fn call_with_kernel_stack(
7+
x0: Reg,
8+
x1: Reg,
9+
x2: Reg,
10+
x3: Reg,
11+
x4: Reg,
12+
x5: Reg,
13+
f: unsafe extern "C" fn(x0: Reg, x1: Reg, x2: Reg, x3: Reg, x4: Reg, x5: Reg) -> Reg,
14+
) -> Reg {
15+
core::arch::naked_asm!(
16+
// Disable IRQs and FIQs while changing stack pointer
17+
"msr daifset, #0b11",
18+
// Preserve return address on the stack
19+
"str x30, [sp, #-16]!",
20+
// Switch to kernel stack
21+
"msr spsel, #1",
22+
// Re-enable IRQs and FIQs
23+
"msr daifclr, #0b11",
24+
// Call the function pointer (stored in x6)
25+
"blr x6",
26+
// Disable IRQs and FIQs before restoring stack
27+
"msr daifset, #0b11",
28+
// Switch back to user stack
29+
"msr spsel, #0",
30+
// Restore return address from the stack
31+
"ldr x30, [sp], 16",
32+
// Re-enable IRQs and FIQs
33+
"msr daifclr, #0b11",
34+
// Return to caller (return value is in x0)
35+
"ret",
36+
)
37+
}
338

439
macro_rules! kernel_function_impl {
5-
($kernel_function:ident($($arg:ident: $A:ident),*) { $($operands:tt)* }) => {
40+
($kernel_function:ident($($arg:ident: $A:ident),*; $($z:ident: Reg),*)) => {
641
/// Executes `f` on the kernel stack.
742
#[allow(dead_code)]
8-
pub unsafe fn $kernel_function<R, $($A),*>(f: unsafe extern "C" fn($($A),*) -> R, $($arg: $A),*) -> R {
43+
#[inline]
44+
pub unsafe extern "C" fn $kernel_function<R, $($A),*>($($arg: $A,)* f: unsafe extern "C" fn($($A),*) -> R) -> R {
945
unsafe {
10-
assert!(mem::size_of::<R>() <= mem::size_of::<usize>());
11-
1246
$(
13-
assert!(mem::size_of::<$A>() <= mem::size_of::<usize>());
14-
let $arg = {
15-
let mut reg = 0usize;
16-
// SAFETY: $A is smaller than usize and directly fits in a register
17-
// Since f takes $A as argument via C calling convention, any upper bytes do not matter.
18-
ptr::write(ptr::from_mut(&mut reg).cast(), $arg);
19-
reg
20-
};
47+
assert!(mem::size_of::<$A>() <= mem::size_of::<Reg>());
2148
)*
49+
assert!(mem::size_of::<R>() <= mem::size_of::<Reg>());
2250

23-
let ret: u64;
24-
asm!(
25-
// Switch to kernel stack
26-
"msr spsel, {l1}",
27-
28-
// To make sure, Rust manages the stack in `f` correctly,
29-
// we keep all arguments and return values in registers
30-
// until we switch the stack back. Thus follows the sizing
31-
// requirements for arguments and return types.
32-
"blr {f}",
33-
34-
// Switch back to user stack
35-
"msr spsel, {l0}",
51+
let call_with_kernel_stack = mem::transmute::<*const (), unsafe extern "C" fn(
52+
$($arg: $A,)*
53+
$($z: Reg,)*
54+
f: unsafe extern "C" fn(
55+
$($arg: $A,)*
56+
) -> R,
57+
) -> R>(call_with_kernel_stack as *const ());
3658

37-
l0 = const 0,
38-
l1 = const 1,
39-
f = in(reg) f,
40-
41-
$($operands)*
42-
43-
// Return argument in x0
44-
lateout("x0") ret,
45-
46-
clobber_abi("C"),
47-
);
59+
$(
60+
let $z = Reg::uninit();
61+
)*
4862

49-
// SAFETY: R is smaller than usize and directly fits in rax
50-
// Since f returns R, we can safely convert ret to R
51-
mem::transmute_copy(&ret)
63+
call_with_kernel_stack(
64+
$($arg,)*
65+
$($z,)*
66+
f,
67+
)
5268
}
5369
}
5470
};
5571
}
5672

57-
kernel_function_impl!(kernel_function0() {});
58-
59-
kernel_function_impl!(kernel_function1(arg1: A1) {
60-
in("x0") arg1,
61-
});
62-
63-
kernel_function_impl!(kernel_function2(arg1: A1, arg2: A2) {
64-
in("x0") arg1,
65-
in("x1") arg2,
66-
});
67-
68-
kernel_function_impl!(kernel_function3(arg1: A1, arg2: A2, arg3: A3) {
69-
in("x0") arg1,
70-
in("x1") arg2,
71-
in("x2") arg3,
72-
});
73-
74-
kernel_function_impl!(kernel_function4(arg1: A1, arg2: A2, arg3: A3, arg4: A4) {
75-
in("x0") arg1,
76-
in("x1") arg2,
77-
in("x2") arg3,
78-
in("x3") arg4,
79-
});
80-
81-
kernel_function_impl!(kernel_function5(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5) {
82-
in("x0") arg1,
83-
in("x1") arg2,
84-
in("x2") arg3,
85-
in("x3") arg4,
86-
in("x4") arg5,
87-
});
88-
89-
kernel_function_impl!(kernel_function6(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5, arg6: A6) {
90-
in("x0") arg1,
91-
in("x1") arg2,
92-
in("x2") arg3,
93-
in("x3") arg4,
94-
in("x4") arg5,
95-
in("x5") arg6,
96-
});
73+
kernel_function_impl!(kernel_function0(; u1: Reg, u2: Reg, u3: Reg, u4: Reg, u5: Reg, u6: Reg));
74+
kernel_function_impl!(kernel_function1(arg1: A1; u2: Reg, u3: Reg, u4: Reg, u5: Reg, u6: Reg));
75+
kernel_function_impl!(kernel_function2(arg1: A1, arg2: A2; u3: Reg, u4: Reg, u5: Reg, u6: Reg));
76+
kernel_function_impl!(kernel_function3(arg1: A1, arg2: A2, arg3: A3; u4: Reg, u5: Reg, u6: Reg));
77+
kernel_function_impl!(kernel_function4(arg1: A1, arg2: A2, arg3: A3, arg4: A4; u5: Reg, u6: Reg));
78+
kernel_function_impl!(kernel_function5(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5; u6: Reg));
79+
kernel_function_impl!(kernel_function6(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5, arg6: A6; ));

‎src/arch/x86_64/kernel/kernel_stack.rs‎

Lines changed: 62 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,105 +1,80 @@
1-
use core::arch::asm;
2-
use core::{mem, ptr};
1+
use core::mem;
32

43
use crate::core_local::CoreLocal;
54

5+
type Reg = core::mem::MaybeUninit<usize>;
6+
7+
#[unsafe(naked)]
8+
pub unsafe extern "C" fn call_with_stack(
9+
rdi: Reg,
10+
rsi: Reg,
11+
rdx: Reg,
12+
rcx: Reg,
13+
r8: Reg,
14+
r9: Reg,
15+
f: unsafe extern "C" fn(rdi: Reg, rsi: Reg, rdx: Reg, rcx: Reg, r8: Reg, r9: Reg) -> Reg,
16+
stack_ptr: *mut (),
17+
) -> Reg {
18+
core::arch::naked_asm!(
19+
// Save user stack pointer and switch to supplied stack
20+
"cli",
21+
"push r12",
22+
"mov r12, rsp",
23+
"mov rsp, [rsp + 24]",
24+
"sti",
25+
// Call f
26+
"call [r12 + 16]",
27+
// Switch back to previous stack
28+
"cli",
29+
"mov rsp, r12",
30+
"pop r12",
31+
"sti",
32+
"ret",
33+
)
34+
}
35+
636
macro_rules! kernel_function_impl {
7-
($kernel_function:ident($($arg:ident: $A:ident),*) { $($operands:tt)* }) => {
37+
($kernel_function:ident($($arg:ident: $A:ident),*; $($z:ident: Reg),*)) => {
838
/// Executes `f` on the kernel stack.
939
#[allow(dead_code)]
10-
pub unsafe fn $kernel_function<R, $($A),*>(f: unsafe extern "C" fn($($A),*) -> R, $($arg: $A),*) -> R {
40+
#[inline]
41+
pub unsafe extern "C" fn $kernel_function<R, $($A),*>($($arg: $A,)* f: unsafe extern "C" fn($($A),*) -> R) -> R {
1142
unsafe {
12-
assert!(mem::size_of::<R>() <= mem::size_of::<usize>());
13-
1443
$(
15-
assert!(mem::size_of::<$A>() <= mem::size_of::<usize>());
16-
let $arg = {
17-
let mut reg = 0usize;
18-
// SAFETY: $A is smaller than usize and directly fits in a register
19-
// Since f takes $A as argument via C calling convention, any upper bytes do not matter.
20-
ptr::write(ptr::from_mut(&mut reg).cast(), $arg);
21-
reg
22-
};
44+
assert!(mem::size_of::<$A>() <= mem::size_of::<Reg>());
2345
)*
46+
assert!(mem::size_of::<R>() <= mem::size_of::<Reg>());
2447

25-
let ret: u64;
26-
asm!(
27-
// Save user stack pointer and switch to kernel stack
28-
"cli",
29-
"mov r12, rsp",
30-
"mov rsp, {kernel_stack_ptr}",
31-
"sti",
32-
33-
// To make sure, Rust manages the stack in `f` correctly,
34-
// we keep all arguments and return values in registers
35-
// until we switch the stack back. Thus follows the sizing
36-
// requirements for arguments and return types.
37-
"call {f}",
38-
39-
// Switch back to user stack
40-
"cli",
41-
"mov rsp, r12",
42-
"sti",
43-
44-
f = in(reg) f,
45-
kernel_stack_ptr = in(reg) CoreLocal::get().kernel_stack.get(),
46-
47-
$($operands)*
48-
49-
// user_stack_ptr saved in r12
50-
out("r12") _,
48+
let call_with_stack = mem::transmute::<*const (), unsafe extern "C" fn(
49+
$($arg: $A,)*
50+
$($z: Reg,)*
51+
f: unsafe extern "C" fn(
52+
$($arg: $A,)*
53+
) -> R,
54+
stack_ptr: *mut (),
55+
) -> R>(call_with_stack as *const ());
5156

52-
// Return argument in rax
53-
out("rax") ret,
57+
$(
58+
let $z = Reg::uninit();
59+
)*
5460

55-
clobber_abi("C"),
56-
);
61+
let kernel_stack = CoreLocal::get().kernel_stack.get().cast();
5762

58-
// SAFETY: R is smaller than usize and directly fits in rax
59-
// Since f returns R, we can safely convert ret to R
60-
mem::transmute_copy(&ret)
63+
call_with_stack(
64+
$($arg,)*
65+
$($z,)*
66+
f,
67+
kernel_stack,
68+
)
6169
}
6270
}
6371
};
6472
}
6573

66-
kernel_function_impl!(kernel_function0() {});
67-
68-
kernel_function_impl!(kernel_function1(arg1: A1) {
69-
in("rdi") arg1,
70-
});
71-
72-
kernel_function_impl!(kernel_function2(arg1: A1, arg2: A2) {
73-
in("rdi") arg1,
74-
in("rsi") arg2,
75-
});
76-
77-
kernel_function_impl!(kernel_function3(arg1: A1, arg2: A2, arg3: A3) {
78-
in("rdi") arg1,
79-
in("rsi") arg2,
80-
in("rdx") arg3,
81-
});
82-
83-
kernel_function_impl!(kernel_function4(arg1: A1, arg2: A2, arg3: A3, arg4: A4) {
84-
in("rdi") arg1,
85-
in("rsi") arg2,
86-
in("rdx") arg3,
87-
in("rcx") arg4,
88-
});
89-
90-
kernel_function_impl!(kernel_function5(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5) {
91-
in("rdi") arg1,
92-
in("rsi") arg2,
93-
in("rdx") arg3,
94-
in("rcx") arg4,
95-
in("r8") arg5,
96-
});
97-
98-
kernel_function_impl!(kernel_function6(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5, arg6: A6) {
99-
in("rdi") arg1,
100-
in("rsi") arg2,
101-
in("rdx") arg3,
102-
in("rcx") arg4,
103-
in("r8") arg5,
104-
in("r9") arg6,
105-
});
74+
kernel_function_impl!(kernel_function0(; u1: Reg, u2: Reg, u3: Reg, u4: Reg, u5: Reg, u6: Reg));
75+
kernel_function_impl!(kernel_function1(arg1: A1; u2: Reg, u3: Reg, u4: Reg, u5: Reg, u6: Reg));
76+
kernel_function_impl!(kernel_function2(arg1: A1, arg2: A2; u3: Reg, u4: Reg, u5: Reg, u6: Reg));
77+
kernel_function_impl!(kernel_function3(arg1: A1, arg2: A2, arg3: A3; u4: Reg, u5: Reg, u6: Reg));
78+
kernel_function_impl!(kernel_function4(arg1: A1, arg2: A2, arg3: A3, arg4: A4; u5: Reg, u6: Reg));
79+
kernel_function_impl!(kernel_function5(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5; u6: Reg));
80+
kernel_function_impl!(kernel_function6(arg1: A1, arg2: A2, arg3: A3, arg4: A4, arg5: A5, arg6: A6; ));

0 commit comments

Comments
 (0)