Skip to content

Commit 7a3eb47

Browse files
fwsGonzofwsGonzo
committed
Perform MSR writes in assembly
1 parent 1d375f4 commit 7a3eb47

File tree

5 files changed

+71
-44
lines changed

5 files changed

+71
-44
lines changed

‎lib/tinykvm/amd64/builtin/interrupts.asm‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,22 @@ ALIGN 0x10
238238

239239
.vm64_remote_disconnect:
240240
out 0, eax
241+
;; RAX contains the original FSBASE of this VM
242+
stac
243+
;; Write to FSBASE MSR
244+
push rcx
245+
push rdx
246+
mov ecx, 0xC0000100 ;; FSBASE
247+
mov rdx, rax
248+
shr rdx, 32
249+
wrmsr
250+
pop rdx
251+
pop rcx
252+
clac
253+
;; Reset pagetables
254+
mov rax, cr3
255+
mov cr3, rax
256+
o64 sysret
241257

242258
.vm64_entrycall:
243259
;; Reset pagetables
@@ -263,15 +279,16 @@ ALIGN 0x10
263279
iretq
264280

265281
.vm64_remote_page_fault:
266-
;; 1. We need to save current usermode state
267-
;; 2. Switch to the remote VMs FSBASE
268-
;; 4. Set up a usermode stack/function call
269-
;; with the faulting address as the function address
270-
;; 5. Exit kernel mode to usermode in a way that
271-
;; returns to the remote VM, performing the function call
272-
;; 6. When the remote VM is done, it should somehow enter
273-
;; kernel mode again, and we should restore our state
274-
;; 7. Return from the page fault
282+
;; RAX: Remote FSBASE
283+
;; Write to FSBASE MSR
284+
push rcx
285+
push rdx
286+
mov ecx, 0xC0000100 ;; FSBASE
287+
mov rdx, rax
288+
shr rdx, 32
289+
wrmsr
290+
pop rdx
291+
pop rcx
275292

276293
;; Make the next function call return to a custom system call location
277294
push rbx

‎lib/tinykvm/amd64/builtin/kernel_assembly.h‎

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
unsigned char interrupts[] = {
2-
0x50, 0x01, 0xc2, 0x02, 0x18, 0x03, 0x08, 0x00, 0xca, 0x02, 0x00, 0x00,
2+
0x50, 0x01, 0xc2, 0x02, 0x50, 0x03, 0x08, 0x00, 0xca, 0x02, 0x00, 0x00,
33
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44
0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x00, 0x00, 0x00, 0x00,
55
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -29,9 +29,9 @@ unsigned char interrupts[] = {
2929
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3030
0x66, 0x3d, 0x9e, 0x00, 0x74, 0x39, 0x66, 0x3d, 0xe4, 0x00, 0x0f, 0x84,
3131
0xee, 0x00, 0x00, 0x00, 0x83, 0xf8, 0x09, 0x0f, 0x84, 0x40, 0x01, 0x00,
32-
0x00, 0x3d, 0x77, 0xf7, 0x01, 0x00, 0x0f, 0x84, 0x5e, 0x01, 0x00, 0x00,
32+
0x00, 0x3d, 0x77, 0xf7, 0x01, 0x00, 0x0f, 0x84, 0x7f, 0x01, 0x00, 0x00,
3333
0x3d, 0x78, 0xf7, 0x01, 0x00, 0x0f, 0x84, 0x51, 0x01, 0x00, 0x00, 0x3d,
34-
0x07, 0xf7, 0x01, 0x00, 0x0f, 0x84, 0x51, 0x01, 0x00, 0x00, 0xe7, 0x00,
34+
0x07, 0xf7, 0x01, 0x00, 0x0f, 0x84, 0x72, 0x01, 0x00, 0x00, 0xe7, 0x00,
3535
0x48, 0x0f, 0x07, 0x0f, 0x01, 0xcb, 0x56, 0x51, 0x52, 0x48, 0x81, 0xff,
3636
0x02, 0x10, 0x00, 0x00, 0x75, 0x1b, 0xb9, 0x00, 0x01, 0x00, 0xc0, 0x89,
3737
0xf0, 0x48, 0xc1, 0xee, 0x20, 0x89, 0xf2, 0x0f, 0x30, 0x48, 0x31, 0xc0,
@@ -59,26 +59,31 @@ unsigned char interrupts[] = {
5959
0x83, 0xf8, 0xff, 0x74, 0x0e, 0x0f, 0x01, 0xcb, 0x50, 0x0f, 0x20, 0xd8,
6060
0x0f, 0x22, 0xd8, 0x58, 0x0f, 0x01, 0xca, 0x48, 0x0f, 0x07, 0xb8, 0x60,
6161
0x00, 0x00, 0x00, 0xe7, 0x00, 0xc3, 0xb8, 0xc2, 0x02, 0x00, 0x00, 0xc3,
62-
0xe7, 0x00, 0x0f, 0x20, 0xd8, 0x0f, 0x22, 0xd8, 0x48, 0x0f, 0x07, 0x48,
63-
0x0f, 0x07, 0x50, 0x57, 0x0f, 0x20, 0xd7, 0xe7, 0x8e, 0x0f, 0x01, 0x3f,
64-
0x5f, 0x85, 0xc0, 0x75, 0x07, 0x58, 0x48, 0x83, 0xc4, 0x08, 0x48, 0xcf,
65-
0x53, 0x48, 0x8b, 0x04, 0x25, 0x0a, 0x20, 0x00, 0x00, 0x48, 0x8b, 0x5c,
66-
0x24, 0x30, 0x0f, 0x01, 0xcb, 0x48, 0x89, 0x03, 0x0f, 0x01, 0xca, 0x5b,
67-
0x58, 0x48, 0x83, 0xc4, 0x08, 0x48, 0xcf, 0xe7, 0xa1, 0x48, 0xcf, 0x90,
68-
0xe7, 0x80, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x81, 0x48, 0xcf,
69-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x82, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
70-
0xe7, 0x83, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x84, 0x48, 0xcf,
71-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x85, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
72-
0xe7, 0x86, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x87, 0x48, 0xcf,
73-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x88, 0xeb, 0x92, 0x90, 0x90, 0x90, 0x90,
74-
0xe7, 0x89, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x8a, 0xeb, 0x82,
75-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x8b, 0xe9, 0x77, 0xff, 0xff, 0xff, 0x90,
76-
0xe7, 0x8c, 0xe9, 0x6f, 0xff, 0xff, 0xff, 0x90, 0xe7, 0x8d, 0xe9, 0x67,
77-
0xff, 0xff, 0xff, 0x90, 0xe9, 0x51, 0xff, 0xff, 0xff, 0x90, 0x90, 0x90,
78-
0xe7, 0x8f, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x90, 0x48, 0xcf,
79-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x91, 0xe9, 0x47, 0xff, 0xff, 0xff, 0x90,
80-
0xe7, 0x92, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x93, 0x48, 0xcf,
81-
0x90, 0x90, 0x90, 0x90, 0xe7, 0x94, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
82-
0xe9, 0x4e, 0xff, 0xff, 0xff
62+
0xe7, 0x00, 0x0f, 0x01, 0xcb, 0x51, 0x52, 0xb9, 0x00, 0x01, 0x00, 0xc0,
63+
0x48, 0x89, 0xc2, 0x48, 0xc1, 0xea, 0x20, 0x0f, 0x30, 0x5a, 0x59, 0x0f,
64+
0x01, 0xca, 0x0f, 0x20, 0xd8, 0x0f, 0x22, 0xd8, 0x48, 0x0f, 0x07, 0x0f,
65+
0x20, 0xd8, 0x0f, 0x22, 0xd8, 0x48, 0x0f, 0x07, 0x48, 0x0f, 0x07, 0x50,
66+
0x57, 0x0f, 0x20, 0xd7, 0xe7, 0x8e, 0x0f, 0x01, 0x3f, 0x5f, 0x85, 0xc0,
67+
0x75, 0x07, 0x58, 0x48, 0x83, 0xc4, 0x08, 0x48, 0xcf, 0x51, 0x52, 0xb9,
68+
0x00, 0x01, 0x00, 0xc0, 0x48, 0x89, 0xc2, 0x48, 0xc1, 0xea, 0x20, 0x0f,
69+
0x30, 0x5a, 0x59, 0x53, 0x48, 0x8b, 0x04, 0x25, 0x0a, 0x20, 0x00, 0x00,
70+
0x48, 0x8b, 0x5c, 0x24, 0x30, 0x0f, 0x01, 0xcb, 0x48, 0x89, 0x03, 0x0f,
71+
0x01, 0xca, 0x5b, 0x58, 0x48, 0x83, 0xc4, 0x08, 0x48, 0xcf, 0xe7, 0xa1,
72+
0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x80, 0x48, 0xcf,
73+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x81, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
74+
0xe7, 0x82, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x83, 0x48, 0xcf,
75+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x84, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
76+
0xe7, 0x85, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe7, 0x86, 0x48, 0xcf,
77+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x87, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
78+
0xe7, 0x88, 0xe9, 0x78, 0xff, 0xff, 0xff, 0x90, 0xe7, 0x89, 0x48, 0xcf,
79+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x8a, 0xe9, 0x68, 0xff, 0xff, 0xff, 0x90,
80+
0xe7, 0x8b, 0xe9, 0x60, 0xff, 0xff, 0xff, 0x90, 0xe7, 0x8c, 0xe9, 0x58,
81+
0xff, 0xff, 0xff, 0x90, 0xe7, 0x8d, 0xe9, 0x50, 0xff, 0xff, 0xff, 0x90,
82+
0xe9, 0x3a, 0xff, 0xff, 0xff, 0x90, 0x90, 0x90, 0xe7, 0x8f, 0x48, 0xcf,
83+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x90, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
84+
0xe7, 0x91, 0xe9, 0x30, 0xff, 0xff, 0xff, 0x90, 0xe7, 0x92, 0x48, 0xcf,
85+
0x90, 0x90, 0x90, 0x90, 0xe7, 0x93, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90,
86+
0xe7, 0x94, 0x48, 0xcf, 0x90, 0x90, 0x90, 0x90, 0xe9, 0x49, 0xff, 0xff,
87+
0xff
8388
};
84-
unsigned int interrupts_len = 965;
89+
unsigned int interrupts_len = 1021;

‎lib/tinykvm/machine.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,8 +248,8 @@ struct Machine
248248

249249
/* Remote VM through address space merging */
250250
void remote_connect(Machine& other, bool connect_now = true);
251-
void remote_activate_now();
252-
void remote_disconnect();
251+
address_t remote_activate_now();
252+
address_t remote_disconnect();
253253
bool is_remote_connected() const noexcept { return m_remote != nullptr; };
254254
address_t remote_base_address() const noexcept;
255255
const Machine& remote() const;

‎lib/tinykvm/remote.cpp‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,21 @@ void Machine::remote_connect(Machine& remote, bool connect_now)
6262
this->m_remote = &remote;
6363
}
6464

65-
void Machine::remote_activate_now()
65+
Machine::address_t Machine::remote_activate_now()
6666
{
6767
this->remote_connect(*this->m_remote, true);
6868

6969
// Set current FSBASE to remote FSBASE
7070
vcpu.remote_original_tls_base = get_fsgs().first;
71-
this->set_tls_base(this->m_remote->get_fsgs().first);
71+
72+
// Return FSBASE of remote, which can be set more efficiently
73+
// in the mini-kernel assembly
74+
return this->m_remote->get_fsgs().first;
7275
}
73-
void Machine::remote_disconnect()
76+
Machine::address_t Machine::remote_disconnect()
7477
{
7578
if (this->m_remote == nullptr)
76-
return;
79+
return 0;
7780

7881
// Unpresent gigabyte entries from remote VM in this VM
7982
const auto remote_vmem = this->m_remote->main_memory().vmem();
@@ -91,8 +94,9 @@ void Machine::remote_disconnect()
9194
}
9295

9396
// Restore original FSBASE
94-
this->set_tls_base(vcpu.remote_original_tls_base);
97+
auto result = vcpu.remote_original_tls_base;
9598
vcpu.remote_original_tls_base = 0;
99+
return result;
96100
}
97101

98102

‎lib/tinykvm/vcpu_run.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -225,11 +225,13 @@ long vCPU::run_once()
225225
printf("Remote VM disconnect syscall, return=0x%lX\n",
226226
this->remote_return_address);
227227
}
228-
machine().remote_disconnect();
228+
const auto result = machine().remote_disconnect();
229229
// Overwrite return address to return to the remote VM handler
230230
machine().copy_to_guest(this->registers().rsp + 24,
231231
&this->remote_return_address, 8);
232232
this->remote_return_address = 0;
233+
this->registers().rax = result;
234+
this->set_registers(this->registers());
233235
return KVM_EXIT_IO;
234236
} else {
235237
Machine::machine_exception("Invalid syscall number", intr);
@@ -278,8 +280,7 @@ long vCPU::run_once()
278280
printf("Page fault in remote VM at 0x%lX return=0x%lX, connecting...\n", addr, retaddr);
279281
}
280282
this->remote_return_address = retaddr;
281-
machine().remote_activate_now();
282-
regs.rax = 1; /* Indicate that it was remote */
283+
regs.rax = machine().remote_activate_now();
283284
this->set_registers(regs);
284285
return KVM_EXIT_IO;
285286
} else {

0 commit comments

Comments
 (0)