Skip to content

Commit 108f6fe

Browse files
committed
using snapshots
Signed-off-by: Simon Davies <[email protected]>
1 parent 3dd58b3 commit 108f6fe

22 files changed

+2998
-190
lines changed

Cargo.lock

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/hyperlight_common/src/mem.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ limitations under the License.
1717
pub const PAGE_SHIFT: u64 = 12;
1818
pub const PAGE_SIZE: u64 = 1 << 12;
1919
pub const PAGE_SIZE_USIZE: usize = 1 << 12;
20+
// The number of pages in 1 "block". A single u64 can be used as bitmap to keep track of all dirty pages in a block.
21+
pub const PAGES_IN_BLOCK: usize = 64;
2022

2123
/// A memory region in the guest address space
2224
#[derive(Debug, Clone, Copy)]

src/hyperlight_host/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ anyhow = "1.0"
4545
metrics = "0.24.2"
4646
serde_json = "1.0"
4747
elfcore = "2.0"
48+
lockfree ="0.5"
4849

4950
[target.'cfg(windows)'.dependencies]
5051
windows = { version = "0.61", features = [

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ use crate::sandbox::SandboxConfiguration;
7676
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
7777
use crate::{Result, log_then_return, new_error};
7878

79+
const CLEAR_DIRTY_BIT_FLAG: u8 = 0b100;
80+
7981
#[cfg(gdb)]
8082
mod debug {
8183
use std::sync::{Arc, Mutex};
@@ -826,6 +828,24 @@ impl Hypervisor for HypervLinuxDriver {
826828
self.interrupt_handle.clone()
827829
}
828830

831+
fn get_and_clear_dirty_pages(&mut self) -> Result<Vec<u64>> {
832+
let first_mshv_region: mshv_user_mem_region = self
833+
.mem_regions
834+
.first()
835+
.ok_or(new_error!(
836+
"tried to get dirty page bitmap of 0-sized region"
837+
))?
838+
.to_owned()
839+
.into();
840+
let total_size = self.mem_regions.iter().map(|r| r.guest_region.len()).sum();
841+
let res = self.vm_fd.get_dirty_log(
842+
first_mshv_region.guest_pfn,
843+
total_size,
844+
CLEAR_DIRTY_BIT_FLAG.into(),
845+
)?;
846+
Ok(res)
847+
}
848+
829849
#[cfg(crashdump)]
830850
fn crashdump_context(&self) -> Result<Option<super::crashdump::CrashDumpContext>> {
831851
if self.rt_cfg.guest_core_dump {

src/hyperlight_host/src/hypervisor/kvm.rs

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@ use std::sync::Arc;
2121
use std::sync::Mutex;
2222
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
2323

24-
use kvm_bindings::{KVM_MEM_READONLY, kvm_fpu, kvm_regs, kvm_userspace_memory_region};
24+
use hyperlight_common::mem::{PAGE_SIZE_USIZE, PAGES_IN_BLOCK};
25+
use kvm_bindings::{
26+
KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY, kvm_fpu, kvm_regs, kvm_userspace_memory_region,
27+
};
2528
use kvm_ioctls::Cap::UserMemory;
2629
use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd};
2730
use log::LevelFilter;
@@ -43,7 +46,8 @@ use super::{
4346
use super::{HyperlightExit, Hypervisor, InterruptHandle, LinuxInterruptHandle, VirtualCPU};
4447
#[cfg(gdb)]
4548
use crate::HyperlightError;
46-
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
49+
use crate::mem::bitmap::{bit_index_iterator, new_page_bitmap};
50+
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags, MemoryRegionType};
4751
use crate::mem::ptr::{GuestPtr, RawPtr};
4852
use crate::sandbox::SandboxConfiguration;
4953
#[cfg(crashdump)]
@@ -284,7 +288,7 @@ mod debug {
284288
/// A Hypervisor driver for KVM on Linux
285289
pub(crate) struct KVMDriver {
286290
_kvm: Kvm,
287-
_vm_fd: VmFd,
291+
vm_fd: VmFd,
288292
vcpu_fd: VcpuFd,
289293
entrypoint: u64,
290294
orig_rsp: GuestPtr,
@@ -329,7 +333,7 @@ impl KVMDriver {
329333
userspace_addr: region.host_region.start as u64,
330334
flags: match perm_flags {
331335
MemoryRegionFlags::READ => KVM_MEM_READONLY,
332-
_ => 0, // normal, RWX
336+
_ => KVM_MEM_LOG_DIRTY_PAGES, // normal, RWX
333337
},
334338
};
335339
unsafe { vm_fd.set_user_memory_region(kvm_region) }
@@ -378,7 +382,7 @@ impl KVMDriver {
378382
#[allow(unused_mut)]
379383
let mut hv = Self {
380384
_kvm: kvm,
381-
_vm_fd: vm_fd,
385+
vm_fd,
382386
vcpu_fd,
383387
entrypoint,
384388
orig_rsp: rsp_gp,
@@ -719,6 +723,45 @@ impl Hypervisor for KVMDriver {
719723
self.interrupt_handle.clone()
720724
}
721725

726+
fn get_and_clear_dirty_pages(&mut self) -> Result<Vec<u64>> {
727+
let mut page_indices = vec![];
728+
let mut current_page = 0;
729+
// Iterate over all memory regions and get the dirty pages for each region ignoring guard pages which cannot be dirty
730+
for (i, mem_region) in self.mem_regions.iter().enumerate() {
731+
let num_pages = mem_region.guest_region.len() / PAGE_SIZE_USIZE;
732+
let bitmap = match mem_region.flags {
733+
MemoryRegionFlags::READ => {
734+
// read-only page. It can never be dirty so return zero dirty pages.
735+
new_page_bitmap(mem_region.guest_region.len(), false)?
736+
}
737+
_ => {
738+
if mem_region.region_type == MemoryRegionType::GuardPage {
739+
// Trying to get dirty pages for a guard page region results in a VMMSysError(2)
740+
new_page_bitmap(mem_region.guest_region.len(), false)?
741+
} else {
742+
// Get the dirty bitmap for the memory region
743+
self.vm_fd
744+
.get_dirty_log(i as u32, mem_region.guest_region.len())?
745+
}
746+
}
747+
};
748+
for page_idx in bit_index_iterator(&bitmap) {
749+
page_indices.push(current_page + page_idx);
750+
}
751+
current_page += num_pages;
752+
}
753+
754+
// covert vec of page indices to vec of blocks
755+
let mut res = new_page_bitmap(current_page * PAGE_SIZE_USIZE, false)?;
756+
for page_idx in page_indices {
757+
let block_idx = page_idx / PAGES_IN_BLOCK;
758+
let bit_idx = page_idx % PAGES_IN_BLOCK;
759+
res[block_idx] |= 1 << bit_idx;
760+
}
761+
762+
Ok(res)
763+
}
764+
722765
#[cfg(crashdump)]
723766
fn crashdump_context(&self) -> Result<Option<crashdump::CrashDumpContext>> {
724767
if self.rt_cfg.guest_core_dump {

src/hyperlight_host/src/hypervisor/mod.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,11 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
187187
None
188188
}
189189

190+
/// Get dirty pages as a bitmap (Vec<u64>).
191+
/// Each bit in a u64 represents a page.
192+
/// This also clears the bitflags, marking the pages as non-dirty.
193+
fn get_and_clear_dirty_pages(&mut self) -> Result<Vec<u64>>;
194+
190195
/// Get InterruptHandle to underlying VM
191196
fn interrupt_handle(&self) -> Arc<dyn InterruptHandle>;
192197

@@ -548,12 +553,14 @@ pub(crate) mod tests {
548553
let rt_cfg: SandboxRuntimeConfig = Default::default();
549554
let sandbox =
550555
UninitializedSandbox::new(GuestBinary::FilePath(filename.clone()), Some(config))?;
556+
let tracker = sandbox.tracker;
551557
let (_hshm, mut gshm) = sandbox.mgr.build();
552558
let mut vm = set_up_hypervisor_partition(
553559
&mut gshm,
554560
&config,
555561
#[cfg(any(crashdump, gdb))]
556562
&rt_cfg,
563+
tracker,
557564
)?;
558565
vm.initialise(
559566
RawPtr::from(0x230000),
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
use std::cmp::Ordering;
2+
3+
use hyperlight_common::mem::{PAGE_SIZE_USIZE, PAGES_IN_BLOCK};
4+
5+
use crate::{Result, log_then_return};
6+
7+
// Contains various helper functions for dealing with bitmaps.
8+
9+
/// Returns a new bitmap of pages. If `init_dirty` is true, all pages are marked as dirty, otherwise all pages are clean.
10+
/// Will return an error if given size is 0.
11+
pub fn new_page_bitmap(size_in_bytes: usize, init_dirty: bool) -> Result<Vec<u64>> {
12+
if size_in_bytes == 0 {
13+
log_then_return!("Tried to create a bitmap with size 0.");
14+
}
15+
let num_pages = size_in_bytes.div_ceil(PAGE_SIZE_USIZE);
16+
let num_blocks = num_pages.div_ceil(PAGES_IN_BLOCK);
17+
match init_dirty {
18+
false => Ok(vec![0; num_blocks]),
19+
true => {
20+
let mut bitmap = vec![!0u64; num_blocks]; // all pages are dirty
21+
let num_unused_bits = num_blocks * PAGES_IN_BLOCK - num_pages;
22+
// set the unused bits to 0, could cause problems otherwise
23+
let last_block = bitmap.last_mut().unwrap(); // unwrap is safe since size_in_bytes>0
24+
*last_block >>= num_unused_bits;
25+
Ok(bitmap)
26+
}
27+
}
28+
}
29+
30+
/// Returns the union (bitwise OR) of two bitmaps. The resulting bitmap will have the same length
31+
/// as the longer of the two input bitmaps.
32+
pub(crate) fn bitmap_union(bitmap: &[u64], other_bitmap: &[u64]) -> Vec<u64> {
33+
let min_len = bitmap.len().min(other_bitmap.len());
34+
let max_len = bitmap.len().max(other_bitmap.len());
35+
36+
let mut result = vec![0; max_len];
37+
38+
for i in 0..min_len {
39+
result[i] = bitmap[i] | other_bitmap[i];
40+
}
41+
42+
match bitmap.len().cmp(&other_bitmap.len()) {
43+
Ordering::Greater => {
44+
result[min_len..].copy_from_slice(&bitmap[min_len..]);
45+
}
46+
Ordering::Less => {
47+
result[min_len..].copy_from_slice(&other_bitmap[min_len..]);
48+
}
49+
Ordering::Equal => {}
50+
}
51+
52+
result
53+
}
54+
55+
// Used as a helper struct to implement an iterator on.
56+
struct SetBitIndices<'a> {
57+
bitmap: &'a [u64],
58+
block_index: usize, // one block is 1 u64, which is 64 pages
59+
current: u64, // the current block we are iterating over, or 0 if first iteration
60+
}
61+
62+
/// Iterates over the zero-based indices of the set bits in the given bitmap.
63+
pub(crate) fn bit_index_iterator(bitmap: &[u64]) -> impl Iterator<Item = usize> + '_ {
64+
SetBitIndices {
65+
bitmap,
66+
block_index: 0,
67+
current: 0,
68+
}
69+
}
70+
71+
impl Iterator for SetBitIndices<'_> {
72+
type Item = usize;
73+
74+
fn next(&mut self) -> Option<Self::Item> {
75+
while self.current == 0 {
76+
// will always enter this on first iteration because current is initialized to 0
77+
if self.block_index >= self.bitmap.len() {
78+
// no more blocks to iterate over
79+
return None;
80+
}
81+
self.current = self.bitmap[self.block_index];
82+
self.block_index += 1;
83+
}
84+
let trailing_zeros = self.current.trailing_zeros();
85+
self.current &= self.current - 1; // Clear the least significant set bit
86+
Some((self.block_index - 1) * 64 + trailing_zeros as usize) // block_index guaranteed to be > 0 at this point
87+
}
88+
}
89+
90+
#[cfg(test)]
91+
mod tests {
92+
use hyperlight_common::mem::PAGE_SIZE_USIZE;
93+
94+
use crate::Result;
95+
use crate::mem::bitmap::{bit_index_iterator, bitmap_union, new_page_bitmap};
96+
97+
#[test]
98+
fn new_page_bitmap_test() -> Result<()> {
99+
let bitmap = new_page_bitmap(1, false)?;
100+
assert_eq!(bitmap.len(), 1);
101+
assert_eq!(bitmap[0], 0);
102+
103+
let bitmap = new_page_bitmap(1, true)?;
104+
assert_eq!(bitmap.len(), 1);
105+
assert_eq!(bitmap[0], 1);
106+
107+
let bitmap = new_page_bitmap(32 * PAGE_SIZE_USIZE, false)?;
108+
assert_eq!(bitmap.len(), 1);
109+
assert_eq!(bitmap[0], 0);
110+
111+
let bitmap = new_page_bitmap(32 * PAGE_SIZE_USIZE, true)?;
112+
assert_eq!(bitmap.len(), 1);
113+
assert_eq!(bitmap[0], 0x0000_0000_FFFF_FFFF);
114+
Ok(())
115+
}
116+
117+
#[test]
118+
fn page_iterator() {
119+
let data = vec![0b1000010100, 0b01, 0b100000000000000011];
120+
let mut iter = bit_index_iterator(&data);
121+
assert_eq!(iter.next(), Some(2));
122+
assert_eq!(iter.next(), Some(4));
123+
assert_eq!(iter.next(), Some(9));
124+
assert_eq!(iter.next(), Some(64));
125+
assert_eq!(iter.next(), Some(128));
126+
assert_eq!(iter.next(), Some(129));
127+
assert_eq!(iter.next(), Some(145));
128+
assert_eq!(iter.next(), None);
129+
130+
let data_2 = vec![0, 0, 0];
131+
let mut iter_2 = bit_index_iterator(&data_2);
132+
assert_eq!(iter_2.next(), None);
133+
134+
let data_3 = vec![0, 0, 0b1, 1 << 63];
135+
let mut iter_3 = bit_index_iterator(&data_3);
136+
assert_eq!(iter_3.next(), Some(128));
137+
assert_eq!(iter_3.next(), Some(255));
138+
assert_eq!(iter_3.next(), None);
139+
140+
let data_4 = vec![];
141+
let mut iter_4 = bit_index_iterator(&data_4);
142+
assert_eq!(iter_4.next(), None);
143+
}
144+
145+
#[test]
146+
fn union() -> Result<()> {
147+
let a = 0b1000010100;
148+
let b = 0b01;
149+
let c = 0b100000000000000011;
150+
let d = 0b101010100000011000000011;
151+
let e = 0b000000000000001000000000000000000000;
152+
let f = 0b100000000000000001010000000001010100000000000;
153+
let bitmap = vec![a, b, c];
154+
let other_bitmap = vec![d, e, f];
155+
let union = bitmap_union(&bitmap, &other_bitmap);
156+
assert_eq!(union, vec![a | d, b | e, c | f]);
157+
158+
// different length
159+
let union = bitmap_union(&[a], &[d, e, f]);
160+
assert_eq!(union, vec![a | d, e, f]);
161+
162+
let union = bitmap_union(&[a, b, c], &[d]);
163+
assert_eq!(union, vec![a | d, b, c]);
164+
165+
let union = bitmap_union(&[], &[d, e]);
166+
assert_eq!(union, vec![d, e]);
167+
168+
let union = bitmap_union(&[a, b, c], &[]);
169+
assert_eq!(union, vec![a, b, c]);
170+
171+
let union = bitmap_union(&[], &[]);
172+
let empty: Vec<u64> = vec![];
173+
assert_eq!(union, empty);
174+
175+
Ok(())
176+
}
177+
}

0 commit comments

Comments
 (0)