feat(sev): add AMD SEV support (#542)

* feat(sev): add AMD SEV support

* x86_64/sev: move SEV structures to codebase
x86_64/sev: remove SEV detection, replace with a simpler c-bit defining routine

* x86_64/memory_encryption: don't enable ME by default

* x86_64/memory_encryption: fix review and clippy issues

* memory_encryption: refactor for Intel TDX support

* memory_encryption: enable memory_encryption flag in CI

* memory_encryption: reduce function duplication in page_table.rs

* memory_encryption: remove feature `dynamic_flags`

* memory_encryption: prevent setting the encryption bit in PhysAddr

* memory_encryption: missing documentation

* ci: test memory_encryption separately

* memory_encryption: address PR feedback

* memory_encryption: keep `const` functions const if `memory_encryption` is not enabled

* memory_encryption: exclude memory_encryption feature when checking for semver

* memory_encryption: fix `const_fn` name already taken

* memory_encryption: fix formatting

* memory_encryption: review fixes

* memory_encryption: review fixes + fix imports changes

* memory_encryption: fix import style

* ci: only run semver checks on default features

Author: zyuiop

.github/workflows/build.yml

@@ -35,10 +35,13 @@ jobs:
         with:
           toolchain: ${{ matrix.rust }}
       - run: cargo build --no-default-features --features instructions
+      - run: cargo build --no-default-features --features memory_encryption
       - run: cargo build --no-default-features
       - run: cargo doc --no-default-features --features instructions
+      - run: cargo doc --no-default-features --features memory_encryption
       - run: cargo doc --no-default-features
       - run: cargo test --no-default-features --features instructions
+      - run: cargo test --no-default-features --features memory_encryption
       - run: cargo test --no-default-features
 
   test:
@@ -173,7 +176,7 @@ jobs:
           cache-targets: false
       - run: cargo install cargo-semver-checks --locked
       - name: Check semver
-        run: cargo +stable semver-checks check-release
+        run: cargo +stable semver-checks check-release --default-features
 
   kani:
     runs-on: ubuntu-latest

Cargo.toml

@@ -24,10 +24,12 @@ bit_field = "0.10.1"
 bitflags = "2.3.2"
 volatile = "0.4.4"
 rustversion = "1.0.5"
+dep_const_fn = { package = "const_fn", version = "0.4.11" }
 
 [features]
 default = ["nightly", "instructions"]
 instructions = []
+memory_encryption = []
 nightly = ["const_fn", "step_trait", "abi_x86_interrupt", "asm_const"]
 abi_x86_interrupt = []
 # deprecated, no longer needed

src/addr.rs

@@ -5,10 +5,16 @@ use core::fmt;
 #[cfg(feature = "step_trait")]
 use core::iter::Step;
 use core::ops::{Add, AddAssign, Sub, SubAssign};
+#[cfg(feature = "memory_encryption")]
+use core::sync::atomic::Ordering;
 
+#[cfg(feature = "memory_encryption")]
+use crate::structures::mem_encrypt::ENC_BIT_MASK;
 use crate::structures::paging::page_table::PageTableLevel;
 use crate::structures::paging::{PageOffset, PageTableIndex};
+
 use bit_field::BitField;
+use dep_const_fn::const_fn;
 
 const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000;
 
@@ -503,7 +509,10 @@ impl PhysAddr {
     /// ## Panics
     ///
     /// This function panics if a bit in the range 52 to 64 is set.
+    // If the `memory_encryption` feature has been enabled and an encryption bit has been
+    // configured, this also panics if the encryption bit is manually set in the address.
     #[inline]
+    #[const_fn(cfg(not(feature = "memory_encryption")))]
     pub const fn new(addr: u64) -> Self {
         // TODO: Replace with .ok().expect(msg) when that works on stable.
         match Self::try_new(addr) {
@@ -513,11 +522,19 @@ impl PhysAddr {
     }
 
     /// Creates a new physical address, throwing bits 52..64 away.
+    #[cfg(not(feature = "memory_encryption"))]
     #[inline]
     pub const fn new_truncate(addr: u64) -> PhysAddr {
         PhysAddr(addr % (1 << 52))
     }
 
+    /// Creates a new physical address, throwing bits 52..64 and the encryption bit away.
+    #[cfg(feature = "memory_encryption")]
+    #[inline]
+    pub fn new_truncate(addr: u64) -> PhysAddr {
+        PhysAddr((addr % (1 << 52)) & !ENC_BIT_MASK.load(Ordering::Relaxed))
+    }
+
     /// Creates a new physical address, without any checks.
     ///
     /// ## Safety
@@ -531,7 +548,10 @@ impl PhysAddr {
     /// Tries to create a new physical address.
     ///
     /// Fails if any bits in the range 52 to 64 are set.
+    /// If the `memory_encryption` feature has been enabled and an encryption bit has been
+    /// configured, this also fails if the encryption bit is manually set in the address.
     #[inline]
+    #[const_fn(cfg(not(feature = "memory_encryption")))]
     pub const fn try_new(addr: u64) -> Result<Self, PhysAddrNotValid> {
         let p = Self::new_truncate(addr);
         if p.0 == addr {

src/structures/mem_encrypt.rs

@@ -0,0 +1,87 @@
+//! Provides glue to support memory encryption features of some CPUs.
+//!
+//! Memory encryption typically relies on using a physical address bit in the page table entry to
+//! mark a page as encrypted/decrypted. This module provides a function that, given the position and
+//! type of that physical address bit, updates the dynamic page-table flags structure to ensure that
+//! the flag is treated properly and not returned as part of the physical address.
+
+use core::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+
+use crate::structures::paging::page_table::PHYSICAL_ADDRESS_MASK;
+use crate::structures::paging::PageTableFlags;
+
+/// Position of the encryption (C/S) bit in the physical address
+pub(crate) static ENC_BIT_MASK: AtomicU64 = AtomicU64::new(0);
+
+/// Is the encryption bit reversed (i.e. its presence denote that the page is _decrypted_ rather
+/// than encrypted)
+static ENC_BIT_REVERSED: AtomicBool = AtomicBool::new(false);
+
+/// Defines the configuration for memory encryption
+#[derive(Debug)]
+pub enum MemoryEncryptionConfiguration {
+    /// Defines that a memory page should be accessed encrypted if this bit of its physical address
+    /// is set in the page table entry.
+    ///
+    /// Use this for AMD Secure Memory Encryption (AMD-SME) and Secure Encrypted Virtualization (SEV)
+    EncryptedBit(u8),
+
+    /// Defines that a memory page should be accessed decrypted if this bit of its physical address
+    /// is set in the page table entry.
+    ///
+    /// Use this for Intel Trust Domain Extension (Intel TDX)
+    SharedBit(u8),
+}
+
+/// Enable memory encryption by defining the physical address bit that is used to mark a page
+/// encrypted (or shared) in a page table entry
+///
+/// # Safety
+/// Caller must make sure that any existing page table entry is discarded or adapted to take this
+/// bit into consideration.
+/// The configuration provided by caller must be correct, otherwise physical address bits will
+/// incorrectly be considered as page table flags.
+pub unsafe fn enable_memory_encryption(configuration: MemoryEncryptionConfiguration) {
+    let (bit_position, reversed) = match configuration {
+        MemoryEncryptionConfiguration::EncryptedBit(pos) => (pos, false),
+        MemoryEncryptionConfiguration::SharedBit(pos) => (pos, true),
+    };
+
+    let c_bit_mask = 1u64 << bit_position;
+
+    PHYSICAL_ADDRESS_MASK.fetch_and(!c_bit_mask, Ordering::Relaxed);
+    ENC_BIT_MASK.store(c_bit_mask, Ordering::Relaxed);
+    ENC_BIT_REVERSED.store(reversed, Ordering::Release);
+}
+
+impl PageTableFlags {
+    #[inline]
+    fn enc_bit_flag() -> Option<PageTableFlags> {
+        let bit_mask = ENC_BIT_MASK.load(Ordering::Relaxed);
+
+        if bit_mask > 0 {
+            Some(PageTableFlags::from_bits_retain(bit_mask))
+        } else {
+            None
+        }
+    }
+
+    /// Marks the page for encryption in the page table entry.
+    ///
+    /// # Panics
+    ///
+    /// Panics if memory encryption has not been previously configured by calling [`enable_memory_encryption`]
+    pub fn set_encrypted(&mut self, encrypted: bool) {
+        let flag = Self::enc_bit_flag().expect("memory encryption is not enabled");
+        self.set(flag, encrypted ^ ENC_BIT_REVERSED.load(Ordering::Relaxed));
+    }
+
+    /// Checks if memory encryption is enabled on the page
+    pub fn is_encrypted(&self) -> bool {
+        if let Some(c_bit_flag) = Self::enc_bit_flag() {
+            self.contains(c_bit_flag) ^ ENC_BIT_REVERSED.load(Ordering::Relaxed)
+        } else {
+            false
+        }
+    }
+}

src/structures/mod.rs

@@ -6,6 +6,8 @@ pub mod gdt;
 
 pub mod idt;
 
+#[cfg(feature = "memory_encryption")]
+pub mod mem_encrypt;
 pub mod paging;
 pub mod port;
 pub mod tss;

src/structures/paging/page_table.rs

@@ -4,11 +4,14 @@ use core::fmt;
 #[cfg(feature = "step_trait")]
 use core::iter::Step;
 use core::ops::{Index, IndexMut};
+#[cfg(feature = "memory_encryption")]
+use core::sync::atomic::{AtomicU64, Ordering};
 
 use super::{PageSize, PhysFrame, Size4KiB};
 use crate::addr::PhysAddr;
 
 use bitflags::bitflags;
+use dep_const_fn::const_fn;
 
 /// The error returned by the `PageTableEntry::frame` method.
 #[derive(Debug, Clone, Copy, PartialEq)]
@@ -20,6 +23,10 @@ pub enum FrameError {
     HugeFrame,
 }
 
+/// The mask used to remove flags from a page table entry to obtain the physical address
+#[cfg(feature = "memory_encryption")]
+pub(crate) static PHYSICAL_ADDRESS_MASK: AtomicU64 = AtomicU64::new(0x000f_ffff_ffff_f000u64);
+
 /// A 64-bit page table entry.
 #[derive(Clone)]
 #[repr(transparent)]
@@ -48,14 +55,15 @@ impl PageTableEntry {
 
     /// Returns the flags of this entry.
     #[inline]
+    #[const_fn(cfg(not(feature = "memory_encryption")))]
     pub const fn flags(&self) -> PageTableFlags {
-        PageTableFlags::from_bits_truncate(self.entry)
+        PageTableFlags::from_bits_retain(self.entry & !Self::physical_address_mask())
     }
 
     /// Returns the physical address mapped by this entry, might be zero.
     #[inline]
     pub fn addr(&self) -> PhysAddr {
-        PhysAddr::new(self.entry & 0x000f_ffff_ffff_f000)
+        PhysAddr::new(self.entry & Self::physical_address_mask())
     }
 
     /// Returns the physical frame mapped by this entry.
@@ -95,6 +103,18 @@ impl PageTableEntry {
     pub fn set_flags(&mut self, flags: PageTableFlags) {
         self.entry = self.addr().as_u64() | flags.bits();
     }
+
+    #[inline(always)]
+    #[cfg(not(feature = "memory_encryption"))]
+    const fn physical_address_mask() -> u64 {
+        0x000f_ffff_ffff_f000u64
+    }
+
+    #[inline(always)]
+    #[cfg(feature = "memory_encryption")]
+    fn physical_address_mask() -> u64 {
+        PHYSICAL_ADDRESS_MASK.load(Ordering::Relaxed)
+    }
 }
 
 impl Default for PageTableEntry {