Add support for arm cca

Makefile

@@ -27,6 +27,9 @@ ifeq ($(SEV),1)
     INIT_SRC += $(SNP_INIT_SRC)
 	BUILD_INIT = 0
 endif
+ifeq ($(CCA), 1)
+    FEATURE_FLAGS := --features cca
+endif
 ifeq ($(VIRGL_RESOURCE_MAP2),1)
 	FEATURE_FLAGS += --features virgl_resource_map2
 endif

src/arch/Cargo.toml

@@ -5,6 +5,7 @@ authors = ["The Chromium OS Authors"]
 edition = "2021"
 
 [features]
+cca = [ "tee" ]
 tee = []
 amd-sev = [ "tee" ]
 efi = []
@@ -19,8 +20,12 @@ smbios = { path = "../smbios" }
 utils = { path = "../utils" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
-kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] }
-kvm-ioctls = ">=0.21"
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+kvm-ioctls = { version = ">=0.17", git = "https://github.com/virtee/kvm-ioctls", branch = "cca" }
+
+
+[target.'cfg(target_arch = "aarch64")'.dependencies]
+vm-fdt = ">= 0.2.0"
 
 [dev-dependencies]
 utils = { path = "../utils" }

src/arch/src/aarch64/linux/regs.rs

@@ -110,6 +110,9 @@ arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
 /// * `boot_ip` - Starting instruction pointer.
 /// * `mem` - Reserved DRAM for current VM.
 pub fn setup_regs(vcpu: &VcpuFd, cpu_id: u8, boot_ip: u64, mem: &GuestMemoryMmap) -> Result<()> {
+    // PSTATE cannot be accesed from the host in CCA
+    #[cfg(not(feature = "cca"))]
+    #[allow(deref_nullptr)]
     // Get the register index of the PSTATE (Processor State) register.
     vcpu.set_one_reg(arm64_core_reg!(pstate), &PSTATE_FAULT_BITS_64.to_le_bytes())
         .map_err(Error::SetCoreRegister)?;

src/cpuid/Cargo.toml

@@ -8,5 +8,5 @@ edition = "2021"
 vmm-sys-util = ">= 0.14"
 
 [target.'cfg(target_os = "linux")'.dependencies]
-kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] }
-kvm-ioctls = ">=0.21"
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+kvm-ioctls = { version = ">=0.17", git = "https://github.com/virtee/kvm-ioctls", branch = "cca" }

src/devices/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 
 [features]
 tee = []
+cca = [ "tee" ]
 amd-sev = ["blk", "tee"]
 net = []
 blk = []
@@ -43,8 +44,8 @@ lru = ">=0.9"
 [target.'cfg(target_os = "linux")'.dependencies]
 rutabaga_gfx = { path = "../rutabaga_gfx", features = ["x"], optional = true }
 caps = "0.5.5"
-kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] }
-kvm-ioctls = ">=0.21"
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+kvm-ioctls = { version = ">=0.17", git = "https://github.com/virtee/kvm-ioctls", branch = "cca" }
 
 [target.'cfg(any(target_arch = "aarch64", target_arch = "riscv64"))'.dependencies]
 vm-fdt = ">= 0.2.0"

src/devices/src/fdt/aarch64.rs

@@ -286,10 +286,12 @@ fn create_psci_node(fdt: &mut FdtWriter) -> Result<()> {
     // Two methods available: hvc and smc.
     // As per documentation, PSCI calls between a guest and hypervisor may use the HVC conduit instead of SMC.
     // So, since we are using kvm, we need to use hvc.
-    #[cfg(target_os = "linux")]
+    #[cfg(all(target_os = "linux", not(feature = "cca")))]
     fdt.property_string("method", "hvc")?;
     #[cfg(target_os = "macos")]
     fdt.property_string("method", "smc")?;
+    #[cfg(feature = "cca")]
+    fdt.property_string("method", "smc")?;
     fdt.end_node(node)?;
 
     Ok(())

src/devices/src/virtio/block/device.rs

@@ -27,6 +27,9 @@ use utils::eventfd::{EventFd, EFD_NONBLOCK};
 use virtio_bindings::{
     virtio_blk::*, virtio_config::VIRTIO_F_VERSION_1, virtio_ring::VIRTIO_RING_F_EVENT_IDX,
 };
+#[cfg(feature = "cca")]
+use virtio_bindings::virtio_config::VIRTIO_F_ACCESS_PLATFORM;
+
 use vm_memory::{ByteValued, GuestMemoryMmap};
 
 use super::worker::BlockWorker;
@@ -240,10 +243,19 @@ impl Block {
         let disk_properties =
             DiskProperties::new(Arc::clone(&disk_image), disk_image_id.clone(), cache_type)?;
 
-        let mut avail_features = (1u64 << VIRTIO_F_VERSION_1)
+
+        let mut avail_features = if cfg!(feature = "cca") {
+            (1u64 << VIRTIO_F_VERSION_1)
             | (1u64 << VIRTIO_BLK_F_FLUSH)
             | (1u64 << VIRTIO_BLK_F_SEG_MAX)
-            | (1u64 << VIRTIO_RING_F_EVENT_IDX);
+            | (1u64 << VIRTIO_RING_F_EVENT_IDX)
+            | (1 << VIRTIO_F_ACCESS_PLATFORM as u64)
+        } else {
+            (1u64 << VIRTIO_F_VERSION_1)
+            | (1u64 << VIRTIO_BLK_F_FLUSH)
+            | (1u64 << VIRTIO_BLK_F_SEG_MAX)
+            | (1u64 << VIRTIO_RING_F_EVENT_IDX)
+        };
 
         if is_disk_read_only {
             avail_features |= 1u64 << VIRTIO_BLK_F_RO;

src/devices/src/virtio/console/device.rs

@@ -10,6 +10,8 @@ use libc::TIOCGWINSZ;
 use nix::ioctl_read_bad;
 use utils::eventfd::EventFd;
 use vm_memory::{ByteValued, Bytes, GuestMemoryMmap};
+#[cfg(feature = "cca")]
+use virtio_bindings::virtio_config::VIRTIO_F_ACCESS_PLATFORM;
 
 use super::super::{
     ActivateError, ActivateResult, ConsoleError, DeviceState, Queue as VirtQueue, VirtioDevice,
@@ -30,9 +32,18 @@ use crate::virtio::{PortDescription, VmmExitObserver};
 pub(crate) const CONTROL_RXQ_INDEX: usize = 2;
 pub(crate) const CONTROL_TXQ_INDEX: usize = 3;
 
-pub(crate) const AVAIL_FEATURES: u64 = (1 << uapi::VIRTIO_CONSOLE_F_SIZE as u64)
-    | (1 << uapi::VIRTIO_CONSOLE_F_MULTIPORT as u64)
-    | (1 << uapi::VIRTIO_F_VERSION_1 as u64);
+// CCA requires VIRTIO_F_ACCESS_PLATFORM to ensure DMA-APIs
+// are triggered for virtio in Linux
+pub(crate) const AVAIL_FEATURES: u64 = if cfg!(feature = "cca") {
+    (1 << uapi::VIRTIO_CONSOLE_F_SIZE as u64)
+        | (1 << uapi::VIRTIO_CONSOLE_F_MULTIPORT as u64)
+        | (1 << uapi::VIRTIO_F_VERSION_1 as u64)
+        | (1 << VIRTIO_F_ACCESS_PLATFORM as u64)
+} else {
+    (1 << uapi::VIRTIO_CONSOLE_F_SIZE as u64)
+        | (1 << uapi::VIRTIO_CONSOLE_F_MULTIPORT as u64)
+        | (1 << uapi::VIRTIO_F_VERSION_1 as u64)
+};
 
 #[repr(C)]
 #[derive(Default)]

src/devices/src/virtio/fs/device.rs

@@ -9,7 +9,10 @@ use std::thread::JoinHandle;
 use utils::eventfd::{EventFd, EFD_NONBLOCK};
 #[cfg(target_os = "macos")]
 use utils::worker_message::WorkerMessage;
-use virtio_bindings::{virtio_config::VIRTIO_F_VERSION_1, virtio_ring::VIRTIO_RING_F_EVENT_IDX};
+use virtio_bindings::{
+    virtio_config::VIRTIO_F_ACCESS_PLATFORM, virtio_config::VIRTIO_F_VERSION_1,
+    virtio_ring::VIRTIO_RING_F_EVENT_IDX,
+};
 use vm_memory::{ByteValued, GuestMemoryMmap};
 
 use super::super::{
@@ -72,7 +75,13 @@ impl Fs {
                 .push(EventFd::new(utils::eventfd::EFD_NONBLOCK).map_err(FsError::EventFd)?);
         }
 
-        let avail_features = (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_RING_F_EVENT_IDX);
+        let avail_features = if cfg!(feature = "cca") {
+            (1u64 << VIRTIO_F_VERSION_1)
+                | (1u64 << VIRTIO_RING_F_EVENT_IDX)
+                | (1 << VIRTIO_F_ACCESS_PLATFORM as u64)
+        } else {
+            (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_RING_F_EVENT_IDX)
+        };
 
         let tag = fs_id.into_bytes();
         let mut config = VirtioFsConfig::default();

src/devices/src/virtio/mod.rs

@@ -21,7 +21,7 @@ pub mod console;
 pub mod descriptor_utils;
 pub mod device;
 pub mod file_traits;
-#[cfg(not(any(feature = "tee", feature = "nitro")))]
+#[cfg(not(any(feature = "tee", feature = "nitro", feature = "cca")))]
 pub mod fs;
 #[cfg(feature = "gpu")]
 pub mod gpu;
@@ -42,7 +42,7 @@ pub use self::balloon::*;
 pub use self::block::{Block, CacheType};
 pub use self::console::*;
 pub use self::device::*;
-#[cfg(not(any(feature = "tee", feature = "nitro")))]
+#[cfg(not(any(feature = "tee", feature = "nitro", feature = "cca")))]
 pub use self::fs::*;
 #[cfg(feature = "gpu")]
 pub use self::gpu::*;

src/devices/src/virtio/rng/device.rs

@@ -13,12 +13,17 @@ use super::super::{
 use super::{defs, defs::uapi};
 use crate::legacy::IrqChip;
 use crate::Error as DeviceError;
+use virtio_bindings::virtio_config::VIRTIO_F_ACCESS_PLATFORM;
 
 // Request queue.
 pub(crate) const REQ_INDEX: usize = 0;
 
 // Supported features.
-pub(crate) const AVAIL_FEATURES: u64 = 1 << uapi::VIRTIO_F_VERSION_1 as u64;
+pub(crate) const AVAIL_FEATURES: u64 = if cfg!(feature = "cca") {
+    1 << uapi::VIRTIO_F_VERSION_1 as u64 | 1 << VIRTIO_F_ACCESS_PLATFORM as u64
+} else {
+    1 << uapi::VIRTIO_F_VERSION_1 as u64
+};
 
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(C, packed)]

src/libkrun/Cargo.toml

@@ -7,6 +7,7 @@ build = "build.rs"
 
 [features]
 tee = []
+cca = [ "tee" ]
 amd-sev = [ "blk", "tee" ]
 net = []
 blk = []
@@ -19,11 +20,14 @@ nitro = [ "dep:nitro", "dep:nitro-enclaves" ]
 [dependencies]
 crossbeam-channel = ">=0.5.15"
 env_logger = "0.11"
+vm-memory = { version = ">=0.13", features = ["backend-mmap"] }
 libc = ">=0.2.39"
 libloading = "0.8"
 log = "0.4.0"
 once_cell = "1.4.1"
 
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+
 devices = { path = "../devices" }
 polly = { path = "../polly" }
 utils = { path = "../utils" }
@@ -33,8 +37,8 @@ vmm = { path = "../vmm" }
 hvf = { path = "../hvf" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
-kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] }
-kvm-ioctls = ">=0.21"
+kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+kvm-ioctls = { version = ">=0.21", git = "https://github.com/virtee/kvm-ioctls", branch = "cca" }
 nitro = { path = "../nitro", optional = true }
 nitro-enclaves = { version = "0.3.0", optional = true }
 vm-memory = ">=0.13"

src/libkrun/build.rs

@@ -12,4 +12,6 @@ fn main() {
     );
     #[cfg(target_os = "macos")]
     println!("cargo:rustc-link-lib=framework=Hypervisor");
+    #[cfg(feature = "cca")]
+    println!("cargo:rustc-link-lib=krunfw");
 }

src/libkrun/src/lib.rs

@@ -41,7 +41,7 @@ use vmm::vmm_config::block::BlockDeviceConfig;
 use vmm::vmm_config::boot_source::{BootSourceConfig, DEFAULT_KERNEL_CMDLINE};
 #[cfg(not(feature = "tee"))]
 use vmm::vmm_config::external_kernel::{ExternalKernel, KernelFormat};
-#[cfg(not(feature = "tee"))]
+#[cfg(any(not(feature = "tee"), feature = "cca"))]
 use vmm::vmm_config::fs::FsDeviceConfig;
 #[cfg(not(feature = "efi"))]
 use vmm::vmm_config::kernel_bundle::KernelBundle;
@@ -85,8 +85,10 @@ pub struct KrunfwBindings {
         unsafe extern "C" fn(*mut u64, *mut u64, *mut size_t) -> *mut c_char,
     >,
     #[cfg(feature = "tee")]
+    #[cfg(target_arch = "x86_64")]
     get_initrd: libloading::Symbol<'static, unsafe extern "C" fn(*mut size_t) -> *mut c_char>,
     #[cfg(feature = "tee")]
+    #[cfg(target_arch = "x86_64")]
     get_qboot: libloading::Symbol<'static, unsafe extern "C" fn(*mut size_t) -> *mut c_char>,
 }
 
@@ -101,8 +103,10 @@ impl KrunfwBindings {
             KrunfwBindings {
                 get_kernel: krunfw.get(b"krunfw_get_kernel")?,
                 #[cfg(feature = "tee")]
+                #[cfg(target_arch = "x86_64")]
                 get_initrd: krunfw.get(b"krunfw_get_initrd")?,
                 #[cfg(feature = "tee")]
+                #[cfg(target_arch = "x86_64")]
                 get_qboot: krunfw.get(b"krunfw_get_qboot")?,
             }
         })
@@ -255,6 +259,7 @@ impl ContextConfig {
     }
 
     #[cfg(feature = "tee")]
+    #[cfg(target_arch = "x86_64")]
     fn get_tee_config_file(&self) -> Option<PathBuf> {
         self.tee_config_file.clone()
     }
@@ -469,7 +474,7 @@ pub extern "C" fn krun_set_vm_config(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -
 
 #[allow(clippy::missing_safety_doc)]
 #[no_mangle]
-#[cfg(not(feature = "tee"))]
+#[cfg(any(not(feature = "tee"), feature = "cca"))]
 pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char) -> i32 {
     let root_path = match CStr::from_ptr(c_root_path).to_str() {
         Ok(root) => root,
@@ -1591,6 +1596,7 @@ unsafe fn load_krunfw_payload(
     vmr.set_kernel_bundle(kernel_bundle).unwrap();
 
     #[cfg(feature = "tee")]
+    #[cfg(target_arch = "x86_64")]
     {
         let mut qboot_size: usize = 0;
         let qboot_host_addr = unsafe { (krunfw.get_qboot)(&mut qboot_size as *mut usize) };
@@ -1738,6 +1744,7 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
      * fail.
      */
     #[cfg(feature = "tee")]
+    #[cfg(target_arch = "x86_64")]
     if let Some(tee_config) = ctx_cfg.get_tee_config_file() {
         if let Err(e) = ctx_cfg.vmr.set_tee_config(tee_config) {
             error!("Error setting up TEE config: {e:?}");
@@ -1850,6 +1857,9 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     #[cfg(feature = "amd-sev")]
     vmm::worker::start_worker_thread(_vmm.clone(), _receiver.clone()).unwrap();
 
+    #[cfg(feature = "cca")]
+    vmm::worker::start_worker_thread(_vmm.clone(), _receiver.clone()).unwrap();
+
     loop {
         match event_manager.run() {
             Ok(_) => {}

src/utils/Cargo.toml

@@ -12,4 +12,4 @@ vmm-sys-util = ">= 0.14"
 crossbeam-channel = ">=0.5.15"
 
 [target.'cfg(target_os = "linux")'.dependencies]
-kvm-bindings = { version = ">=0.10", features = ["fam-wrappers"] }
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }

src/vmm/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 [features]
 tee = []
 amd-sev = [ "blk", "tee", "codicon", "kbs-types", "procfs", "rdrand", "serde", "serde_json", "sev", "curl" ]
+cca = [ "tee" ]
 net = []
 blk = []
 efi = [ "blk", "net" ]
@@ -40,14 +41,16 @@ sev = { version = "6.0.0", features = ["openssl"], optional = true }
 curl = { version = "0.4", optional = true }
 nix = "0.24.1"
 
+cca = { git = "https://github.com/virtee/cca" }
+
 [target.'cfg(target_arch = "x86_64")'.dependencies]
 bzip2 = "0.5"
 cpuid = { path = "../cpuid" }
 zstd = "0.13"
 
 [target.'cfg(target_os = "linux")'.dependencies]
-kvm-bindings = { version = ">=0.11", features = ["fam-wrappers"] }
-kvm-ioctls = ">=0.21"
+kvm-bindings = { version = ">=0.8", features = ["fam-wrappers"] , git = "https://github.com/virtee/kvm-bindings", branch = "add_bindings_for_realms" }
+kvm-ioctls = { version = ">=0.17", git = "https://github.com/virtee/kvm-ioctls", branch = "cca" }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 hvf = { path = "../hvf" }