tool: add support for kernel domain scheduler

Signed-off-by: James Archer <j.archer@unsw.edu.au>

Author: JE-Archer

tool/microkit/src/main.rs

@@ -72,6 +72,7 @@ const INIT_VSPACE_CAP_ADDRESS: u64 = 3;
 const IRQ_CONTROL_CAP_ADDRESS: u64 = 4; // Singleton
 const INIT_ASID_POOL_CAP_ADDRESS: u64 = 6;
 const SMC_CAP_ADDRESS: u64 = 15;
+const DOMAIN_CAP_ADDRESS: u64 = 11;
 
 // const ASID_CONTROL_CAP_ADDRESS: u64 = 5; // Singleton
 // const IO_PORT_CONTROL_CAP_ADDRESS: u64 = 7; // Null on this platform
@@ -895,6 +896,7 @@ fn build_system(
     cap_address_names.insert(INIT_ASID_POOL_CAP_ADDRESS, "ASID Pool: init".to_string());
     cap_address_names.insert(IRQ_CONTROL_CAP_ADDRESS, "IRQ Control".to_string());
     cap_address_names.insert(SMC_CAP_IDX, "SMC".to_string());
+    cap_address_names.insert(DOMAIN_CAP_ADDRESS, "Domain Cap".to_string());
 
     let system_cnode_bits = system_cnode_size.ilog2() as u64;
 
@@ -2699,6 +2701,19 @@ fn build_system(
         system_invocations.push(tcb_cap_copy_invocation);
     }
 
+    for (pd_idx, pd) in system.protection_domains.iter().enumerate() {
+        if let Some(domain_id) = pd.domain_id {
+            system_invocations.push(Invocation::new(
+                config,
+                InvocationArgs::DomainSetSet {
+                    domain_set: DOMAIN_CAP_ADDRESS,
+                    domain: domain_id as u8,
+                    tcb: pd_tcb_objs[pd_idx].cap_addr,
+                },
+            ));
+        }
+    }
+
     // Set VSpace and CSpace
     let mut pd_set_space_invocation = Invocation::new(
         config,
@@ -3345,6 +3360,7 @@ fn main() -> Result<(), String> {
         arm_pa_size_bits,
         arm_smc,
         riscv_pt_levels: Some(RiscvVirtualMemory::Sv39),
+        domain_scheduler: json_str_as_u64(&kernel_config_json, "NUM_DOMAINS")? != 1,
     };
 
     if let Arch::Aarch64 = kernel_config.arch {
@@ -3378,9 +3394,27 @@ fn main() -> Result<(), String> {
         system_invocation_count_symbol_name: "system_invocation_count",
     };
 
-    let kernel_elf = ElfFile::from_path(&kernel_elf_path)?;
+    let mut kernel_elf = ElfFile::from_path(&kernel_elf_path)?;
     let mut monitor_elf = ElfFile::from_path(&monitor_elf_path)?;
 
+    if let Some(domain_schedule) = &system.domain_schedule {
+        let schedule = &domain_schedule.schedule;
+        kernel_elf.write_symbol(
+            "ksDomScheduleLength",
+            &(schedule.len() as u64).to_le_bytes(),
+        )?;
+
+        let mut out = Vec::new();
+        out.reserve_exact(schedule.len() * 16);
+
+        for timeslice in schedule.iter() {
+            out.extend(timeslice.id.to_le_bytes());
+            out.extend(timeslice.length.to_le_bytes());
+        }
+
+        kernel_elf.write_symbol("ksDomSchedule", &out)?;
+    }
+
     if monitor_elf.segments.iter().filter(|s| s.loadable).count() > 1 {
         eprintln!(
             "Monitor ({}) has {} segments, it must only have one",

tool/microkit/src/sdf.rs

@@ -7,6 +7,7 @@
 use crate::sel4::{Config, IrqTrigger, PageSize};
 use crate::util::str_to_bool;
 use crate::MAX_PDS;
+use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 ///
@@ -45,6 +46,9 @@ const PD_DEFAULT_STACK_SIZE: u64 = 0x1000;
 const PD_MIN_STACK_SIZE: u64 = 0x1000;
 const PD_MAX_STACK_SIZE: u64 = 1024 * 1024 * 16;
 
+/// The maximum length of domain schedule supported by the kernel
+const DOMAIN_SCHEDULE_MAX_LENGTH: usize = 256;
+
 /// The purpose of this function is to parse an integer that could
 /// either be in decimal or hex format, unlike the normal parsing
 /// functionality that the Rust standard library provides.
@@ -159,6 +163,8 @@ pub struct ProtectionDomain {
     pub parent: Option<usize>,
     /// Location in the parsed SDF file
     text_pos: roxmltree::TextPos,
+    /// Index into the domain schedule vector if the system is using domain scheduling
+    pub domain_id: Option<u64>,
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -176,6 +182,18 @@ pub struct VirtualCpu {
     pub id: u64,
 }
 
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct DomainTimeslice {
+    pub id: u64,
+    pub length: u64,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct DomainSchedule {
+    pub domain_ids: HashMap<String, u64>,
+    pub schedule: Vec<DomainTimeslice>,
+}
+
 impl SysMapPerms {
     fn from_str(s: &str) -> Result<u8, ()> {
         let mut perms = 0;
@@ -277,6 +295,7 @@ impl ProtectionDomain {
         xml_sdf: &XmlSystemDescription,
         node: &roxmltree::Node,
         is_child: bool,
+        domain_schedule: &Option<DomainSchedule>,
     ) -> Result<ProtectionDomain, String> {
         let mut attrs = vec![
             "name",
@@ -289,6 +308,7 @@ impl ProtectionDomain {
             // The SMC field is only available in certain configurations
             // but we do the error-checking further down.
             "smc",
+            "domain",
         ];
         if is_child {
             attrs.push("id");
@@ -418,6 +438,27 @@ impl ProtectionDomain {
             ));
         }
 
+        let mut domain_id = None;
+        match (domain_schedule, checked_lookup(xml_sdf, node, "domain")) {
+            (Some(domain_schedule), Ok(domain_name)) => {
+                domain_id = domain_schedule.domain_ids.get(domain_name);
+                if domain_id.is_none() {
+                    return Err(format!("Protection domain {} specifies a domain {} that is not in the domain schedule", name, domain_name));
+                }
+            }
+            (Some(_), _) => {
+                return Err(format!("System specifies a domain schedule but protection domain {} does not specify a domain", name))
+            }
+            (_, Ok(domain)) => {
+                if config.domain_scheduler {
+                    return Err(format!("Protection domain {} specifies a domain {} but system does not specify a domain schedule", name, domain));
+                } else {
+                    return Err("Assigning PDs to domains is only supported if SDK is built with --experimental-domain-support".to_string());
+                }
+            }
+            (_, _) => {}
+        }
+
         let mut maps = Vec::new();
         let mut irqs = Vec::new();
         let mut setvars: Vec<SysSetVar> = Vec::new();
@@ -549,9 +590,13 @@ impl ProtectionDomain {
                         vaddr: None,
                     })
                 }
-                "protection_domain" => {
-                    child_pds.push(ProtectionDomain::from_xml(config, xml_sdf, &child, true)?)
-                }
+                "protection_domain" => child_pds.push(ProtectionDomain::from_xml(
+                    config,
+                    xml_sdf,
+                    &child,
+                    true,
+                    domain_schedule,
+                )?),
                 "virtual_machine" => {
                     if virtual_machine.is_some() {
                         return Err(value_error(
@@ -604,6 +649,7 @@ impl ProtectionDomain {
             has_children,
             parent: None,
             text_pos: xml_sdf.doc.text_pos_at(node.range().start),
+            domain_id: domain_id.copied(),
         })
     }
 }
@@ -860,13 +906,82 @@ impl Channel {
     }
 }
 
+impl DomainSchedule {
+    fn from_xml(
+        xml_sdf: &XmlSystemDescription,
+        node: &roxmltree::Node,
+    ) -> Result<DomainSchedule, String> {
+        let pos = xml_sdf.doc.text_pos_at(node.range().start);
+
+        check_attributes(xml_sdf, node, &[])?;
+
+        let mut next_domain_id = 0;
+        let mut domain_ids = HashMap::new();
+        let mut schedule = Vec::new();
+        for child in node.children() {
+            if !child.is_element() {
+                continue;
+            }
+
+            let child_name = child.tag_name().name();
+            if child_name != "domain" {
+                return Err(format!(
+                    "Error: invalid XML element '{}': {}",
+                    child_name,
+                    loc_string(xml_sdf, pos)
+                ));
+            }
+
+            if schedule.len() == DOMAIN_SCHEDULE_MAX_LENGTH {
+                return Err(format!(
+                    "Error: length of domain schedule exceeds maximum of {}: {}",
+                    DOMAIN_SCHEDULE_MAX_LENGTH,
+                    loc_string(xml_sdf, pos)
+                ));
+            }
+
+            check_attributes(xml_sdf, &child, &["name", "length"])?;
+            let name = checked_lookup(xml_sdf, &child, "name")?;
+            let length = checked_lookup(xml_sdf, &child, "length")?.parse::<u64>();
+            if length.is_err() {
+                return Err(format!(
+                    "Error: invalid domain timeslice length '{}': {}",
+                    name,
+                    loc_string(xml_sdf, pos)
+                ));
+            }
+
+            let id = match domain_ids.get(name) {
+                Some(&id) => id,
+                None => {
+                    let id = next_domain_id;
+                    next_domain_id += 1;
+                    domain_ids.insert(name.to_string(), id);
+                    id
+                }
+            };
+
+            schedule.push(DomainTimeslice {
+                id,
+                length: length.unwrap(),
+            });
+        }
+
+        Ok(DomainSchedule {
+            domain_ids,
+            schedule,
+        })
+    }
+}
+
 struct XmlSystemDescription<'a> {
     filename: &'a str,
     doc: &'a roxmltree::Document<'a>,
 }
 
 #[derive(Debug)]
 pub struct SystemDescription {
+    pub domain_schedule: Option<DomainSchedule>,
     pub protection_domains: Vec<ProtectionDomain>,
     pub memory_regions: Vec<SysMemoryRegion>,
     pub channels: Vec<Channel>,
@@ -1041,6 +1156,7 @@ pub fn parse(filename: &str, xml: &str, config: &Config) -> Result<SystemDescrip
         doc: &doc,
     };
 
+    let mut domain_schedule = None;
     let mut root_pds = vec![];
     let mut mrs = vec![];
     let mut channels = vec![];
@@ -1059,18 +1175,38 @@ pub fn parse(filename: &str, xml: &str, config: &Config) -> Result<SystemDescrip
     // then parse the channels.
     let mut channel_nodes = Vec::new();
 
+    if config.domain_scheduler {
+        if let Some(domain_schedule_node) = system
+            .children()
+            .filter(|&child| child.is_element())
+            .find(|&child| child.tag_name().name() == "domain_schedule")
+        {
+            domain_schedule = Some(DomainSchedule::from_xml(&xml_sdf, &domain_schedule_node)?);
+        }
+    }
+
     for child in system.children() {
         if !child.is_element() {
             continue;
         }
 
         let child_name = child.tag_name().name();
         match child_name {
-            "protection_domain" => {
-                root_pds.push(ProtectionDomain::from_xml(config, &xml_sdf, &child, false)?)
-            }
+            "protection_domain" => root_pds.push(ProtectionDomain::from_xml(
+                config,
+                &xml_sdf,
+                &child,
+                false,
+                &domain_schedule,
+            )?),
             "channel" => channel_nodes.push(child),
             "memory_region" => mrs.push(SysMemoryRegion::from_xml(config, &xml_sdf, &child)?),
+            "domain_schedule" => {
+                if !config.domain_scheduler {
+                    let pos = xml_sdf.doc.text_pos_at(child.range().start);
+                    return Err(format!("Domain schedule is only supported if SDK is built with --experimental-domain-support: {}", loc_string(&xml_sdf, pos)));
+                }
+            }
             _ => {
                 let pos = xml_sdf.doc.text_pos_at(child.range().start);
                 return Err(format!(
@@ -1286,6 +1422,7 @@ pub fn parse(filename: &str, xml: &str, config: &Config) -> Result<SystemDescrip
     }
 
     Ok(SystemDescription {
+        domain_schedule,
         protection_domains: pds,
         memory_regions: mrs,
         channels,

tool/microkit/src/sel4.rs

@@ -56,6 +56,7 @@ pub struct Config {
     pub arm_smc: Option<bool>,
     /// RISC-V specific, what kind of virtual memory system (e.g Sv39)
     pub riscv_pt_levels: Option<RiscvVirtualMemory>,
+    pub domain_scheduler: bool,
 }
 
 impl Config {
@@ -1094,6 +1095,15 @@ impl Invocation {
                 arg_strs.push(Invocation::fmt_field_cap("tcb", tcb, cap_lookup));
                 (vcpu, cap_lookup.get(&vcpu).unwrap())
             }
+            InvocationArgs::DomainSetSet {
+                domain_set,
+                domain,
+                tcb,
+            } => {
+                arg_strs.push(Invocation::fmt_field("domain", domain as u64));
+                arg_strs.push(Invocation::fmt_field_cap("tcb", tcb, cap_lookup));
+                (domain_set, cap_lookup.get(&domain_set).unwrap().as_str())
+            }
         };
         _ = writeln!(
             f,
@@ -1129,6 +1139,7 @@ impl Invocation {
             InvocationLabel::CnodeCopy | InvocationLabel::CnodeMint => "CNode",
             InvocationLabel::SchedControlConfigureFlags => "SchedControl",
             InvocationLabel::ArmVcpuSetTcb => "VCPU",
+            InvocationLabel::DomainSetSet => "DomainSet",
             _ => panic!(
                 "Internal error: unexpected label when getting object type '{:?}'",
                 self.label
@@ -1157,6 +1168,7 @@ impl Invocation {
             InvocationLabel::CnodeMint => "Mint",
             InvocationLabel::SchedControlConfigureFlags => "ConfigureFlags",
             InvocationLabel::ArmVcpuSetTcb => "VCPUSetTcb",
+            InvocationLabel::DomainSetSet => "Set",
             _ => panic!(
                 "Internal error: unexpected label when getting method name '{:?}'",
                 self.label
@@ -1198,6 +1210,7 @@ impl InvocationArgs {
                 InvocationLabel::SchedControlConfigureFlags
             }
             InvocationArgs::ArmVcpuSetTcb { .. } => InvocationLabel::ArmVcpuSetTcb,
+            InvocationArgs::DomainSetSet { .. } => InvocationLabel::DomainSetSet,
         }
     }
 
@@ -1349,6 +1362,11 @@ impl InvocationArgs {
                 vec![sched_context],
             ),
             InvocationArgs::ArmVcpuSetTcb { vcpu, tcb } => (vcpu, vec![], vec![tcb]),
+            InvocationArgs::DomainSetSet {
+                domain_set,
+                domain,
+                tcb,
+            } => (domain_set, vec![domain as u64], vec![tcb]),
         }
     }
 }
@@ -1462,4 +1480,9 @@ pub enum InvocationArgs {
         vcpu: u64,
         tcb: u64,
     },
+    DomainSetSet {
+        domain_set: u64,
+        domain: u8,
+        tcb: u64,
+    },
 }