Skip to content

Commit 4be023f

Browse files
JamesC1305JamesC1305
committed
MMIO Portion of hotplugging
Implement MMIO for the CpuContainer. This allows the AML methods to read from and write to the defined AML fields. Signed-off-by: James Curtis <[email protected]>
1 parent 915a344 commit 4be023f

File tree

7 files changed

+171
-22
lines changed

7 files changed

+171
-22
lines changed

‎src/vmm/src/arch/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ pub enum DeviceType {
4141
Rtc,
4242
/// Device Type: BootTimer.
4343
BootTimer,
44+
/// Device Type: CpuContainer
45+
#[cfg(target_arch = "x86_64")]
46+
CpuContainer,
4447
}
4548

4649
/// Type for passing information about the initrd in the guest memory.

‎src/vmm/src/builder.rs

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,8 +1028,16 @@ fn attach_cpu_container_device(vmm: &mut Vmm, vcpu_count: u8) -> Result<(), Star
10281028
vcpu_count,
10291029
)?));
10301030
vmm.acpi_device_manager
1031-
.attach_cpu_container(cpu_container, vmm.vm.fd())
1031+
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd())
10321032
.map_err(StartMicrovmError::AttachCpuContainerDevice)?;
1033+
vmm.mmio_device_manager
1034+
.register_mmio_cpu_container_for_boot(
1035+
vmm.vm.fd(),
1036+
&mut vmm.resource_allocator,
1037+
cpu_container,
1038+
)
1039+
.map_err(StartMicrovmError::RegisterMmioDevice)?;
1040+
10331041
Ok(())
10341042
}
10351043

@@ -1143,6 +1151,9 @@ pub mod tests {
11431151
#[cfg(target_arch = "aarch64")]
11441152
let resource_allocator = ResourceAllocator::new().unwrap();
11451153

1154+
#[cfg(target_arch = "x86_64")]
1155+
let mut mmio_device_manager = MMIODeviceManager::new();
1156+
#[cfg(target_arch = "aarch64")]
11461157
let mmio_device_manager = MMIODeviceManager::new();
11471158

11481159
#[cfg(target_arch = "x86_64")]
@@ -1166,16 +1177,21 @@ pub mod tests {
11661177
)
11671178
.unwrap();
11681179

1169-
#[cfg(target_arch = "x86_64")]
1170-
setup_interrupt_controller(&mut vm).unwrap();
1171-
11721180
#[cfg(target_arch = "x86_64")]
11731181
{
1182+
setup_interrupt_controller(&mut vm).unwrap();
11741183
let cpu_container = Arc::new(Mutex::new(
11751184
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
11761185
));
11771186
acpi_device_manager
1178-
.attach_cpu_container(cpu_container, vm.fd())
1187+
.attach_cpu_container(cpu_container.clone(), vm.fd())
1188+
.unwrap();
1189+
mmio_device_manager
1190+
.register_mmio_cpu_container_for_boot(
1191+
vm.fd(),
1192+
&mut resource_allocator,
1193+
cpu_container,
1194+
)
11791195
.unwrap();
11801196
}
11811197

@@ -1559,8 +1575,8 @@ pub mod tests {
15591575
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
15601576
assert!(cmdline_contains(
15611577
&cmdline,
1562-
"virtio_mmio.device=4K@0xd0001000:6 virtio_mmio.device=4K@0xd0002000:7 \
1563-
virtio_mmio.device=4K@0xd0003000:8"
1578+
"virtio_mmio.device=4K@0xd0001000:7 virtio_mmio.device=4K@0xd0002000:8 \
1579+
virtio_mmio.device=4K@0xd0003000:9"
15641580
));
15651581
}
15661582

@@ -1655,7 +1671,7 @@ pub mod tests {
16551671
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
16561672
assert!(cmdline_contains(
16571673
&cmdline,
1658-
"virtio_mmio.device=4K@0xd0001000:6"
1674+
"virtio_mmio.device=4K@0xd0001000:7"
16591675
));
16601676
}
16611677

@@ -1672,7 +1688,7 @@ pub mod tests {
16721688
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
16731689
assert!(cmdline_contains(
16741690
&cmdline,
1675-
"virtio_mmio.device=4K@0xd0001000:6"
1691+
"virtio_mmio.device=4K@0xd0001000:7"
16761692
));
16771693
}
16781694

@@ -1691,7 +1707,7 @@ pub mod tests {
16911707
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
16921708
assert!(cmdline_contains(
16931709
&cmdline,
1694-
"virtio_mmio.device=4K@0xd0001000:6"
1710+
"virtio_mmio.device=4K@0xd0001000:7"
16951711
));
16961712
}
16971713
}

‎src/vmm/src/device_manager/mmio.rs

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ use super::resources::ResourceAllocator;
2424
use crate::arch::aarch64::DeviceInfoForFDT;
2525
use crate::arch::DeviceType;
2626
use crate::arch::DeviceType::Virtio;
27+
#[cfg(target_arch = "x86_64")]
28+
use crate::devices::acpi::cpu_container::CpuContainer;
2729
#[cfg(target_arch = "aarch64")]
2830
use crate::devices::legacy::{RTCDevice, SerialDevice};
2931
use crate::devices::pseudo::BootTimer;
@@ -343,6 +345,52 @@ impl MMIODeviceManager {
343345
)
344346
}
345347

348+
#[cfg(target_arch = "x86_64")]
349+
pub fn register_mmio_cpu_container(
350+
&mut self,
351+
vm: &VmFd,
352+
device: Arc<Mutex<CpuContainer>>,
353+
device_info: &MMIODeviceInfo,
354+
) -> Result<(), MmioError> {
355+
let identifier = (
356+
DeviceType::CpuContainer,
357+
DeviceType::CpuContainer.to_string(),
358+
);
359+
{
360+
let container = device.lock().expect("Poisoned lock");
361+
vm.register_irqfd(&container.mmio_interrupt_evt, device_info.irqs[0])
362+
.map_err(MmioError::RegisterIrqFd)?;
363+
}
364+
365+
self.register_mmio_device(
366+
identifier,
367+
device_info.clone(),
368+
BusDevice::CpuContainer(device),
369+
)?;
370+
Ok(())
371+
}
372+
373+
#[cfg(target_arch = "x86_64")]
374+
pub fn register_mmio_cpu_container_for_boot(
375+
&mut self,
376+
vm: &VmFd,
377+
resource_allocator: &mut ResourceAllocator,
378+
device: Arc<Mutex<CpuContainer>>,
379+
) -> Result<(), MmioError> {
380+
let device_info = {
381+
let locked_container = device.lock().expect("Poisoned lock");
382+
let irqs = resource_allocator.allocate_gsi(1)?;
383+
MMIODeviceInfo {
384+
addr: locked_container.mmio_address.0,
385+
len: MMIO_LEN,
386+
irqs,
387+
}
388+
};
389+
390+
self.register_mmio_cpu_container(vm, device, &device_info)?;
391+
Ok(())
392+
}
393+
346394
/// Gets the information of the devices registered up to some point in time.
347395
pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> {
348396
&self.id_to_dev_info
@@ -831,4 +879,24 @@ mod tests {
831879
.allocate_mmio_resources(&mut resource_allocator, 0)
832880
.unwrap();
833881
}
882+
883+
#[test]
884+
#[cfg(target_arch = "x86_64")]
885+
fn test_register_cpu_container() {
886+
let mut device_manager = MMIODeviceManager::new();
887+
let mut resource_allocator = ResourceAllocator::new().unwrap();
888+
let mut vm = Vm::new(vec![]).unwrap();
889+
builder::setup_interrupt_controller(&mut vm).unwrap();
890+
let cpu_container = Arc::new(Mutex::new(
891+
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
892+
));
893+
894+
device_manager
895+
.register_mmio_cpu_container_for_boot(vm.fd(), &mut resource_allocator, cpu_container)
896+
.unwrap();
897+
898+
device_manager
899+
.get_device(DeviceType::CpuContainer, "CpuContainer")
900+
.unwrap();
901+
}
834902
}

‎src/vmm/src/devices/acpi/cpu_container.rs

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ pub enum CpuContainerError {
4141
GuestMemory(#[from] GuestMemoryError),
4242
/// Failed to allocate requested resource: {0}
4343
Allocator(#[from] vm_allocator::Error),
44+
/// Failed to register file descriptor: {0}
45+
RegisterIrqFd(#[from] kvm_ioctls::Error),
4446
}
4547

4648
pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;
@@ -78,7 +80,7 @@ impl CpuContainer {
7880
boot_count: u8,
7981
) -> Result<Self, CpuContainerError> {
8082
let gsi = resource_allocator.allocate_gsi(1)?;
81-
let mmio_address = resource_allocator.allocate_system_memory(
83+
let mmio_address = resource_allocator.allocate_mmio_memory(
8284
MMIO_LEN,
8385
MMIO_LEN,
8486
vm_allocator::AllocPolicy::FirstMatch,
@@ -103,6 +105,49 @@ impl CpuContainer {
103105
debug!("hotplug: notifying guest about new vcpus available");
104106
Ok(())
105107
}
108+
109+
pub fn bus_read(&mut self, offset: u64, data: &mut [u8]) {
110+
data.fill(0);
111+
match offset {
112+
CPU_SELECTION_OFFSET => {
113+
data[0] = self.selected_cpu;
114+
}
115+
CPU_STATUS_OFFSET => {
116+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
117+
let cpu_device = &self.cpu_devices[self.selected_cpu as usize];
118+
if cpu_device.active {
119+
data[0] |= CPU_ENABLE_BIT;
120+
}
121+
if cpu_device.inserting {
122+
data[0] |= CPU_INSERTING_BIT;
123+
}
124+
} else {
125+
error!("Out of range vCPU id: {}", self.selected_cpu)
126+
}
127+
}
128+
_ => error!("Unexpected CPU container offset"),
129+
}
130+
}
131+
132+
pub fn bus_write(&mut self, offset: u64, data: &[u8]) {
133+
match offset {
134+
CPU_SELECTION_OFFSET => self.selected_cpu = data[0],
135+
CPU_STATUS_OFFSET => {
136+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
137+
let cpu_device = &mut self.cpu_devices[self.selected_cpu as usize];
138+
if data[0] & CPU_INSERTING_BIT != 0 {
139+
cpu_device.inserting = false;
140+
}
141+
if data[0] & CPU_ENABLE_BIT != 0 {
142+
cpu_device.active = true;
143+
}
144+
} else {
145+
error!("Out of range vCPU id: {}", self.selected_cpu)
146+
}
147+
}
148+
_ => error!("Unexpected CPU container offset"),
149+
}
150+
}
106151
}
107152

108153
impl Aml for CpuContainer {

‎src/vmm/src/devices/bus.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ pub struct Bus {
5252

5353
use event_manager::{EventOps, Events, MutEventSubscriber};
5454

55+
#[cfg(target_arch = "x86_64")]
56+
use super::acpi::cpu_container::CpuContainer;
5557
#[cfg(target_arch = "aarch64")]
5658
use super::legacy::RTCDevice;
5759
use super::legacy::{I8042Device, SerialDevice};
@@ -66,6 +68,8 @@ pub enum BusDevice {
6668
BootTimer(Arc<Mutex<BootTimer>>),
6769
MmioTransport(Arc<Mutex<MmioTransport>>),
6870
Serial(Arc<Mutex<SerialDevice<std::io::Stdin>>>),
71+
#[cfg(target_arch = "x86_64")]
72+
CpuContainer(Arc<Mutex<CpuContainer>>),
6973
#[cfg(test)]
7074
Dummy(Arc<Mutex<DummyDevice>>),
7175
#[cfg(test)]
@@ -130,6 +134,8 @@ impl BusDevice {
130134
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
131135
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
132136
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
137+
#[cfg(target_arch = "x86_64")]
138+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
133139
#[cfg(test)]
134140
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
135141
#[cfg(test)]
@@ -145,6 +151,8 @@ impl BusDevice {
145151
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
146152
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
147153
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
154+
#[cfg(target_arch = "x86_64")]
155+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
148156
#[cfg(test)]
149157
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
150158
#[cfg(test)]

‎src/vmm/src/lib.rs

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -620,14 +620,20 @@ impl Vmm {
620620

621621
#[allow(clippy::cast_possible_truncation)]
622622
let start_idx = self.vcpus_handles.len().try_into().unwrap();
623-
for cpu_idx in start_idx..(start_idx + config.add) {
624-
let exit_evt = self
625-
.vcpus_exit_evt
626-
.try_clone()
627-
.map_err(HotplugVcpuError::EventFd)?;
628-
let vcpu =
629-
Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
630-
vcpus.push(vcpu);
623+
if let Some(devices::BusDevice::CpuContainer(cont)) =
624+
self.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
625+
{
626+
let mut locked_container = cont.lock().expect("Poisoned lock");
627+
for cpu_idx in start_idx..(start_idx + config.add) {
628+
let exit_evt = self
629+
.vcpus_exit_evt
630+
.try_clone()
631+
.map_err(HotplugVcpuError::EventFd)?;
632+
let vcpu =
633+
Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
634+
locked_container.cpu_devices[cpu_idx as usize].inserting = true;
635+
vcpus.push(vcpu);
636+
}
631637
}
632638

633639
self.start_vcpus(
@@ -653,6 +659,8 @@ impl Vmm {
653659
huge_pages: None,
654660
};
655661

662+
self.acpi_device_manager.notify_cpu_container()?;
663+
656664
Ok(new_machine_config)
657665
}
658666

‎tests/integration_tests/functional/test_max_devices.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@
66

77
import pytest
88

9-
# IRQs are available from 5 to 23. We always use one IRQ for VMGenID device, so
10-
# the maximum number of devices supported at the same time is 18.
11-
MAX_DEVICES_ATTACHED = 18
9+
# IRQs are available from 5 to 23. We always use one IRQ for VMGenID device and
10+
# two for CPU hotplugging, so the maximum number of devices supported at the same
11+
# time is 16.
12+
MAX_DEVICES_ATTACHED = 16
1213

1314

1415
@pytest.mark.skipif(

0 commit comments

Comments
 (0)