VirtualMachineManager: Rework protocol (#341)

dcantah · web-flow · commit 056a90908118 · 2025-10-21T01:23:00.000-07:00
Today this protocols create method is just odd. Our only implementation
of it immediately casts to LinuxContainer, failing if it cannot do so. I
think what might make more sense is to pass in a configuration itself
with core parameters that we expect every vmm to be able to support, and
then in a specific implementation they can continue to cast this type to
a specific one to possibly extract some extra configuration values
(rosetta for VZ for example).

This rework will also make it simpler to support a Pod type, as the vm
setup is identical and simple.
diff --git a/Sources/Containerization/ContainerManager.swift b/Sources/Containerization/ContainerManager.swift
@@ -197,30 +197,38 @@ public struct ContainerManager: Sendable {
     }
 
     /// Create a new manager with the provided kernel, initfs mount, image store
-    /// and optional network implementation.
+    /// and optional network implementation. This will use a Virtualization.framework
+    /// backed VMM implicitly.
     public init(
         kernel: Kernel,
         initfs: Mount,
         imageStore: ImageStore,
-        network: Network? = nil
+        network: Network? = nil,
+        rosetta: Bool = false,
+        nestedVirtualization: Bool = false
     ) throws {
         self.imageStore = imageStore
         self.network = network
         try Self.createRootDirectory(path: self.imageStore.path)
         self.vmm = VZVirtualMachineManager(
             kernel: kernel,
             initialFilesystem: initfs,
-            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath()
+            bootlog: imageStore.path.appendingPathComponent("bootlog.log").absolutePath(),
+            rosetta: rosetta,
+            nestedVirtualization: nestedVirtualization
         )
     }
 
     /// Create a new manager with the provided kernel, initfs mount, root state
-    /// directory and optional network implementation.
+    /// directory and optional network implementation. This will use a Virtualization.framework
+    /// backed VMM implicitly.
     public init(
         kernel: Kernel,
         initfs: Mount,
         root: URL? = nil,
-        network: Network? = nil
+        network: Network? = nil,
+        rosetta: Bool = false,
+        nestedVirtualization: Bool = false
     ) throws {
         if let root {
             self.imageStore = try ImageStore(path: root)
@@ -232,17 +240,22 @@ public struct ContainerManager: Sendable {
         self.vmm = VZVirtualMachineManager(
             kernel: kernel,
             initialFilesystem: initfs,
-            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath()
+            bootlog: imageStore.path.appendingPathComponent("bootlog.log").absolutePath(),
+            rosetta: rosetta,
+            nestedVirtualization: nestedVirtualization
         )
     }
 
     /// Create a new manager with the provided kernel, initfs reference, image store
-    /// and optional network implementation.
+    /// and optional network implementation. This will use a Virtualization.framework
+    /// backed VMM implicitly.
     public init(
         kernel: Kernel,
         initfsReference: String,
         imageStore: ImageStore,
-        network: Network? = nil
+        network: Network? = nil,
+        rosetta: Bool = false,
+        nestedVirtualization: Bool = false
     ) async throws {
         self.imageStore = imageStore
         self.network = network
@@ -269,16 +282,21 @@ public struct ContainerManager: Sendable {
         self.vmm = VZVirtualMachineManager(
             kernel: kernel,
             initialFilesystem: initfs,
-            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath()
+            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath(),
+            rosetta: rosetta,
+            nestedVirtualization: nestedVirtualization
         )
     }
 
     /// Create a new manager with the provided kernel and image reference for the initfs.
+    /// This will use a Virtualization.framework backed VMM implicitly.
     public init(
         kernel: Kernel,
         initfsReference: String,
         root: URL? = nil,
-        network: Network? = nil
+        network: Network? = nil,
+        rosetta: Bool = false,
+        nestedVirtualization: Bool = false
     ) async throws {
         if let root {
             self.imageStore = try ImageStore(path: root)
@@ -309,7 +327,9 @@ public struct ContainerManager: Sendable {
         self.vmm = VZVirtualMachineManager(
             kernel: kernel,
             initialFilesystem: initfs,
-            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath()
+            bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath(),
+            rosetta: rosetta,
+            nestedVirtualization: nestedVirtualization
         )
     }
 
diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift
@@ -52,16 +52,14 @@ public final class LinuxContainer: Container, Sendable {
         public var interfaces: [any Interface] = []
         /// The Unix domain socket relays to setup for the container.
         public var sockets: [UnixSocketConfiguration] = []
-        /// Whether rosetta x86-64 emulation should be setup for the container.
-        public var rosetta: Bool = false
-        /// Whether nested virtualization should be turned on for the container.
-        public var virtualization: Bool = false
         /// The mounts for the container.
         public var mounts: [Mount] = LinuxContainer.defaultMounts()
         /// The DNS configuration for the container.
         public var dns: DNS?
         /// The hosts to add to /etc/hosts for the container.
         public var hosts: Hosts?
+        /// Enable nested virtualization support.
+        public var virtualization: Bool = false
 
         public init() {}
     }
@@ -303,15 +301,27 @@ extension LinuxContainer {
         try await self.state.withLock { state in
             try state.validateForCreate()
 
-            let vm = try await self.vmm.create(container: self)
+            let vmConfig = VMConfiguration(
+                cpus: self.cpus,
+                memoryInBytes: self.memoryInBytes,
+                interfaces: self.interfaces,
+                mountsByID: [self.id: [self.rootfs] + self.config.mounts],
+                nestedVirtualization: self.config.virtualization
+            )
+            let creationConfig = StandardVMConfig(configuration: vmConfig)
+            let vm = try await self.vmm.create(config: creationConfig)
+
             try await vm.start()
             do {
                 let relayManager = UnixSocketRelayManager(vm: vm)
                 try await vm.withAgent { agent in
                     try await agent.standardSetup()
 
                     // Mount the rootfs.
-                    var rootfs = vm.mounts[0].to
+                    guard let attachments = vm.mounts[self.id], let rootfsAttachment = attachments.first else {
+                        throw ContainerizationError(.notFound, message: "rootfs mount not found")
+                    }
+                    var rootfs = rootfsAttachment.to
                     rootfs.destination = Self.guestRootfsPath(self.id)
                     try await agent.mount(rootfs)
 
@@ -364,7 +374,8 @@ extension LinuxContainer {
             do {
                 var spec = self.generateRuntimeSpec()
                 // We don't need the rootfs, nor do OCI runtimes want it included.
-                spec.mounts = createdState.vm.mounts.dropFirst().map { $0.to }
+                let containerMounts = createdState.vm.mounts[self.id] ?? []
+                spec.mounts = containerMounts.dropFirst().map { $0.to }
 
                 let stdio = Self.setupIO(
                     portAllocator: self.hostVsockPorts,
diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2025 Apple Inc. and the Containerization project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import ContainerizationOCI
+import Foundation
+
+/// Protocol for VM creation configuration. Allows VMMs to extend with specific settings
+/// while maintaining a common core configuration.
+public protocol VMCreationConfig: Sendable {
+    /// The common VM configuration that all VMMs must support.
+    var configuration: VMConfiguration { get }
+}
+
+/// Standard VM creation configuration with only common settings.
+public struct StandardVMConfig: VMCreationConfig {
+    public var configuration: VMConfiguration
+
+    public init(configuration: VMConfiguration) {
+        self.configuration = configuration
+    }
+}
+
+/// Configuration for creating a virtual machine instance.
+public struct VMConfiguration: Sendable {
+    /// The amount of CPUs to allocate.
+    public var cpus: Int
+    /// The memory in bytes to allocate.
+    public var memoryInBytes: UInt64
+    /// The network interfaces to attach.
+    public var interfaces: [any Interface]
+    /// Mounts organized by metadata ID (e.g. container ID).
+    /// Each ID maps to an array of mounts for that workload.
+    public var mountsByID: [String: [Mount]]
+    /// Optional file path to store serial boot logs.
+    public var bootlog: URL?
+    /// Enable nested virtualization support. If the VirtualMachineManager
+    /// does not support this feature, it MUST return an .unsupported ContainerizationError.
+    public var nestedVirtualization: Bool
+
+    public init(
+        cpus: Int = 4,
+        memoryInBytes: UInt64 = 1024 * 1024 * 1024,
+        interfaces: [any Interface] = [],
+        mountsByID: [String: [Mount]] = [:],
+        bootlog: URL? = nil,
+        nestedVirtualization: Bool = false
+    ) {
+        self.cpus = cpus
+        self.memoryInBytes = memoryInBytes
+        self.interfaces = interfaces
+        self.mountsByID = mountsByID
+        self.bootlog = bootlog
+        self.nestedVirtualization = nestedVirtualization
+    }
+}
diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift
@@ -28,8 +28,8 @@ import Virtualization
 struct VZVirtualMachineInstance: VirtualMachineInstance, Sendable {
     typealias Agent = Vminitd
 
-    /// Attached mounts on the sandbox.
-    public let mounts: [AttachedFilesystem]
+    /// Attached mounts on the sandbox, organized by metadata ID.
+    public let mounts: [String: [AttachedFilesystem]]
 
     /// Returns the runtime state of the vm.
     public var state: VirtualMachineInstanceState {
@@ -47,8 +47,8 @@ struct VZVirtualMachineInstance: VirtualMachineInstance, Sendable {
         public var rosetta: Bool
         /// Toggle nested virtualization support.
         public var nestedVirtualization: Bool
-        /// Mount attachments.
-        public var mounts: [Mount]
+        /// Mount attachments organized by metadata ID.
+        public var mountsByID: [String: [Mount]]
         /// Network interface attachments.
         public var interfaces: [any Interface]
         /// Kernel image.
@@ -63,7 +63,7 @@ struct VZVirtualMachineInstance: VirtualMachineInstance, Sendable {
             self.memoryInBytes = 1024.mib()
             self.rosetta = false
             self.nestedVirtualization = false
-            self.mounts = []
+            self.mountsByID = [:]
             self.interfaces = []
         }
     }
@@ -317,8 +317,10 @@ extension VZVirtualMachineInstance.Configuration {
         config.bootLoader = loader
 
         try initialFilesystem.configure(config: &config)
-        for mount in self.mounts {
-            try mount.configure(config: &config)
+        for (_, mounts) in self.mountsByID {
+            for mount in mounts {
+                try mount.configure(config: &config)
+            }
         }
 
         let platform = VZGenericPlatformConfiguration()
@@ -337,7 +339,7 @@ extension VZVirtualMachineInstance.Configuration {
         return config
     }
 
-    func mountAttachments() throws -> [AttachedFilesystem] {
+    func mountAttachments() throws -> [String: [AttachedFilesystem]] {
         let allocator = Character.blockDeviceTagAllocator()
         if let initialFilesystem {
             // When the initial filesystem is a blk, allocate the first letter "vd(a)"
@@ -347,11 +349,15 @@ extension VZVirtualMachineInstance.Configuration {
             }
         }
 
-        var attachments: [AttachedFilesystem] = []
-        for mount in self.mounts {
-            attachments.append(try .init(mount: mount, allocator: allocator))
+        var attachmentsByID: [String: [AttachedFilesystem]] = [:]
+        for (id, mounts) in self.mountsByID {
+            var attachments: [AttachedFilesystem] = []
+            for mount in mounts {
+                attachments.append(try .init(mount: mount, allocator: allocator))
+            }
+            attachmentsByID[id] = attachments
         }
-        return attachments
+        return attachmentsByID
     }
 }
 
diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift
@@ -23,47 +23,53 @@ import Logging
 /// A virtualization.framework backed `VirtualMachineManager` implementation.
 public struct VZVirtualMachineManager: VirtualMachineManager {
     private let kernel: Kernel
-    private let bootlog: String?
     private let initialFilesystem: Mount
+    private let bootlog: String?
+    private let rosetta: Bool
+    private let nestedVirtualization: Bool
     private let logger: Logger?
 
     public init(
         kernel: Kernel,
         initialFilesystem: Mount,
         bootlog: String? = nil,
+        rosetta: Bool = false,
+        nestedVirtualization: Bool = false,
         logger: Logger? = nil
     ) {
         self.kernel = kernel
-        self.bootlog = bootlog
         self.initialFilesystem = initialFilesystem
+        self.bootlog = bootlog
+        self.rosetta = rosetta
+        self.nestedVirtualization = nestedVirtualization
         self.logger = logger
     }
 
-    public func create(container: Container) throws -> any VirtualMachineInstance {
-        guard let c = container as? LinuxContainer else {
-            throw ContainerizationError(
-                .invalidArgument,
-                message: "provided container is not a LinuxContainer"
-            )
-        }
+    public func create(config: some VMCreationConfig) throws -> any VirtualMachineInstance {
+        let vmConfig = config.configuration
+
+        // Use nested virtualization if requested in config or set as default in manager
+        let useNestedVirtualization = vmConfig.nestedVirtualization || self.nestedVirtualization
 
         return try VZVirtualMachineInstance(
             logger: self.logger,
-            with: { config in
-                config.cpus = container.cpus
-                config.memoryInBytes = container.memoryInBytes
+            with: { instanceConfig in
+                instanceConfig.cpus = vmConfig.cpus
+                instanceConfig.memoryInBytes = vmConfig.memoryInBytes
 
-                config.kernel = self.kernel
-                config.initialFilesystem = self.initialFilesystem
+                instanceConfig.kernel = self.kernel
+                instanceConfig.initialFilesystem = self.initialFilesystem
 
-                config.interfaces = container.interfaces
-                if let bootlog {
-                    config.bootlog = URL(filePath: bootlog)
+                instanceConfig.interfaces = vmConfig.interfaces
+                if let bootlog = vmConfig.bootlog {
+                    instanceConfig.bootlog = bootlog
+                } else if let bootlog = self.bootlog {
+                    instanceConfig.bootlog = URL(filePath: bootlog)
                 }
-                config.rosetta = c.config.rosetta
-                config.nestedVirtualization = c.config.virtualization
+                instanceConfig.rosetta = self.rosetta
+                instanceConfig.nestedVirtualization = useNestedVirtualization
 
-                config.mounts = [c.rootfs] + c.config.mounts
+                instanceConfig.mountsByID = vmConfig.mountsByID
             })
     }
 }
diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift
@@ -33,7 +33,7 @@ public protocol VirtualMachineInstance: Sendable {
     // The state of the virtual machine.
     var state: VirtualMachineInstanceState { get }
 
-    var mounts: [AttachedFilesystem] { get }
+    var mounts: [String: [AttachedFilesystem]] { get }
     /// Dial the Agent. It's up the VirtualMachineInstance to determine
     /// what port the agent is listening on.
     func dialAgent() async throws -> Agent
diff --git a/Sources/Containerization/VirtualMachineManager.swift b/Sources/Containerization/VirtualMachineManager.swift
@@ -16,5 +16,5 @@
 
 /// A protocol to implement for virtual machine isolated containers.
 public protocol VirtualMachineManager: Sendable {
-    func create(container: Container) async throws -> any VirtualMachineInstance
+    func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance
 }
diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift

Original file line number	Diff line number	Diff line change
`@@ -16,5 +16,5 @@`
`16`	`16`
`17`	`17`	`/// A protocol to implement for virtual machine isolated containers.`
`18`	`18`	`public protocol VirtualMachineManager: Sendable {`
`19`		`- func create(container: Container) async throws -> any VirtualMachineInstance`
	`19`	`+ func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance`
`20`	`20`	`}`