Support multiple hub devices

Author: nbdd0121

README.md

@@ -15,8 +15,8 @@ However this would still give the container access to all the devices handled by
 This program tries to solve that problem by listening to udev events to detect when a device is (un)plugged.
 It then interfaces directly with the container's cgroup to grant it access to that specific device.
 
-To limit the devices the container can access, a _root device_ is specified.
-The container will receive access to any device descending from the root device.
+To limit the devices the container can access, _root devices_ are specified.
+The container will receive access to any device descending from one of the root devices.
 This is particularly useful if the root device is set to a USB hub.
 The hub can be specified directly, or it can be specified as "the parent of device X",
 e.g., we can giving a container access to all devices connected to the same hub as an Arduino board.
@@ -31,7 +31,7 @@ This tool wraps `runc` with the additional hotplug feature, therefore it can be
 many container managers/orchestrators such as Docker, Podman, and Kubernetes. You need to ensure `runc` is available in your `PATH`
 so `container-hotplug` can find it.
 
-It supports two annotations, `org.lowrisc.hotplug.device` and `org.lowrisc.hotplug.symlinks`.
+It supports two annotations, `org.lowrisc.hotplug.devices` and `org.lowrisc.hotplug.symlinks`.
 
 For Docker, you can specify an alternative runtime by [changing /etc/docker/daemon.json](https://docs.docker.com/engine/alternative-runtimes/#youki):
 ```json
@@ -45,12 +45,12 @@ For Docker, you can specify an alternative runtime by [changing /etc/docker/daem
 ```
 and use it with the `--runtime hotplug` flag and appropriate annotation, e.g.
 ```bash
-sudo docker run --runtime hotplug -it --annotation org.lowrisc.hotplug.device=parent-of:usb:2b2e:c310 ubuntu:latest
+sudo docker run --runtime hotplug -it --annotation org.lowrisc.hotplug.devices=parent-of:usb:2b2e:c310 ubuntu:latest
 ```
 
 For podman, you can specify the path directly, by:
 ```bash
-sudo podman run --runtime /path/to/container-hotplug/binary -it --annotation org.lowrisc.hotplug.device=parent-of:usb:2b2e:c310 ubuntu:latest
+sudo podman run --runtime /path/to/container-hotplug/binary -it --annotation org.lowrisc.hotplug.devices=parent-of:usb:2b2e:c310 ubuntu:latest
 ```
 
 For containerd (e.g. when using kubernetes), you can edit `/etc/containerd/config.toml` to add:
@@ -78,7 +78,7 @@ kind: Pod
 metadata:
   name: ubuntu
   annotations:
-    org.lowrisc.hotplug.device: parent-of:usb:0bda:5634
+    org.lowrisc.hotplug.devices: parent-of:usb:0bda:5634
 spec:
   runtimeClassName: hotplug
   containers:

src/dev/monitor.rs

@@ -21,8 +21,8 @@ pub enum DeviceEvent {
 }
 
 pub struct DeviceMonitor {
-    /// Root path for devices to monitor. This is usually a USB hub.
-    root: PathBuf,
+    /// Root paths for devices to monitor. This is usually a USB hub.
+    roots: Vec<PathBuf>,
     /// Udev monitor socket.
     // Use `Rc` to avoid lifecycle issues in async stream impl.
     socket: Rc<AsyncFd<udev::MonitorSocket>>,
@@ -38,15 +38,15 @@ impl DeviceMonitor {
     /// Create a new device monitor.
     ///
     /// Devices that are already plugged will each generate an `Add` event immediately.
-    pub fn new(root: PathBuf) -> Result<Self> {
+    pub fn new(roots: Vec<PathBuf>) -> Result<Self> {
         // Create a socket before enumerating devices to avoid missing events.
         let socket = Rc::new(AsyncFd::new(udev::MonitorBuilder::new()?.listen()?)?);
 
         // Process all devices that are already plugged.
         let mut enumerator = Enumerator::new()?;
         let enumerated = enumerator
             .scan_devices()?
-            .filter(|device| device.syspath().starts_with(&root))
+            .filter(|device| roots.iter().any(|root| device.syspath().starts_with(root)))
             .map(Device::from_udev)
             .collect::<VecDeque<_>>();
 
@@ -56,7 +56,7 @@ impl DeviceMonitor {
         }
 
         Ok(Self {
-            root,
+            roots,
             socket,
             seen,
             enumerated,
@@ -74,7 +74,12 @@ impl DeviceMonitor {
             };
 
             match event.event_type() {
-                EventType::Add if event.syspath().starts_with(&self.root) => {
+                EventType::Add
+                    if self
+                        .roots
+                        .iter()
+                        .any(|root| event.syspath().starts_with(root)) =>
+                {
                     match self.seen.entry(event.syspath().to_owned()) {
                         Entry::Occupied(occupied) => {
                             log::info!("Device already seen: {}", occupied.key().display());

src/hotplug/mod.rs

@@ -28,10 +28,10 @@ pub struct HotPlug {
 impl HotPlug {
     pub fn new(
         container: Arc<Container>,
-        hub_path: PathBuf,
+        hub_path: Vec<PathBuf>,
         symlinks: Vec<cli::Symlink>,
     ) -> Result<Self> {
-        let monitor = DeviceMonitor::new(hub_path.clone())?;
+        let monitor = DeviceMonitor::new(hub_path)?;
         let devices = Default::default();
 
         let udev_sender = UdevSender::new(crate::util::namespace::NetNamespace::of_pid(

src/main.rs

@@ -57,13 +57,18 @@ async fn create(global: GlobalOptions, create: CreateOptions, notifier: OwnedFd)
     let mut notifier = Some(notifier);
 
     let config = runc::config::Config::from_bundle(&create.bundle)?;
-    let device: DeviceRef = config
+    let mut devices = Vec::new();
+    let device_annotation = config
         .annotations
-        .get("org.lowrisc.hotplug.device")
+        .get("org.lowrisc.hotplug.devices")
+        .or_else(|| config.annotations.get("org.lowrisc.hotplug.device"))
         .context(
-            "Cannot find annotation `org.lowrisc.hotplug.device`. Please use normal runc instead.",
-        )?
-        .parse()?;
+            "Cannot find annotation `org.lowrisc.hotplug.devices`. Please use normal runc instead.",
+        )?;
+    for device in device_annotation.split(',') {
+        let devref: DeviceRef = device.parse()?;
+        devices.push(devref.device()?.syspath().to_owned());
+    }
 
     let mut symlinks = Vec::<Symlink>::new();
     if let Some(symlink_annotation) = config.annotations.get("org.lowrisc.hotplug.symlinks") {
@@ -72,9 +77,6 @@ async fn create(global: GlobalOptions, create: CreateOptions, notifier: OwnedFd)
         }
     }
 
-    // Run this before calling into runc to create the container.
-    let hub_path = device.device()?.syspath().to_owned();
-
     // Switch the logger to syslog. The runc logs are barely forwarded to the user or syslog by
     // container managers and orchestrators, while we do want to preserve the hotplug events.
     util::log::global_replace(Box::new(util::log::SyslogLogger::new()?));
@@ -107,7 +109,7 @@ async fn create(global: GlobalOptions, create: CreateOptions, notifier: OwnedFd)
     rustix::stdio::dup2_stdout(&null)?;
     rustix::stdio::dup2_stderr(null)?;
 
-    let mut hotplug = HotPlug::new(Arc::clone(&container), hub_path.clone(), symlinks)?;
+    let mut hotplug = HotPlug::new(Arc::clone(&container), devices.clone(), symlinks)?;
     let hotplug_stream = hotplug.run();
 
     let container_stream = {
@@ -133,7 +135,7 @@ async fn create(global: GlobalOptions, create: CreateOptions, notifier: OwnedFd)
                 let notifier = notifier.take().context("Initialized event seen twice")?;
                 rustix::io::write(notifier, &[0])?;
             }
-            Event::Detach(dev) if dev.syspath() == hub_path => {
+            Event::Detach(dev) if devices.iter().any(|hub| dev.syspath() == hub) => {
                 info!("Hub device detached. Stopping container.");
                 let _ = container.kill(Signal::KILL).await;
                 container.wait().await?;