Add support for UFFDIO_CONTINUE / MODE_MINOR

With Linux 5.13, uffd got support for minor page fault notifications
(e.g. a page was already in the page cache, but not present in the
userspace page tables). Notifications are enabled by registering with
UFFDIO_REGISTER_MODE_MINOR, and resolved via the UFFDIO_CONTINUE ioctl.
Add support for these in userfaultfd-sys, as well as utility wrappers in
the main crate.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

Cargo.toml

@@ -23,3 +23,4 @@ nix = { version = "0.27", features = ["poll", "mman", "feature"] }
 default = []
 linux4_14 = ["userfaultfd-sys/linux4_14", "nix/process"]
 linux5_7 = ["userfaultfd-sys/linux5_7"]
+linux5_13 = ["userfaultfd-sys/linux5_13"]

src/event.rs

@@ -21,6 +21,10 @@ pub enum FaultKind {
     /// The fault was a write on a write-protected page.
     #[cfg(feature = "linux5_7")]
     WriteProtected,
+    // The fault was a minor page fault, meaning the page was present in the page cache,
+    // but the userspace page table entry was missing.
+    #[cfg(feature = "linux5_13")]
+    Minor,
 }
 
 /// Events from the userfaultfd object that are read by `Uffd::read_event()`.
@@ -82,17 +86,21 @@ impl Event {
         match msg.event {
             raw::UFFD_EVENT_PAGEFAULT => {
                 let pagefault = unsafe { msg.arg.pagefault };
-                cfg_if::cfg_if!(
-                    if #[cfg(feature = "linux5_7")] {
-                        let kind = if pagefault.flags & raw::UFFD_PAGEFAULT_FLAG_WP != 0 {
-                            FaultKind::WriteProtected
-                        } else {
-                            FaultKind::Missing
-                        };
-                    } else {
-                        let kind = FaultKind::Missing;
-                    }
-                );
+
+                #[allow(unused_mut)]
+                let mut kind = FaultKind::Missing;
+
+                // The below two flags are mutually exclusive (it does not make sense
+                // to have a minor fault that is a write-protect fault at the same time.
+                #[cfg(feature = "linux5_7")]
+                if pagefault.flags & raw::UFFD_PAGEFAULT_FLAG_WP != 0 {
+                    kind = FaultKind::WriteProtected;
+                }
+
+                #[cfg(feature = "linux5_13")]
+                if pagefault.flags & raw::UFFD_PAGEFAULT_FLAG_MINOR != 0 {
+                    kind = FaultKind::Minor
+                }
 
                 let rw = if pagefault.flags & raw::UFFD_PAGEFAULT_FLAG_WRITE == 0 {
                     ReadWrite::Read

src/lib.rs

@@ -87,6 +87,9 @@ bitflags! {
         /// Registers the range for write faults.
         #[cfg(feature = "linux5_7")]
         const WRITE_PROTECT = raw::UFFDIO_REGISTER_MODE_WP;
+        // Registers the range for minor faults.
+        #[cfg(feature = "linux5_13")]
+        const MINOR = raw::UFFDIO_REGISTER_MODE_MINOR;
     }
 }
 
@@ -269,6 +272,38 @@ impl Uffd {
         Ok(())
     }
 
+    /// Resolves minor faults for a range.
+    ///
+    /// If `wake` is `true`, wake up the thread waiting for page fault resolution on the memory
+    /// address range.
+    ///
+    /// Returns the number of bytes actually mapped. If this differs from `len`, then the ioctl
+    /// returned EAGAIN.
+    #[cfg(feature = "linux5_13")]
+    pub fn r#continue(&self, start: *mut c_void, len: usize, wake: bool) -> Result<u64> {
+        let mut ioctl = raw::uffdio_continue {
+            range: raw::uffdio_range {
+                start: start as u64,
+                len: len as u64,
+            },
+            mode: if wake {
+                0
+            } else {
+                raw::UFFDIO_CONTINUE_MODE_DONTWAKE
+            },
+            mapped: 0,
+        };
+
+        let r =
+            unsafe { raw::r#continue(self.as_raw_fd(), &mut ioctl as *mut raw::uffdio_continue) };
+
+        match r {
+            Err(Errno::EAGAIN) if ioctl.mapped > 0 => Ok(ioctl.mapped as u64),
+            Err(err) => Err(err.into()),
+            Ok(_) => Ok(ioctl.mapped as u64),
+        }
+    }
+
     /// Read an `Event` from the userfaultfd object.
     ///
     /// If the `Uffd` object was created with `non_blocking` set to `false`, this will block until

src/raw.rs

@@ -23,6 +23,8 @@ nix::ioctl_readwrite!(
     _UFFDIO_WRITEPROTECT,
     uffdio_writeprotect
 );
+#[cfg(feature = "linux5_13")]
+nix::ioctl_readwrite!(r#continue, UFFDIO, _UFFDIO_CONTINUE, uffdio_continue);
 
 // ioctls for /dev/userfaultfd
 

userfaultfd-sys/Cargo.toml

@@ -20,3 +20,4 @@ cc = "1.0"
 default = []
 linux4_14 = []
 linux5_7 = ["linux4_14"]
+linux5_13 = ["linux5_7"]

userfaultfd-sys/src/consts.c

@@ -30,6 +30,10 @@ const __u64 _const_UFFDIO_REGISTER_MODE_MISSING = UFFDIO_REGISTER_MODE_MISSING;
 const __u64 _const_UFFDIO_REGISTER_MODE_WP = UFFDIO_REGISTER_MODE_WP;
 #endif
 
+#ifdef UFFDIO_REGISTER_MODE_MINOR
+const __u64 _const_UFFDIO_REGISTER_MODE_MINOR = UFFDIO_REGISTER_MODE_MINOR;
+#endif
+
 #ifdef UFFDIO_COPY_MODE_DONTWAKE
 const __u64 _const_UFFDIO_COPY_MODE_DONTWAKE = UFFDIO_COPY_MODE_DONTWAKE;
 #endif
@@ -46,6 +50,10 @@ const __u64 _const_UFFDIO_ZEROPAGE_MODE_DONTWAKE = UFFDIO_ZEROPAGE_MODE_DONTWAKE
 const __u64 _const_UFFDIO_WRITEPROTECT_MODE_WP = UFFDIO_WRITEPROTECT_MODE_WP;
 #endif
 
+#ifdef UFFDIO_CONTINUE_MODE_DONTWAKE
+const __u64 _const_UFFDIO_CONTINUE_MODE_DONTWAKE = UFFDIO_CONTINUE_MODE_DONTWAKE;
+#endif
+
 #ifdef UFFDIO_WRITEPROTECT_MODE_DONTWAKE
 const __u64 _const_UFFDIO_WRITEPROTECT_MODE_DONTWAKE = UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
 #endif
@@ -78,6 +86,10 @@ const __u32 _const_UFFDIO_ZEROPAGE = UFFDIO_ZEROPAGE;
 const __u32 _const_UFFDIO_WRITEPROTECT = UFFDIO_WRITEPROTECT;
 #endif
 
+#ifdef UFFDIO_CONTINUE
+const __u32 _const_UFFDIO_CONTINUE = UFFDIO_CONTINUE;
+#endif
+
 #ifdef USERFAULTFD_IOC
 const __u32 _const_USERFAULTFD_IOC = USERFAULTFD_IOC;
 #endif

userfaultfd-sys/src/lib.rs

@@ -15,8 +15,13 @@ mod linux4_14;
 #[cfg(feature = "linux5_7")]
 mod linux5_7;
 
+#[cfg(feature = "linux5_13")]
+mod linux5_13;
+
 cfg_if! {
-    if #[cfg(feature = "linux5_7")] {
+    if #[cfg(feature = "linux5_13")] {
+        pub use crate::linux5_13::*;
+    } else if #[cfg(feature = "linux5_7")] {
         pub use crate::linux5_7::*;
     } else if #[cfg(feature = "linux4_14")] {
         pub use crate::linux4_14::*;

userfaultfd-sys/src/linux5_13.rs

@@ -0,0 +1,43 @@
+use super::*;
+
+pub use linux5_7::{
+    UFFDIO_API, UFFDIO_COPY, UFFDIO_COPY_MODE_DONTWAKE, UFFDIO_COPY_MODE_WP, UFFDIO_REGISTER,
+    UFFDIO_REGISTER_MODE_MISSING, UFFDIO_REGISTER_MODE_WP, UFFDIO_UNREGISTER, UFFDIO_WAKE,
+    UFFDIO_WRITEPROTECT, UFFDIO_WRITEPROTECT_MODE_DONTWAKE, UFFDIO_WRITEPROTECT_MODE_WP,
+    UFFDIO_ZEROPAGE, UFFDIO_ZEROPAGE_MODE_DONTWAKE, UFFD_API, UFFD_API_FEATURES, UFFD_API_IOCTLS,
+    UFFD_API_RANGE_IOCTLS_BASIC,
+};
+
+pub const UFFD_API_RANGE_IOCTLS: u64 = linux5_7::UFFD_API_RANGE_IOCTLS | 1 << _UFFDIO_CONTINUE;
+
+pub const UFFDIO_REGISTER_MODE_MINOR: u64 = 1 << 2;
+
+pub const UFFDIO_CONTINUE_MODE_DONTWAKE: u64 = 1 << 0;
+
+pub const UFFDIO_CONTINUE: u32 = 0xc020aa07;
+
+#[cfg(test)]
+mod const_tests {
+    use super::*;
+
+    extern "C" {
+        static _const_UFFDIO_REGISTER_MODE_MINOR: u64;
+        static _const_UFFDIO_CONTINUE_MODE_DONTWAKE: u64;
+        static _const_UFFDIO_CONTINUE: u32;
+    }
+
+    #[test]
+    fn consts_correct() {
+        unsafe {
+            assert_eq!(
+                UFFDIO_REGISTER_MODE_MINOR, _const_UFFDIO_REGISTER_MODE_MINOR,
+                "UFFDIO_REGISTER_MODE_MINOR"
+            );
+            assert_eq!(
+                UFFDIO_CONTINUE_MODE_DONTWAKE, _const_UFFDIO_CONTINUE_MODE_DONTWAKE,
+                "UFFDIO_CONTINUE_MODE_DONTWAKE"
+            );
+            assert_eq!(UFFDIO_CONTINUE, _const_UFFDIO_CONTINUE, "UFFDIO_CONTINUE");
+        }
+    }
+}