avoid excessive Arc<Mapping> contention

Conceptually it wasn't wrong for submappings to `Arc::clone` the
pointed-to base mapping, but in practice all the accessors off of
MemCtx return SubMapping with lifetimes constrained by that MemCtx.

The sheer number of submappings being created and destroyed during I/O
operations makes the refcount very hot under load. Lean into the de
facto lifetime constraint on SubMapping and just take a borrow of the
backing Mapping instead of Arc::clone (and implicit ref decrement in
later drop).

Author: iximeow

lib/propolis/src/vmm/mem.rs

@@ -378,7 +378,7 @@ unsafe impl Sync for Mapping {}
 // not reference them directly as a field.
 #[allow(dead_code)]
 enum Backing<'a> {
-    Base(Arc<Mapping>),
+    Base(&'a Mapping),
     Sub(&'a SubMapping<'a>),
 }
 
@@ -401,12 +401,9 @@ pub struct SubMapping<'a> {
 impl SubMapping<'_> {
     /// Create `SubMapping` using the entire region offered by an underlying
     /// `Mapping` object.
-    fn new_base<'a>(
-        _mem: &'a MemCtx,
-        base: &'_ Arc<Mapping>,
-    ) -> SubMapping<'a> {
+    fn new_base<'a>(base: &'a Mapping) -> SubMapping<'a> {
         SubMapping {
-            backing: Backing::Base(base.clone()),
+            backing: Backing::Base(base),
 
             ptr: base.ptr,
             len: base.len,
@@ -1121,7 +1118,7 @@ impl MemCtx {
                     format!("memory region {} not found", name),
                 )
             })?;
-        Ok(SubMapping::new_base(self, ent).constrain_access(Prot::WRITE))
+        Ok(SubMapping::new_base(ent).constrain_access(Prot::WRITE))
     }
 
     /// Like `writable_region`, but accesses the underlying memory segment
@@ -1167,12 +1164,12 @@ impl MemCtx {
                 MapKind::MmioReserve => None,
             }?;
 
-            let guest_map = SubMapping::new_base(self, &seg.map_guest)
+            let guest_map = SubMapping::new_base(&seg.map_guest)
                 .constrain_access(prot)
                 .constrain_region(req_offset, len)
                 .expect("mapping offset should be valid");
 
-            let seg_map = SubMapping::new_base(self, &seg.map_seg)
+            let seg_map = SubMapping::new_base(&seg.map_seg)
                 .constrain_region(req_offset, len)
                 .expect("mapping offset should be valid");