nodes: mmap detach weights before they are pinned (#355)

Comfy core recently introduced a feature where weights may be pinned
when loading, particularly for the case of offloading.

Intercept this, and immediately detached each weight before the pinning.
This avoids a crash that at least some users are experiencing.

Use a little dict on the modules to keep track of whats already done,
and when the catch-all detacher loop comes through, use this dict
(which has already done modules removed) as the iterator basis.

Author: rattus128

nodes.py

@@ -76,16 +76,29 @@ def unpatch_model(self, device_to=None, unpatch_weights=True):
         # TODO: Find another way to not unload after patches
         return super().unpatch_model(device_to=device_to, unpatch_weights=unpatch_weights)
 
+
+    def pin_weight_to_device(self, key):
+        op_key = key.rsplit('.', 1)[0]
+        if self.named_modules_to_munmap is not None and op_key in self.named_modules_to_munmap:
+            # TODO: possible to OOM, find better way to detach
+            self.named_modules_to_munmap[op_key].to(self.load_device).to(self.offload_device)
+            del self.named_modules_to_munmap[op_key]
+        super().pin_weight_to_device(key)
+
     mmap_released = False
+
     def load(self, *args, force_patch_weights=False, **kwargs):
+        if not self.mmap_released:
+            self.named_modules_to_munmap = dict(self.model.named_modules())
+
         # always call `patch_weight_to_device` even for lowvram
         super().load(*args, force_patch_weights=True, **kwargs)
 
         # make sure nothing stays linked to mmap after first load
         if not self.mmap_released:
             linked = []
             if kwargs.get("lowvram_model_memory", 0) > 0:
-                for n, m in self.model.named_modules():
+                for n, m in self.named_modules_to_munmap.items():
                     if hasattr(m, "weight"):
                         device = getattr(m.weight, "device", None)
                         if device == self.offload_device:
@@ -102,6 +115,7 @@ def load(self, *args, force_patch_weights=False, **kwargs):
                     # TODO: possible to OOM, find better way to detach
                     m.to(self.load_device).to(self.offload_device)
             self.mmap_released = True
+        self.named_modules_to_munmap = None
 
     def clone(self, *args, **kwargs):
         src_cls = self.__class__