Fix potential issue. (#11201)

Author: comfyanonymous

comfy/model_patcher.py

@@ -923,7 +923,7 @@ def partially_unload(self, device_to, memory_to_free=0, force_patch_weights=Fals
                                     patch_counter += 1
                             cast_weight = True
 
-                        if cast_weight:
+                        if cast_weight and hasattr(m, "comfy_cast_weights"):
                             m.prev_comfy_cast_weights = m.comfy_cast_weights
                             m.comfy_cast_weights = True
                         m.comfy_patched_weights = False