fix

Co-authored-by: BenjaminBossan <[email protected]>

Author: sayakpaul

src/diffusers/loaders/lora_base.py

@@ -465,6 +465,7 @@ class LoraBaseMixin:
     """Utility class for handling LoRAs."""
 
     _lora_loadable_modules = []
+    _merged_adapters = set()
 
     def load_lora_weights(self, **kwargs):
         raise NotImplementedError("`load_lora_weights()` is not implemented.")
@@ -504,12 +505,6 @@ def _best_guess_weight_name(cls, *args, **kwargs):
         deprecate("_best_guess_weight_name", "0.35.0", deprecation_message)
         return _best_guess_weight_name(*args, **kwargs)
 
-    @property
-    def _merged_adapters(self):
-        if "_merged_adapters" not in self.__dict__:
-            self.__dict__["_merged_adapters"] = set()
-        return self.__dict__["_merged_adapters"]
-
     def unload_lora_weights(self):
         """
         Unloads the LoRA parameters.
@@ -597,6 +592,9 @@ def fuse_lora(
         if len(components) == 0:
             raise ValueError("`components` cannot be an empty list.")
 
+        # Need to retrieve the names as `adapter_names` can be None. So we cannot directly use it
+        # in `self._merged_adapters = self._merged_adapters | merged_adapter_names`.
+        merged_adapter_names = set()
         for fuse_component in components:
             if fuse_component not in self._lora_loadable_modules:
                 raise ValueError(f"{fuse_component} is not found in {self._lora_loadable_modules=}.")
@@ -608,15 +606,17 @@ def fuse_lora(
                     model.fuse_lora(lora_scale, safe_fusing=safe_fusing, adapter_names=adapter_names)
                     for module in model.modules():
                         if isinstance(module, BaseTunerLayer):
-                            self._merged_adapters.update(set(module.merged_adapters))
+                            merged_adapter_names.update(set(module.merged_adapters))
                 # handle transformers models.
                 if issubclass(model.__class__, PreTrainedModel):
                     fuse_text_encoder_lora(
                         model, lora_scale=lora_scale, safe_fusing=safe_fusing, adapter_names=adapter_names
                     )
                     for module in model.modules():
                         if isinstance(module, BaseTunerLayer):
-                            self._merged_adapters.update(set(module.merged_adapters))
+                            merged_adapter_names.update(set(module.merged_adapters))
+
+        self._merged_adapters = self._merged_adapters | merged_adapter_names
 
     def unfuse_lora(self, components: List[str] = [], **kwargs):
         r"""