fix

Author: sayakpaul

src/diffusers/loaders/lora_pipeline.py

@@ -1981,7 +1981,8 @@ def _load_norm_into_transformer(
         logger.info(
             "The provided state dict contains normalization layers in addition to LoRA layers. The normalization layers will directly update the state_dict of the transformer "
             'as opposed to the LoRA layers that will co-exist separately until the "fuse_lora()" method is called. That is to say, the normalization layers will always be directly '
-            "fused into the transformer and can only be unfused if `discard_original_layers=True` is passed."
+            "fused into the transformer and can only be unfused if `discard_original_layers=True` is passed. This might also have implications when dealing with multiple LoRAs. "
+            "If you notice something unexpected, please open an issue: https://github.com/huggingface/diffusers/issues."
         )
 
         # We can't load with strict=True because the current state_dict does not contain all the transformer keys
@@ -2286,6 +2287,10 @@ def _maybe_expand_transformer_param_shape_or_error_(
         norm_state_dict=None,
         prefix=None,
     ) -> bool:
+        """
+        Control LoRA expands the shape of the input layer from (3072, 64) to (3072, 128). This method handles that and
+        generalizes things a bit so that any parameter that needs expansion receives appropriate treatement.
+        """
         state_dict = {}
         if lora_state_dict is not None:
             state_dict.update(lora_state_dict)

src/diffusers/loaders/peft.py

@@ -57,6 +57,12 @@
 
 
 def _maybe_adjust_config(config):
+    """
+    We may run into some ambiguous configuration values when a model has module names, sharing a common prefix
+    (`proj_out.weight` and `blocks.transformer.proj_out.weight`, for example) and they have different LoRA ranks. This
+    method removes the ambiguity by following what is described here:
+    https://github.com/huggingface/diffusers/pull/9985#issuecomment-2493840028.
+    """
     rank_pattern = config["rank_pattern"].copy()
     target_modules = config["target_modules"]
     original_r = config["r"]
@@ -65,6 +71,10 @@ def _maybe_adjust_config(config):
         key_rank = rank_pattern[key]
 
         # try to detect ambiguity
+        # `target_modules` can also be a str, in which case this loop would loop
+        # over the chars of the str. The technically correct way to match LoRA keys
+        # in PEFT is to use LoraModel._check_target_module_exists (lora_config, key).
+        # But this cuts it for now.
         exact_matches = [mod for mod in target_modules if mod == key]
         substring_matches = [mod for mod in target_modules if key in mod and mod != key]
         ambiguous_key = key

src/diffusers/utils/peft_utils.py

@@ -181,7 +181,7 @@ def get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True
     target_modules = list({name.split(".lora")[0] for name in peft_state_dict.keys()})
     use_dora = any("lora_magnitude_vector" in k for k in peft_state_dict)
     # for now we know that the "bias" keys are only associated with `lora_B`.
-    lora_bias = any("lora_B" and "bias" in k for k in peft_state_dict)
+    lora_bias = any("lora_B" in k and k.endswith(".bias") for k in peft_state_dict)
 
     lora_config_kwargs = {
         "r": r,