diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py
index e2dd3322fdcb..4be6971755d2 100644
--- a/src/diffusers/loaders/lora_conversion_utils.py
+++ b/src/diffusers/loaders/lora_conversion_utils.py
@@ -654,6 +654,7 @@ def _convert(original_key, diffusers_key, state_dict, new_state_dict):
 
                 _convert(k, diffusers_key, state_dict, new_state_dict)
 
+        remaining_all_unet = False
         if state_dict:
             remaining_all_unet = all(k.startswith("lora_unet_") for k in state_dict)
         if remaining_all_unet: