From 068adea13e2aadb3dff25cbedccbf57246892c19 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 19:37:37 +0300 Subject: [PATCH 01/16] testing --- src/diffusers/loaders/lora_pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index b85f51db83bc..036d67e90246 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4814,6 +4814,8 @@ def _maybe_expand_t2v_lora_for_i2v( return state_dict if any(k.startswith("transformer.blocks.") for k in state_dict): + test = [k.split("blocks.")[1] for k in state_dict] + print("wtf", test[:10]) num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) From d73bdd1ed8607eb74f1cb22ba84f9a72f1d24813 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 19:45:29 +0300 Subject: [PATCH 02/16] testing --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 036d67e90246..7c993e1ae012 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4814,7 +4814,7 @@ def _maybe_expand_t2v_lora_for_i2v( return state_dict if any(k.startswith("transformer.blocks.") for k in state_dict): - test = [k.split("blocks.")[1] for k in state_dict] + test = [k.split("blocks.") for k in state_dict] print("wtf", test[:10]) num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) From 23e3c1c999505cdffae5397979f3971167d5439b Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 19:53:24 +0300 Subject: [PATCH 03/16] testing --- src/diffusers/loaders/lora_pipeline.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 7c993e1ae012..acac04904d2f 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4814,9 +4814,7 @@ def _maybe_expand_t2v_lora_for_i2v( return state_dict if any(k.startswith("transformer.blocks.") for k in state_dict): - test = [k.split("blocks.") for k in state_dict] - print("wtf", test[:10]) - num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict}) + num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in original_state_dict if "blocks." in k}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) if is_i2v_lora: From 3ce7f9c9385ee6d977a82646b012941d6719643a Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 19:55:14 +0300 Subject: [PATCH 04/16] testing --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index acac04904d2f..8562bf013b20 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4814,7 +4814,7 @@ def _maybe_expand_t2v_lora_for_i2v( return state_dict if any(k.startswith("transformer.blocks.") for k in state_dict): - num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in original_state_dict if "blocks." in k}) + num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) if is_i2v_lora: From c909d841b99ba9e2c2e7a91756ae26a51b1464af Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 20:43:27 +0300 Subject: [PATCH 05/16] testing --- src/diffusers/loaders/lora_pipeline.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 8562bf013b20..96909c3fe179 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4813,6 +4813,9 @@ def _maybe_expand_t2v_lora_for_i2v( if transformer.config.image_dim is None: return state_dict + orig_keys = set(state_dict.keys()) + print("BEFORE:", state_dict.keys()) + if any(k.startswith("transformer.blocks.") for k in state_dict): num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) @@ -4828,6 +4831,7 @@ def _maybe_expand_t2v_lora_for_i2v( state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"] ) + print("AFTER:", set(state_dict.keys()) - orig_keys) return state_dict From 80b6b9449ee130032008c69307550a180da82fd9 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 21:00:19 +0300 Subject: [PATCH 06/16] i2v --- src/diffusers/loaders/lora_pipeline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 96909c3fe179..9e82c0c02276 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4819,6 +4819,7 @@ def _maybe_expand_t2v_lora_for_i2v( if any(k.startswith("transformer.blocks.") for k in state_dict): num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) + is_bias = any("bias" in k for k in state_dict) if is_i2v_lora: return state_dict @@ -4831,6 +4832,10 @@ def _maybe_expand_t2v_lora_for_i2v( state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"] ) + if is_bias: + converted_state_dict[f"blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.bias"] + ) print("AFTER:", set(state_dict.keys()) - orig_keys) return state_dict From e4f3938502d90592cbe84bc2fee85f3b3e7409fd Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Mon, 19 May 2025 21:01:52 +0300 Subject: [PATCH 07/16] i2v --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 9e82c0c02276..42e9a9e32712 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4833,7 +4833,7 @@ def _maybe_expand_t2v_lora_for_i2v( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"] ) if is_bias: - converted_state_dict[f"blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( + state_dict[f"blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.bias"] ) print("AFTER:", set(state_dict.keys()) - orig_keys) From cd94c12a08eac368952530925b5397303beed2a0 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 15:35:38 +0300 Subject: [PATCH 08/16] device fix --- src/diffusers/loaders/lora_pipeline.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 42e9a9e32712..c4e066af120e 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4813,8 +4813,7 @@ def _maybe_expand_t2v_lora_for_i2v( if transformer.config.image_dim is None: return state_dict - orig_keys = set(state_dict.keys()) - print("BEFORE:", state_dict.keys()) + target_device = transformer.device if any(k.startswith("transformer.blocks.") for k in state_dict): num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) @@ -4827,16 +4826,16 @@ def _maybe_expand_t2v_lora_for_i2v( for i in range(num_blocks): for o, c in zip(["k_img", "v_img"], ["add_k_proj", "add_v_proj"]): state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_A.weight"] + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_A.weight"], device=target_device ) state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"] + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"], device=target_device ) if is_bias: state_dict[f"blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.bias"] + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.bias"], device=target_device ) - print("AFTER:", set(state_dict.keys()) - orig_keys) + return state_dict From 85a618d0b41c674904e8f7d93d7bb2f3d9006f6c Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 15:44:05 +0300 Subject: [PATCH 09/16] testing --- src/diffusers/loaders/lora_pipeline.py | 87 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 14 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index c4e066af120e..7a50f1440772 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4813,30 +4813,89 @@ def _maybe_expand_t2v_lora_for_i2v( if transformer.config.image_dim is None: return state_dict - target_device = transformer.device + # Determine the target device from the transformer + try: + target_device = next(transformer.parameters()).device + except StopIteration: + # Fallback if transformer has no parameters (should be rare for a full model) + # Try to infer from existing tensors in state_dict, else default to CPU + target_device = torch.device("cpu") + for v in state_dict.values(): + if torch.is_tensor(v): + target_device = v.device + break + + # Debug prints, can be removed after verification + logger.debug(f"Target device for new LoRA tensors: {target_device}") + logger.debug(f"Keys BEFORE _maybe_expand_t2v_lora_for_i2v: {list(state_dict.keys())[:5]}...") if any(k.startswith("transformer.blocks.") for k in state_dict): - num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) + block_indices = set() + # Iterate over a copy of keys if state_dict might be modified during iteration elsewhere (not here though) + for k in list(state_dict.keys()): + if k.startswith("transformer.blocks.") and ".attn2.to_k.lora_A.weight" in k: + try: + block_idx_str = k.split("blocks.")[1].split(".")[0] + block_indices.add(int(block_idx_str)) + except (IndexError, ValueError): + logger.warning(f"Could not parse block index from key: {k}") + continue + + if not block_indices: + logger.debug( + "No valid blocks found for T2V to I2V expansion referencing based on '.attn2.to_k.lora_A.weight'.") + return state_dict + + num_total_blocks = max(block_indices) + 1 + is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) - is_bias = any("bias" in k for k in state_dict) - if is_i2v_lora: + # Check for bias keys that would have been converted by _convert_non_diffusers_wan_lora_to_diffusers + # e.g., 'transformer.blocks.0.attn2.to_k.lora_B.bias' + # This helps decide if zero biases should be added for the new projections. + has_bias_key_pattern_in_sd = any(".lora_B.bias" in k for k in state_dict) + + if is_i2v_lora: # If it's already an I2V LoRA, no expansion needed return state_dict - for i in range(num_blocks): - for o, c in zip(["k_img", "v_img"], ["add_k_proj", "add_v_proj"]): - state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_A.weight"], device=target_device + logger.info( + "Adapting a T2V LoRA for I2V model by adding zero-initialized weights for image-specific cross-attention layers." + ) + for i in range(num_total_blocks): + # Define reference key patterns carefully. These keys should exist if the block `i` was part of the T2V LoRA. + ref_key_lora_A = f"transformer.blocks.{i}.attn2.to_k.lora_A.weight" + ref_key_lora_B = f"transformer.blocks.{i}.attn2.to_k.lora_B.weight" + + # Only proceed if the reference LoRA weights for this block exist in the T2V LoRA + if ref_key_lora_A not in state_dict or ref_key_lora_B not in state_dict: + continue + + ref_lora_A_weight = state_dict[ref_key_lora_A] + ref_lora_B_weight = state_dict[ref_key_lora_B] + + # Use dtype from reference LoRA tensors, device from transformer + lora_dtype = ref_lora_A_weight.dtype + + for _unused_orig_name, diffusers_name_part in zip(["k_img", "v_img"], ["add_k_proj", "add_v_proj"]): + # Create new tensors on the transformer's device and with the LoRA's dtype + state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_A.weight"] = torch.zeros( + ref_lora_A_weight.shape, device=target_device, dtype=lora_dtype ) - state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"], device=target_device + state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.weight"] = torch.zeros( + ref_lora_B_weight.shape, device=target_device, dtype=lora_dtype ) - if is_bias: - state_dict[f"blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( - state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.bias"], device=target_device - ) + # If the original LoRA had biases (indicated by has_bias_key_pattern_in_sd) + # AND the specific reference bias key exists for this block. + ref_key_lora_B_bias = f"transformer.blocks.{i}.attn2.to_k.lora_B.bias" + if has_bias_key_pattern_in_sd and ref_key_lora_B_bias in state_dict: + ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] + state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros( + ref_lora_B_bias_tensor.shape, device=target_device, dtype=lora_dtype + ) + # new_keys = set(state_dict.keys()) - orig_keys + # logger.debug(f"Keys ADDED by _maybe_expand_t2v_lora_for_i2v: {new_keys}") return state_dict def load_lora_weights( From c5a753a050665eed27af164f978571c57c794f82 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 15:57:10 +0300 Subject: [PATCH 10/16] fix --- src/diffusers/loaders/lora_pipeline.py | 82 ++++++-------------------- 1 file changed, 18 insertions(+), 64 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 7a50f1440772..33f61dc49e21 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4813,89 +4813,43 @@ def _maybe_expand_t2v_lora_for_i2v( if transformer.config.image_dim is None: return state_dict - # Determine the target device from the transformer - try: - target_device = next(transformer.parameters()).device - except StopIteration: - # Fallback if transformer has no parameters (should be rare for a full model) - # Try to infer from existing tensors in state_dict, else default to CPU - target_device = torch.device("cpu") - for v in state_dict.values(): - if torch.is_tensor(v): - target_device = v.device - break - - # Debug prints, can be removed after verification - logger.debug(f"Target device for new LoRA tensors: {target_device}") - logger.debug(f"Keys BEFORE _maybe_expand_t2v_lora_for_i2v: {list(state_dict.keys())[:5]}...") + target_device = transformer.device if any(k.startswith("transformer.blocks.") for k in state_dict): - block_indices = set() - # Iterate over a copy of keys if state_dict might be modified during iteration elsewhere (not here though) - for k in list(state_dict.keys()): - if k.startswith("transformer.blocks.") and ".attn2.to_k.lora_A.weight" in k: - try: - block_idx_str = k.split("blocks.")[1].split(".")[0] - block_indices.add(int(block_idx_str)) - except (IndexError, ValueError): - logger.warning(f"Could not parse block index from key: {k}") - continue - - if not block_indices: - logger.debug( - "No valid blocks found for T2V to I2V expansion referencing based on '.attn2.to_k.lora_A.weight'.") - return state_dict - - num_total_blocks = max(block_indices) + 1 - + num_blocks = len({k.split("blocks.")[1].split(".")[0] for k in state_dict if "blocks." in k}) is_i2v_lora = any("add_k_proj" in k for k in state_dict) and any("add_v_proj" in k for k in state_dict) + has_bias = any(".lora_B.bias" in k for k in state_dict) - # Check for bias keys that would have been converted by _convert_non_diffusers_wan_lora_to_diffusers - # e.g., 'transformer.blocks.0.attn2.to_k.lora_B.bias' - # This helps decide if zero biases should be added for the new projections. - has_bias_key_pattern_in_sd = any(".lora_B.bias" in k for k in state_dict) - - if is_i2v_lora: # If it's already an I2V LoRA, no expansion needed + if is_i2v_lora: return state_dict - logger.info( - "Adapting a T2V LoRA for I2V model by adding zero-initialized weights for image-specific cross-attention layers." - ) - for i in range(num_total_blocks): - # Define reference key patterns carefully. These keys should exist if the block `i` was part of the T2V LoRA. - ref_key_lora_A = f"transformer.blocks.{i}.attn2.to_k.lora_A.weight" - ref_key_lora_B = f"transformer.blocks.{i}.attn2.to_k.lora_B.weight" - - # Only proceed if the reference LoRA weights for this block exist in the T2V LoRA - if ref_key_lora_A not in state_dict or ref_key_lora_B not in state_dict: - continue + for i in range(num_blocks): + for o, c in zip(["k_img", "v_img"], ["add_k_proj", "add_v_proj"]): + # These keys should exist if the block `i` was part of the T2V LoRA. + ref_key_lora_A = f"transformer.blocks.{i}.attn2.to_k.lora_A.weight" + ref_key_lora_B = f"transformer.blocks.{i}.attn2.to_k.lora_B.weight" - ref_lora_A_weight = state_dict[ref_key_lora_A] - ref_lora_B_weight = state_dict[ref_key_lora_B] - - # Use dtype from reference LoRA tensors, device from transformer - lora_dtype = ref_lora_A_weight.dtype + if ref_key_lora_A not in state_dict or ref_key_lora_B not in state_dict: + continue + lora_dtype = ref_lora_A_weight.dtype - for _unused_orig_name, diffusers_name_part in zip(["k_img", "v_img"], ["add_k_proj", "add_v_proj"]): - # Create new tensors on the transformer's device and with the LoRA's dtype - state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_A.weight"] = torch.zeros( + state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros( ref_lora_A_weight.shape, device=target_device, dtype=lora_dtype ) - state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.weight"] = torch.zeros( - ref_lora_B_weight.shape, device=target_device, dtype=lora_dtype + state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"], device=target_device ) - # If the original LoRA had biases (indicated by has_bias_key_pattern_in_sd) + # If the original LoRA had biases (indicated by has_bias) # AND the specific reference bias key exists for this block. + ref_key_lora_B_bias = f"transformer.blocks.{i}.attn2.to_k.lora_B.bias" - if has_bias_key_pattern_in_sd and ref_key_lora_B_bias in state_dict: + if has_bias and ref_key_lora_B_bias in state_dict: ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros( ref_lora_B_bias_tensor.shape, device=target_device, dtype=lora_dtype ) - # new_keys = set(state_dict.keys()) - orig_keys - # logger.debug(f"Keys ADDED by _maybe_expand_t2v_lora_for_i2v: {new_keys}") return state_dict def load_lora_weights( From 36fea4e813d978b511b0ae6575958c4297f937f0 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 16:03:41 +0300 Subject: [PATCH 11/16] fix --- src/diffusers/loaders/lora_pipeline.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 33f61dc49e21..5f9876f0d51b 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4831,10 +4831,9 @@ def _maybe_expand_t2v_lora_for_i2v( if ref_key_lora_A not in state_dict or ref_key_lora_B not in state_dict: continue - lora_dtype = ref_lora_A_weight.dtype - state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros( - ref_lora_A_weight.shape, device=target_device, dtype=lora_dtype + state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros_like( + ref_lora_A_weight.shape, device=target_device ) state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"], device=target_device @@ -4846,8 +4845,8 @@ def _maybe_expand_t2v_lora_for_i2v( ref_key_lora_B_bias = f"transformer.blocks.{i}.attn2.to_k.lora_B.bias" if has_bias and ref_key_lora_B_bias in state_dict: ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] - state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros( - ref_lora_B_bias_tensor.shape, device=target_device, dtype=lora_dtype + state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros_like( + ref_lora_B_bias_tensor.shape, device=target_device, ) return state_dict From 6d62f4b47fe587354e32327c884ef171e143d250 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 16:05:28 +0300 Subject: [PATCH 12/16] fix --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 5f9876f0d51b..201d357452e8 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4833,7 +4833,7 @@ def _maybe_expand_t2v_lora_for_i2v( continue state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_A.weight"] = torch.zeros_like( - ref_lora_A_weight.shape, device=target_device + state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_A.weight"], device=target_device ) state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.weight"] = torch.zeros_like( state_dict[f"transformer.blocks.{i}.attn2.to_k.lora_B.weight"], device=target_device From 97168f4ce2d9ef7b7ea4b608a9f8984f63762f69 Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 16:09:30 +0300 Subject: [PATCH 13/16] fix --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 201d357452e8..25f0574eb1c8 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4846,7 +4846,7 @@ def _maybe_expand_t2v_lora_for_i2v( if has_bias and ref_key_lora_B_bias in state_dict: ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros_like( - ref_lora_B_bias_tensor.shape, device=target_device, + ref_lora_B_bias_tensor, device=target_device, ) return state_dict From 693277cd1a7a659fa94d21c4e5097f350ef38cab Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 16:13:53 +0300 Subject: [PATCH 14/16] fix --- src/diffusers/loaders/lora_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 25f0574eb1c8..c97a2d85ee3b 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4845,7 +4845,7 @@ def _maybe_expand_t2v_lora_for_i2v( ref_key_lora_B_bias = f"transformer.blocks.{i}.attn2.to_k.lora_B.bias" if has_bias and ref_key_lora_B_bias in state_dict: ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] - state_dict[f"transformer.blocks.{i}.attn2.{diffusers_name_part}.lora_B.bias"] = torch.zeros_like( + state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( ref_lora_B_bias_tensor, device=target_device, ) From 8ba2e90aea849bbaa6c3ab51ea53d2addb586603 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 20 May 2025 13:18:58 +0000 Subject: [PATCH 15/16] Apply style fixes --- src/diffusers/loaders/lora_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index c97a2d85ee3b..10b6a8f02710 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -4846,7 +4846,8 @@ def _maybe_expand_t2v_lora_for_i2v( if has_bias and ref_key_lora_B_bias in state_dict: ref_lora_B_bias_tensor = state_dict[ref_key_lora_B_bias] state_dict[f"transformer.blocks.{i}.attn2.{c}.lora_B.bias"] = torch.zeros_like( - ref_lora_B_bias_tensor, device=target_device, + ref_lora_B_bias_tensor, + device=target_device, ) return state_dict From 3a23d941f559759195dd30b5d206008f9e34f2bb Mon Sep 17 00:00:00 2001 From: linoytsaban Date: Tue, 20 May 2025 16:19:22 +0300 Subject: [PATCH 16/16] empty commit