[LoRA] kijai wan lora support for I2V

[linoytsaban]

modify _maybe_expand_t2v_lora_for_i2v to allow kijai t2v lora loading in i2v wan

output.44.mp4

import torch
from diffusers import AutoencoderKLWan, WanPipeline, UniPCMultistepScheduler
from diffusers.utils import export_to_video
from diffusers.loaders.lora_conversion_utils import _convert_non_diffusers_wan_lora_to_diffusers 
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
import torch
import numpy as np
from diffusers import AutoencoderKLWan, WanImageToVideoPipeline
from diffusers.utils import export_to_video, load_image
from transformers import CLIPVisionModel

MODEL_ID = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"
LORA_REPO_ID = "Kijai/WanVideo_comfy"
LORA_FILENAME = "Wan21_CausVid_14B_T2V_lora_rank32.safetensors"

image_encoder = CLIPVisionModel.from_pretrained(MODEL_ID, subfolder="image_encoder", torch_dtype=torch.float32)

vae = AutoencoderKLWan.from_pretrained(
    MODEL_ID,
    subfolder="vae",
    torch_dtype=torch.float32 # float32 for VAE stability
)
pipe = WanImageToVideoPipeline.from_pretrained(MODEL_ID, vae=vae, 
                                               image_encoder=image_encoder, 
                                               torch_dtype=torch.bfloat16)
flow_shift = 8.0
pipe.scheduler = UniPCMultistepScheduler.from_config(
    pipe.scheduler.config, flow_shift=flow_shift
)
pipe.to("cuda")

# --- LoRA Loading ---
causvid_path = hf_hub_download(repo_id=LORA_REPO_ID, filename=LORA_FILENAME)
pipe.load_lora_weights(causvid_path,adapter_name="causvid_lora")
pipe.set_adapters(["causvid_lora"], adapter_weights=[1.0])

image = load_image(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/penguin.png"
)
max_area = 480 * 1280
aspect_ratio = image.height / image.width
mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
image = image.resize((width, height))
prompt = (
    "a penguin playfully dancing in the snow"
)
negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"

output = pipe(
    image=image,
    num_inference_steps=4,
    generator = torch.Generator(device="cuda").manual_seed(0),
    prompt=prompt, 
    negative_prompt=negative_prompt, 
    height=height, 
    width=width, num_frames=81, 
    guidance_scale=1.0
).frames[0]
export_to_video(output, "output.mp4", fps=16)

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

[linoytsaban]

@bot /style

[github-actions]

Style fixes have been applied. View the workflow run here.

[sayakpaul]

Maybe update the PR description with the cool penguin you got dancing?

[a-r-r-o-w]

Nice, LGTM!