Skip to content

Commit bb482ba

Browse files
DN6sayakpaul
andcommitted
[Single File] Add GGUF support for LTX (#10298)
* update * add docs. --------- Co-authored-by: Sayak Paul <[email protected]>
1 parent 7ffa043 commit bb482ba

File tree

2 files changed

+46
-8
lines changed

2 files changed

+46
-8
lines changed

docs/source/en/api/pipelines/ltx_video.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,45 @@ pipe = LTXImageToVideoPipeline.from_single_file(
6161
)
6262
```
6363

64+
Loading [LTX GGUF checkpoints](https://huggingface.co/city96/LTX-Video-gguf) are also supported:
65+
66+
```py
67+
import torch
68+
from diffusers.utils import export_to_video
69+
from diffusers import LTXPipeline, LTXVideoTransformer3DModel, GGUFQuantizationConfig
70+
71+
ckpt_path = (
72+
"https://huggingface.co/city96/LTX-Video-gguf/blob/main/ltx-video-2b-v0.9-Q3_K_S.gguf"
73+
)
74+
transformer = LTXVideoTransformer3DModel.from_single_file(
75+
ckpt_path,
76+
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
77+
torch_dtype=torch.bfloat16,
78+
)
79+
pipe = LTXPipeline.from_pretrained(
80+
"Lightricks/LTX-Video",
81+
transformer=transformer,
82+
generator=torch.manual_seed(0),
83+
torch_dtype=torch.bfloat16,
84+
)
85+
pipe.enable_model_cpu_offload()
86+
87+
prompt = "A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage"
88+
negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
89+
90+
video = pipe(
91+
prompt=prompt,
92+
negative_prompt=negative_prompt,
93+
width=704,
94+
height=480,
95+
num_frames=161,
96+
num_inference_steps=50,
97+
).frames[0]
98+
export_to_video(video, "output_gguf_ltx.mp4", fps=24)
99+
```
100+
101+
Make sure to read the [documentation on GGUF](../../quantization/gguf) to learn more about our GGUF support.
102+
64103
Refer to [this section](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogvideox#memory-optimization) to learn more about optimizing memory consumption.
65104

66105
## LTXPipeline

src/diffusers/loaders/single_file_utils.py

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,11 @@
9999
"model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale",
100100
],
101101
"ltx-video": [
102-
(
103-
"model.diffusion_model.patchify_proj.weight",
104-
"model.diffusion_model.transformer_blocks.27.scale_shift_table",
105-
),
102+
"model.diffusion_model.patchify_proj.weight",
103+
"model.diffusion_model.transformer_blocks.27.scale_shift_table",
104+
"patchify_proj.weight",
105+
"transformer_blocks.27.scale_shift_table",
106+
"vae.per_channel_statistics.mean-of-means",
106107
],
107108
"autoencoder-dc": "decoder.stages.1.op_list.0.main.conv.conv.bias",
108109
"autoencoder-dc-sana": "encoder.project_in.conv.bias",
@@ -601,7 +602,7 @@ def infer_diffusers_model_type(checkpoint):
601602
else:
602603
model_type = "flux-schnell"
603604

604-
elif any(all(key in checkpoint for key in key_list) for key_list in CHECKPOINT_KEY_NAMES["ltx-video"]):
605+
elif any(key in checkpoint for key in CHECKPOINT_KEY_NAMES["ltx-video"]):
605606
model_type = "ltx-video"
606607

607608
elif CHECKPOINT_KEY_NAMES["autoencoder-dc"] in checkpoint:
@@ -2266,9 +2267,7 @@ def swap_scale_shift(weight):
22662267

22672268

22682269
def convert_ltx_transformer_checkpoint_to_diffusers(checkpoint, **kwargs):
2269-
converted_state_dict = {
2270-
key: checkpoint.pop(key) for key in list(checkpoint.keys()) if "model.diffusion_model." in key
2271-
}
2270+
converted_state_dict = {key: checkpoint.pop(key) for key in list(checkpoint.keys()) if "vae" not in key}
22722271

22732272
TRANSFORMER_KEYS_RENAME_DICT = {
22742273
"model.diffusion_model.": "",

0 commit comments

Comments
 (0)