V2: SDXL, Lightning, FreeInit, FreeNoise, IP-Adapter, Prompt Travel

Major architecture upgrade wrapping official diffusers AnimateDiff pipelines:

Pipeline Architecture:
- pipeline_v2.py: SD1.5 via diffusers AnimateDiffPipeline with all features
- pipeline_sdxl.py: SDXL support (1024x1024, dual text encoder)
- pipeline_lightning.py: ByteDance 1-8 step ultra-fast inference
- Keep legacy pipeline for backward compat and SparseCtrl

New Features (all via --pipeline v2):
- FreeInit: temporal consistency (--freeinit-iters 3)
- FreeNoise: long video >64 frames (--context-length 16)
- Prompt Travel: frame-varying prompts via YAML config
- IP-Adapter: image-conditioned generation (--ip-adapter-image)
- 6 noise schedulers (DDIM, Euler, Euler A, DPM++, Karras, PNDM)

CLI: python scripts/animate.py --pipeline [legacy|v2|sdxl|lightning]

Engineering:
- pyproject.toml for pip install (animatediff-cli command)
- Dockerfile + docker-compose.yml for deployment
- Example configs for all new pipelines
- diffusers>=0.28.0, peft>=0.6.0

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: GitLab CI

.gitignore

@@ -16,6 +16,7 @@ build/
 
 scripts/*
 !scripts/animate.py
+!scripts/__init__.py
 
 *.ipynb
 *.safetensors

Dockerfile

@@ -0,0 +1,24 @@
+FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime
+
+WORKDIR /app
+
+# System dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git ffmpeg libsm6 libxext6 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy project
+COPY . .
+
+# Pre-download tokenizer for faster cold start
+RUN python -c "from transformers import CLIPTokenizer; CLIPTokenizer.from_pretrained('openai/clip-vit-large-patch14')" 2>/dev/null || true
+
+# Expose Gradio port
+EXPOSE 7860
+
+# Default: run Gradio UI
+CMD ["python", "app.py"]

animatediff/pipelines/pipeline_lightning.py

@@ -0,0 +1,102 @@
+"""
+AnimateDiff-Lightning Pipeline — ultra-fast 1-8 step video generation.
+
+Based on ByteDance's cross-model diffusion distillation (arXiv:2403.12706).
+Produces high-quality animations in as few as 2-4 denoising steps.
+"""
+
+import torch
+from typing import Optional, Union, Dict
+
+from diffusers import AnimateDiffPipeline, MotionAdapter, EulerDiscreteScheduler
+from diffusers.utils import export_to_gif, export_to_video
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+
+
+LIGHTNING_REPO = "ByteDance/AnimateDiff-Lightning"
+VALID_STEPS = {1, 2, 4, 8}
+
+
+class AnimateDiffLightning:
+    """Ultra-fast AnimateDiff inference using distilled motion modules."""
+
+    def __init__(self, pipe: AnimateDiffPipeline, num_steps: int):
+        self.pipe = pipe
+        self.num_steps = num_steps
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_path: str = "emilianJR/epiCRealism",
+        num_steps: int = 4,
+        torch_dtype: torch.dtype = torch.float16,
+        device: str = "cuda",
+        enable_vae_slicing: bool = True,
+    ) -> "AnimateDiffLightning":
+        if num_steps not in VALID_STEPS:
+            raise ValueError(f"num_steps must be one of {VALID_STEPS}, got {num_steps}")
+
+        ckpt = f"animatediff_lightning_{num_steps}step_diffusers.safetensors"
+
+        adapter = MotionAdapter().to(device, torch_dtype)
+        adapter.load_state_dict(
+            load_file(hf_hub_download(LIGHTNING_REPO, ckpt), device=device)
+        )
+
+        pipe = AnimateDiffPipeline.from_pretrained(
+            model_path,
+            motion_adapter=adapter,
+            torch_dtype=torch_dtype,
+        ).to(device)
+
+        pipe.scheduler = EulerDiscreteScheduler.from_config(
+            pipe.scheduler.config,
+            timestep_spacing="trailing",
+            beta_schedule="linear",
+        )
+
+        if enable_vae_slicing:
+            pipe.enable_vae_slicing()
+
+        return cls(pipe, num_steps)
+
+    @torch.no_grad()
+    def generate(
+        self,
+        prompt: Union[str, Dict[int, str]],
+        negative_prompt: str = "",
+        num_frames: int = 16,
+        height: int = 512,
+        width: int = 512,
+        guidance_scale: float = 1.0,
+        seed: int = -1,
+        output_type: str = "pil",
+        decode_chunk_size: int = 4,
+    ):
+        generator = None
+        if seed >= 0:
+            generator = torch.Generator(device=self.pipe.device).manual_seed(seed)
+
+        output = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_frames=num_frames,
+            height=height,
+            width=width,
+            num_inference_steps=self.num_steps,
+            guidance_scale=guidance_scale,
+            generator=generator,
+            output_type=output_type,
+            decode_chunk_size=decode_chunk_size,
+        )
+        return output
+
+    def save(self, output, path: str, fps: int = 8):
+        frames = output.frames[0]
+        if path.endswith(".gif"):
+            export_to_gif(frames, path)
+        elif path.endswith(".mp4"):
+            export_to_video(frames, path, fps=fps)
+        else:
+            export_to_gif(frames, path)

animatediff/pipelines/pipeline_sdxl.py

@@ -0,0 +1,129 @@
+"""
+AnimateDiff SDXL Pipeline — wraps diffusers AnimateDiffSDXLPipeline.
+
+Generates higher quality videos using Stable Diffusion XL as the base model.
+Supports SDXL-specific features like dual text encoders.
+"""
+
+import torch
+from typing import Optional, Union, Dict
+
+from diffusers import (
+    AnimateDiffSDXLPipeline,
+    MotionAdapter,
+    DDIMScheduler,
+    EulerDiscreteScheduler,
+    DPMSolverMultistepScheduler,
+)
+from diffusers.utils import export_to_gif, export_to_video
+
+SDXL_SCHEDULER_MAP = {
+    "ddim": lambda config: DDIMScheduler.from_config(
+        config, clip_sample=False, timestep_spacing="linspace",
+        beta_schedule="linear", steps_offset=1,
+    ),
+    "euler": lambda config: EulerDiscreteScheduler.from_config(config, beta_schedule="linear"),
+    "dpm++": lambda config: DPMSolverMultistepScheduler.from_config(
+        config, beta_schedule="linear", algorithm_type="dpmsolver++",
+    ),
+    "dpm++-karras": lambda config: DPMSolverMultistepScheduler.from_config(
+        config, beta_schedule="linear", algorithm_type="dpmsolver++", use_karras_sigmas=True,
+    ),
+}
+
+
+class AnimateDiffSDXL:
+    """High-level wrapper around diffusers AnimateDiffSDXLPipeline."""
+
+    def __init__(self, pipe: AnimateDiffSDXLPipeline):
+        self.pipe = pipe
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_path: str = "stabilityai/stable-diffusion-xl-base-1.0",
+        motion_adapter_path: str = "guoyww/animatediff-motion-adapter-sdxl-beta",
+        torch_dtype: torch.dtype = torch.float16,
+        device: str = "cuda",
+        scheduler: str = "ddim",
+        enable_vae_slicing: bool = True,
+        enable_vae_tiling: bool = True,
+    ) -> "AnimateDiffSDXL":
+        adapter = MotionAdapter.from_pretrained(motion_adapter_path, torch_dtype=torch_dtype)
+
+        scheduler_cls = SDXL_SCHEDULER_MAP.get(scheduler)
+        sched_instance = None
+        if scheduler_cls:
+            sched_config = DDIMScheduler.from_pretrained(
+                model_path, subfolder="scheduler"
+            ).config
+            sched_instance = scheduler_cls(sched_config)
+
+        pipe = AnimateDiffSDXLPipeline.from_pretrained(
+            model_path,
+            motion_adapter=adapter,
+            torch_dtype=torch_dtype,
+            variant="fp16",
+        )
+
+        if sched_instance is not None:
+            pipe.scheduler = sched_instance
+
+        if enable_vae_slicing:
+            pipe.enable_vae_slicing()
+        if enable_vae_tiling:
+            pipe.enable_vae_tiling()
+
+        pipe.to(device)
+        return cls(pipe)
+
+    def load_lora(self, lora_path: str, adapter_name: str = "default", scale: float = 1.0):
+        self.pipe.load_lora_weights(lora_path, adapter_name=adapter_name)
+        self.pipe.set_adapters([adapter_name], [scale])
+
+    def set_scheduler(self, name: str):
+        if name in SDXL_SCHEDULER_MAP:
+            self.pipe.scheduler = SDXL_SCHEDULER_MAP[name](self.pipe.scheduler.config)
+        else:
+            raise ValueError(f"Unknown scheduler: {name}. Choose from: {list(SDXL_SCHEDULER_MAP.keys())}")
+
+    @torch.no_grad()
+    def generate(
+        self,
+        prompt: Union[str, Dict[int, str]],
+        negative_prompt: str = "low quality, worst quality, blurry",
+        num_frames: int = 16,
+        height: int = 1024,
+        width: int = 1024,
+        num_inference_steps: int = 20,
+        guidance_scale: float = 8.0,
+        seed: int = -1,
+        output_type: str = "pil",
+        decode_chunk_size: int = 4,
+    ):
+        generator = None
+        if seed >= 0:
+            generator = torch.Generator(device=self.pipe.device).manual_seed(seed)
+
+        output = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_frames=num_frames,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            generator=generator,
+            output_type=output_type,
+            decode_chunk_size=decode_chunk_size,
+        )
+        return output
+
+    def save(self, output, path: str, fps: int = 8):
+        frames = output.frames[0]
+        if path.endswith(".gif"):
+            export_to_gif(frames, path)
+        elif path.endswith(".mp4"):
+            export_to_video(frames, path, fps=fps)
+        else:
+            export_to_gif(frames, path)