diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py
index c4a3a5bffcbc..dad85f2b4382 100644
--- a/src/diffusers/__init__.py
+++ b/src/diffusers/__init__.py
@@ -227,8 +227,10 @@
[
"AudioPipelineOutput",
"AutoPipelineForImage2Image",
+ "AutoPipelineForImage2Video",
"AutoPipelineForInpainting",
"AutoPipelineForText2Image",
+ "AutoPipelineForText2Video",
"ConsistencyModelPipeline",
"DanceDiffusionPipeline",
"DDIMPipeline",
@@ -818,8 +820,10 @@
from .pipelines import (
AudioPipelineOutput,
AutoPipelineForImage2Image,
+ AutoPipelineForImage2Video,
AutoPipelineForInpainting,
AutoPipelineForText2Image,
+ AutoPipelineForText2Video,
BlipDiffusionControlNetPipeline,
BlipDiffusionPipeline,
CLIPImageProjection,
diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py
index 011f23ed371c..abf1b150f581 100644
--- a/src/diffusers/pipelines/__init__.py
+++ b/src/diffusers/pipelines/__init__.py
@@ -46,6 +46,8 @@
"AutoPipelineForImage2Image",
"AutoPipelineForInpainting",
"AutoPipelineForText2Image",
+ "AutoPipelineForText2Video",
+ "AutoPipelineForImage2Video",
]
_import_structure["consistency_models"] = ["ConsistencyModelPipeline"]
_import_structure["dance_diffusion"] = ["DanceDiffusionPipeline"]
@@ -468,8 +470,10 @@
else:
from .auto_pipeline import (
AutoPipelineForImage2Image,
+ AutoPipelineForImage2Video,
AutoPipelineForInpainting,
AutoPipelineForText2Image,
+ AutoPipelineForText2Video,
)
from .consistency_models import ConsistencyModelPipeline
from .dance_diffusion import DanceDiffusionPipeline
diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py
index 6a5f6098b6fb..428580f6adcd 100644
--- a/src/diffusers/pipelines/auto_pipeline.py
+++ b/src/diffusers/pipelines/auto_pipeline.py
@@ -20,9 +20,12 @@
from ..configuration_utils import ConfigMixin
from ..models.controlnets import ControlNetUnionModel
from ..utils import is_sentencepiece_available
+from .allegro import AllegroPipeline
from .aura_flow import AuraFlowPipeline
+from .cogvideo import CogVideoXImageToVideoPipeline, CogVideoXPipeline
from .cogview3 import CogView3PlusPipeline
from .cogview4 import CogView4ControlPipeline, CogView4Pipeline
+from .consisid import ConsisIDPipeline
from .controlnet import (
StableDiffusionControlNetImg2ImgPipeline,
StableDiffusionControlNetInpaintPipeline,
@@ -39,6 +42,7 @@
StableDiffusion3ControlNetPipeline,
)
from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline
+from .easyanimate import EasyAnimatePipeline
from .flux import (
FluxControlImg2ImgPipeline,
FluxControlInpaintPipeline,
@@ -50,6 +54,7 @@
FluxInpaintPipeline,
FluxPipeline,
)
+from .hunyuan_video import HunyuanVideoImageToVideoPipeline, HunyuanVideoPipeline
from .hunyuandit import HunyuanDiTPipeline
from .kandinsky import (
KandinskyCombinedPipeline,
@@ -69,8 +74,10 @@
)
from .kandinsky3 import Kandinsky3Img2ImgPipeline, Kandinsky3Pipeline
from .latent_consistency_models import LatentConsistencyModelImg2ImgPipeline, LatentConsistencyModelPipeline
+from .ltx import LTXImageToVideoPipeline, LTXPipeline
from .lumina import LuminaPipeline
from .lumina2 import Lumina2Pipeline
+from .mochi import MochiPipeline
from .pag import (
HunyuanDiTPAGPipeline,
PixArtSigmaPAGPipeline,
@@ -106,6 +113,9 @@
StableDiffusionXLInpaintPipeline,
StableDiffusionXLPipeline,
)
+from .stable_video_diffusion import StableVideoDiffusionPipeline
+from .text_to_video_synthesis import TextToVideoSDPipeline, VideoToVideoSDPipeline
+from .wan import WanImageToVideoPipeline, WanPipeline
from .wuerstchen import WuerstchenCombinedPipeline, WuerstchenDecoderPipeline
@@ -214,6 +224,33 @@
]
)
+
+AUTO_TEXT2VIDEO_PIPELINES_MAPPING = OrderedDict(
+ [
+ ("allegro", AllegroPipeline),
+ ("cogvideox", CogVideoXPipeline),
+ ("consisid", ConsisIDPipeline),
+ ("easyanimate", EasyAnimatePipeline),
+ ("hunyuan-video", HunyuanVideoPipeline),
+ ("ltx", LTXPipeline),
+ ("mochi", MochiPipeline),
+ ("svd", StableVideoDiffusionPipeline),
+ ("t2v-sd", TextToVideoSDPipeline),
+ ("wan", WanPipeline),
+ ]
+)
+
+AUTO_IMAGE2VIDEO_PIPELINES_MAPPING = OrderedDict(
+ [
+ ("cogvideox", CogVideoXImageToVideoPipeline),
+ ("hunyuan-video", HunyuanVideoImageToVideoPipeline),
+ ("ltx", LTXImageToVideoPipeline),
+ ("t2v-sd", VideoToVideoSDPipeline),
+ ("wan", WanImageToVideoPipeline),
+ ]
+)
+
+
if is_sentencepiece_available():
from .kolors import KolorsImg2ImgPipeline, KolorsPipeline
from .pag import KolorsPAGPipeline
@@ -226,6 +263,8 @@
AUTO_TEXT2IMAGE_PIPELINES_MAPPING,
AUTO_IMAGE2IMAGE_PIPELINES_MAPPING,
AUTO_INPAINT_PIPELINES_MAPPING,
+ AUTO_TEXT2VIDEO_PIPELINES_MAPPING,
+ AUTO_IMAGE2VIDEO_PIPELINES_MAPPING,
_AUTO_TEXT2IMAGE_DECODER_PIPELINES_MAPPING,
_AUTO_IMAGE2IMAGE_DECODER_PIPELINES_MAPPING,
_AUTO_INPAINT_DECODER_PIPELINES_MAPPING,
@@ -244,6 +283,14 @@ def _get_connected_pipeline(pipeline_cls):
)
if pipeline_cls in _AUTO_INPAINT_DECODER_PIPELINES_MAPPING.values():
return _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False)
+ if pipeline_cls in AUTO_TEXT2VIDEO_PIPELINES_MAPPING.values():
+ return _get_task_class(
+ AUTO_TEXT2VIDEO_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False
+ )
+ if pipeline_cls in AUTO_IMAGE2VIDEO_PIPELINES_MAPPING.values():
+ return _get_task_class(
+ AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False
+ )
def _get_task_class(mapping, pipeline_class_name, throw_error_if_not_exist: bool = True):
@@ -1174,3 +1221,512 @@ def from_pipe(cls, pipeline, **kwargs):
model.register_to_config(**unused_original_config)
return model
+
+
+class AutoPipelineForText2Video(ConfigMixin):
+ r"""
+
+ [`AutoPipelineForText2Video`] is a generic pipeline class that instantiates a text-to-video pipeline class. The
+ specific underlying pipeline class is automatically selected from either the
+ [`~AutoPipelineForText2Video.from_pretrained`] or [`~AutoPipelineForText2Video.from_pipe`] methods.
+
+ This class cannot be instantiated using `__init__()` (throws an error).
+
+ Class attributes:
+
+ - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the
+ diffusion pipeline's components.
+
+ """
+
+ config_name = "model_index.json"
+
+ def __init__(self, *args, **kwargs):
+ raise EnvironmentError(
+ f"{self.__class__.__name__} is designed to be instantiated "
+ f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or "
+ f"`{self.__class__.__name__}.from_pipe(pipeline)` methods."
+ )
+
+ @classmethod
+ @validate_hf_hub_args
+ def from_pretrained(cls, pretrained_model_or_path, **kwargs):
+ r"""
+ Instantiates a text-to-video Pytorch diffusion pipeline from pretrained pipeline weight.
+
+ The from_pretrained() method takes care of returning the correct pipeline class instance by:
+ 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its
+ config object
+ 2. Find the text-to-video pipeline linked to the pipeline class using pattern matching on pipeline class
+ name.
+
+ If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetPipeline`] object.
+
+ The pipeline is set in evaluation mode (`model.eval()`) by default.
+
+ If you get the error message below, you need to finetune the weights for your downstream task:
+
+ ```
+ Some weights of UNet2DConditionModel were not initialized from the model checkpoint at stable-diffusion-v1-5/stable-diffusion-v1-5 and are newly initialized because the shapes did not match:
+ - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated
+ You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+ ```
+
+ Parameters:
+ pretrained_model_or_path (`str` or `os.PathLike`, *optional*):
+ Can be either:
+
+ - A string, the *repo id* (for example `CompVis/ldm-text2vm-large-256`) of a pretrained pipeline
+ hosted on the Hub.
+ - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights
+ saved using
+ [`~DiffusionPipeline.save_pretrained`].
+ torch_dtype (`str` or `torch.dtype`, *optional*):
+ Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the
+ dtype is automatically derived from the model's weights.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force the (re-)download of the model weights and configuration files, overriding the
+ cached versions if they exist.
+ cache_dir (`Union[str, os.PathLike]`, *optional*):
+ Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
+ is not used.
+
+ proxies (`Dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ output_loading_info(`bool`, *optional*, defaults to `False`):
+ Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ Whether to only load local model weights and configuration files or not. If set to `True`, the model
+ won't be downloaded from the Hub.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
+ `diffusers-cli login` (stored in `~/.huggingface`) is used.
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
+ allowed by Git.
+ custom_revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id similar to
+ `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a
+ custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub.
+ mirror (`str`, *optional*):
+ Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not
+ guarantee the timeliness or safety of the source, and you should refer to the mirror site for more
+ information.
+ device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*):
+ A map that specifies where each submodule should go. It doesn’t need to be defined for each
+ parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the
+ same device.
+
+ Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For
+ more information about each option see [designing a device
+ map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
+ max_memory (`Dict`, *optional*):
+ A dictionary device identifier for the maximum memory. Will default to the maximum memory available for
+ each GPU and the available CPU RAM if unset.
+ offload_folder (`str` or `os.PathLike`, *optional*):
+ The path to offload weights if device_map contains the value `"disk"`.
+ offload_state_dict (`bool`, *optional*):
+ If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if
+ the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True`
+ when there is some disk offload.
+ low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`):
+ Speed up model loading only loading the pretrained weights and not initializing the weights. This also
+ tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
+ Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this
+ argument to `True` will raise an error.
+ use_safetensors (`bool`, *optional*, defaults to `None`):
+ If set to `None`, the safetensors weights are downloaded if they're available **and** if the
+ safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors
+ weights. If set to `False`, safetensors weights are not loaded.
+ kwargs (remaining dictionary of keyword arguments, *optional*):
+ Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline
+ class). The overwritten components are passed directly to the pipelines `__init__` method. See example
+ below for more information.
+ variant (`str`, *optional*):
+ Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when
+ loading `from_flax`.
+
+
+
+ To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
+ `huggingface-cli login`.
+
+
+
+ Examples:
+
+ ```py
+ >>> from diffusers import AutoPipelineForText2Video
+
+ >>> pipeline = AutoPipelineForText2Video.from_pretrained("Lightricks/LTX-Video")
+ >>> video = pipeline(prompt).videos[0]
+ ```
+ """
+ cache_dir = kwargs.pop("cache_dir", None)
+ force_download = kwargs.pop("force_download", False)
+ proxies = kwargs.pop("proxies", None)
+ token = kwargs.pop("token", None)
+ local_files_only = kwargs.pop("local_files_only", False)
+ revision = kwargs.pop("revision", None)
+
+ load_config_kwargs = {
+ "cache_dir": cache_dir,
+ "force_download": force_download,
+ "proxies": proxies,
+ "token": token,
+ "local_files_only": local_files_only,
+ "revision": revision,
+ }
+
+ config = cls.load_config(pretrained_model_or_path, **load_config_kwargs)
+ orig_class_name = config["_class_name"]
+
+ text_2_video_cls = _get_task_class(AUTO_TEXT2VIDEO_PIPELINES_MAPPING, orig_class_name)
+
+ kwargs = {**load_config_kwargs, **kwargs}
+ return text_2_video_cls.from_pretrained(pretrained_model_or_path, **kwargs)
+
+ @classmethod
+ def from_pipe(cls, pipeline, **kwargs):
+ r"""
+ Instantiates a text-to-video Pytorch diffusion pipeline from another instantiated diffusion pipeline class.
+
+ The from_pipe() method takes care of returning the correct pipeline class instance by finding the text-to-video
+ pipeline linked to the pipeline class using pattern matching on pipeline class name.
+
+ All the modules the pipeline contains will be used to initialize the new pipeline without reallocating
+ additional memory.
+
+ The pipeline is set in evaluation mode (`model.eval()`) by default.
+
+ Parameters:
+ pipeline (`DiffusionPipeline`):
+ an instantiated `DiffusionPipeline` object
+
+ ```py
+ >>> from diffusers import AutoPipelineForText2Video, AutoPipelineForImage2Video
+
+ >>> pipe_i2v = AutoPipelineForImage2Video.from_pretrained("Lightricks/LTX-Video")
+
+ >>> pipe_t2v = AutoPipelineForText2Video.from_pipe(pipe_i2v)
+ >>> video = pipe_t2v(prompt).videos[0]
+ ```
+ """
+
+ original_config = dict(pipeline.config)
+ original_cls_name = pipeline.__class__.__name__
+
+ # derive the pipeline class to instantiate
+ text_2_video_cls = _get_task_class(AUTO_TEXT2VIDEO_PIPELINES_MAPPING, original_cls_name)
+
+ # define expected module and optional kwargs given the pipeline signature
+ expected_modules, optional_kwargs = text_2_video_cls._get_signature_keys(text_2_video_cls)
+
+ pretrained_model_name_or_path = original_config.pop("_name_or_path", None)
+
+ # allow users pass modules in `kwargs` to override the original pipeline's components
+ passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}
+ original_class_obj = {
+ k: pipeline.components[k]
+ for k, v in pipeline.components.items()
+ if k in expected_modules and k not in passed_class_obj
+ }
+
+ # allow users pass optional kwargs to override the original pipelines config attribute
+ passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs}
+ original_pipe_kwargs = {
+ k: original_config[k]
+ for k, v in original_config.items()
+ if k in optional_kwargs and k not in passed_pipe_kwargs
+ }
+
+ # config that were not expected by original pipeline is stored as private attribute
+ # we will pass them as optional arguments if they can be accepted by the pipeline
+ additional_pipe_kwargs = [
+ k[1:]
+ for k in original_config.keys()
+ if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs
+ ]
+ for k in additional_pipe_kwargs:
+ original_pipe_kwargs[k] = original_config.pop(f"_{k}")
+
+ text_2_video_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs}
+
+ # store unused config as private attribute
+ unused_original_config = {
+ f"{'' if k.startswith('_') else '_'}{k}": original_config[k]
+ for k, v in original_config.items()
+ if k not in text_2_video_kwargs
+ }
+
+ missing_modules = (
+ set(expected_modules) - set(text_2_video_cls._optional_components) - set(text_2_video_kwargs.keys())
+ )
+
+ if len(missing_modules) > 0:
+ raise ValueError(
+ f"Pipeline {text_2_video_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed"
+ )
+
+ model = text_2_video_cls(**text_2_video_kwargs)
+ model.register_to_config(_name_or_path=pretrained_model_name_or_path)
+ model.register_to_config(**unused_original_config)
+
+ return model
+
+
+class AutoPipelineForImage2Video(ConfigMixin):
+ r"""
+
+ [`AutoPipelineForImage2Video`] is a generic pipeline class that instantiates an image-to-video pipeline class. The
+ specific underlying pipeline class is automatically selected from either the
+ [`~AutoPipelineForImage2Video.from_pretrained`] or [`~AutoPipelineForImage2Video.from_pipe`] methods.
+
+ This class cannot be instantiated using `__init__()` (throws an error).
+
+ Class attributes:
+
+ - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the
+ diffusion pipeline's components.
+
+ """
+
+ config_name = "model_index.json"
+
+ def __init__(self, *args, **kwargs):
+ raise EnvironmentError(
+ f"{self.__class__.__name__} is designed to be instantiated "
+ f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or "
+ f"`{self.__class__.__name__}.from_pipe(pipeline)` methods."
+ )
+
+ @classmethod
+ @validate_hf_hub_args
+ def from_pretrained(cls, pretrained_model_or_path, **kwargs):
+ r"""
+ Instantiates a image-to-video Pytorch diffusion pipeline from pretrained pipeline weight.
+
+ The from_pretrained() method takes care of returning the correct pipeline class instance by:
+ 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its
+ config object
+ 2. Find the image-to-video pipeline linked to the pipeline class using pattern matching on pipeline class
+ name.
+
+ If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetImg2ImgPipeline`]
+ object.
+
+ The pipeline is set in evaluation mode (`model.eval()`) by default.
+
+ If you get the error message below, you need to finetune the weights for your downstream task:
+
+ ```
+ Some weights of UNet2DConditionModel were not initialized from the model checkpoint at stable-diffusion-v1-5/stable-diffusion-v1-5 and are newly initialized because the shapes did not match:
+ - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated
+ You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+ ```
+
+ Parameters:
+ pretrained_model_or_path (`str` or `os.PathLike`, *optional*):
+ Can be either:
+
+ - A string, the *repo id* (for example `CompVis/ldm-text2vm-large-256`) of a pretrained pipeline
+ hosted on the Hub.
+ - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights
+ saved using
+ [`~DiffusionPipeline.save_pretrained`].
+ torch_dtype (`str` or `torch.dtype`, *optional*):
+ Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the
+ dtype is automatically derived from the model's weights.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force the (re-)download of the model weights and configuration files, overriding the
+ cached versions if they exist.
+ cache_dir (`Union[str, os.PathLike]`, *optional*):
+ Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
+ is not used.
+
+ proxies (`Dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ output_loading_info(`bool`, *optional*, defaults to `False`):
+ Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ Whether to only load local model weights and configuration files or not. If set to `True`, the model
+ won't be downloaded from the Hub.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
+ `diffusers-cli login` (stored in `~/.huggingface`) is used.
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
+ allowed by Git.
+ custom_revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id similar to
+ `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a
+ custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub.
+ mirror (`str`, *optional*):
+ Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not
+ guarantee the timeliness or safety of the source, and you should refer to the mirror site for more
+ information.
+ device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*):
+ A map that specifies where each submodule should go. It doesn’t need to be defined for each
+ parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the
+ same device.
+
+ Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For
+ more information about each option see [designing a device
+ map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
+ max_memory (`Dict`, *optional*):
+ A dictionary device identifier for the maximum memory. Will default to the maximum memory available for
+ each GPU and the available CPU RAM if unset.
+ offload_folder (`str` or `os.PathLike`, *optional*):
+ The path to offload weights if device_map contains the value `"disk"`.
+ offload_state_dict (`bool`, *optional*):
+ If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if
+ the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True`
+ when there is some disk offload.
+ low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`):
+ Speed up model loading only loading the pretrained weights and not initializing the weights. This also
+ tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
+ Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this
+ argument to `True` will raise an error.
+ use_safetensors (`bool`, *optional*, defaults to `None`):
+ If set to `None`, the safetensors weights are downloaded if they're available **and** if the
+ safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors
+ weights. If set to `False`, safetensors weights are not loaded.
+ kwargs (remaining dictionary of keyword arguments, *optional*):
+ Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline
+ class). The overwritten components are passed directly to the pipelines `__init__` method. See example
+ below for more information.
+ variant (`str`, *optional*):
+ Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when
+ loading `from_flax`.
+
+
+
+ To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
+ `huggingface-cli login`.
+
+
+
+ Examples:
+
+ ```py
+ >>> from diffusers import AutoPipelineForImage2Video
+
+ >>> pipeline = AutoPipelineForImage2Video.from_pretrained("Lightricks/LTX-Video")
+ >>> video = pipeline(prompt, image).videos[0]
+ ```
+ """
+ cache_dir = kwargs.pop("cache_dir", None)
+ force_download = kwargs.pop("force_download", False)
+ proxies = kwargs.pop("proxies", None)
+ token = kwargs.pop("token", None)
+ local_files_only = kwargs.pop("local_files_only", False)
+ revision = kwargs.pop("revision", None)
+
+ load_config_kwargs = {
+ "cache_dir": cache_dir,
+ "force_download": force_download,
+ "proxies": proxies,
+ "token": token,
+ "local_files_only": local_files_only,
+ "revision": revision,
+ }
+
+ config = cls.load_config(pretrained_model_or_path, **load_config_kwargs)
+ orig_class_name = config["_class_name"]
+
+ image_2_video_cls = _get_task_class(AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, orig_class_name)
+
+ kwargs = {**load_config_kwargs, **kwargs}
+ return image_2_video_cls.from_pretrained(pretrained_model_or_path, **kwargs)
+
+ @classmethod
+ def from_pipe(cls, pipeline, **kwargs):
+ r"""
+ Instantiates a image-to-video Pytorch diffusion pipeline from another instantiated diffusion pipeline class.
+
+ The from_pipe() method takes care of returning the correct pipeline class instance by finding the
+ image-to-video pipeline linked to the pipeline class using pattern matching on pipeline class name.
+
+ All the modules the pipeline contains will be used to initialize the new pipeline without reallocating
+ additional memory.
+
+ The pipeline is set in evaluation mode (`model.eval()`) by default.
+
+ Parameters:
+ pipeline (`DiffusionPipeline`):
+ an instantiated `DiffusionPipeline` object
+
+ Examples:
+
+ ```py
+ >>> from diffusers import AutoPipelineForText2vmage, AutoPipelineForImage2Video
+
+ >>> pipe_t2v = AutoPipelineForText2vmage.from_pretrained("Lightricks/LTX-Video")
+
+ >>> pipe_i2v = AutoPipelineForImage2Video.from_pipe(pipe_t2v)
+ >>> video = pipe_i2v(prompt, image).videos[0]
+ ```
+ """
+
+ original_config = dict(pipeline.config)
+ original_cls_name = pipeline.__class__.__name__
+
+ # derive the pipeline class to instantiate
+ image_2_video_cls = _get_task_class(AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, original_cls_name)
+
+ # define expected module and optional kwargs given the pipeline signature
+ expected_modules, optional_kwargs = image_2_video_cls._get_signature_keys(image_2_video_cls)
+
+ pretrained_model_name_or_path = original_config.pop("_name_or_path", None)
+
+ # allow users pass modules in `kwargs` to override the original pipeline's components
+ passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}
+ original_class_obj = {
+ k: pipeline.components[k]
+ for k, v in pipeline.components.items()
+ if k in expected_modules and k not in passed_class_obj
+ }
+
+ # allow users pass optional kwargs to override the original pipelines config attribute
+ passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs}
+ original_pipe_kwargs = {
+ k: original_config[k]
+ for k, v in original_config.items()
+ if k in optional_kwargs and k not in passed_pipe_kwargs
+ }
+
+ # config attribute that were not expected by original pipeline is stored as its private attribute
+ # we will pass them as optional arguments if they can be accepted by the pipeline
+ additional_pipe_kwargs = [
+ k[1:]
+ for k in original_config.keys()
+ if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs
+ ]
+ for k in additional_pipe_kwargs:
+ original_pipe_kwargs[k] = original_config.pop(f"_{k}")
+
+ image_2_video_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs}
+
+ # store unused config as private attribute
+ unused_original_config = {
+ f"{'' if k.startswith('_') else '_'}{k}": original_config[k]
+ for k, v in original_config.items()
+ if k not in image_2_video_kwargs
+ }
+
+ missing_modules = (
+ set(expected_modules) - set(image_2_video_cls._optional_components) - set(image_2_video_kwargs.keys())
+ )
+
+ if len(missing_modules) > 0:
+ raise ValueError(
+ f"Pipeline {image_2_video_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed"
+ )
+
+ model = image_2_video_cls(**image_2_video_kwargs)
+ model.register_to_config(_name_or_path=pretrained_model_name_or_path)
+ model.register_to_config(**unused_original_config)
+
+ return model
diff --git a/src/diffusers/utils/dummy_pt_objects.py b/src/diffusers/utils/dummy_pt_objects.py
index bf2f19ee2d26..3cc6b41a67dc 100644
--- a/src/diffusers/utils/dummy_pt_objects.py
+++ b/src/diffusers/utils/dummy_pt_objects.py
@@ -1163,6 +1163,21 @@ def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["torch"])
+class AutoPipelineForImage2Video(metaclass=DummyObject):
+ _backends = ["torch"]
+
+ def __init__(self, *args, **kwargs):
+ requires_backends(self, ["torch"])
+
+ @classmethod
+ def from_config(cls, *args, **kwargs):
+ requires_backends(cls, ["torch"])
+
+ @classmethod
+ def from_pretrained(cls, *args, **kwargs):
+ requires_backends(cls, ["torch"])
+
+
class AutoPipelineForInpainting(metaclass=DummyObject):
_backends = ["torch"]
@@ -1193,6 +1208,21 @@ def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["torch"])
+class AutoPipelineForText2Video(metaclass=DummyObject):
+ _backends = ["torch"]
+
+ def __init__(self, *args, **kwargs):
+ requires_backends(self, ["torch"])
+
+ @classmethod
+ def from_config(cls, *args, **kwargs):
+ requires_backends(cls, ["torch"])
+
+ @classmethod
+ def from_pretrained(cls, *args, **kwargs):
+ requires_backends(cls, ["torch"])
+
+
class BlipDiffusionControlNetPipeline(metaclass=DummyObject):
_backends = ["torch"]
diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py
index 561a9011c6ae..84327a19c440 100644
--- a/tests/pipelines/test_pipelines_auto.py
+++ b/tests/pipelines/test_pipelines_auto.py
@@ -25,8 +25,10 @@
from diffusers import (
AutoPipelineForImage2Image,
+ AutoPipelineForImage2Video,
AutoPipelineForInpainting,
AutoPipelineForText2Image,
+ AutoPipelineForText2Video,
ControlNetModel,
DiffusionPipeline,
)
@@ -454,6 +456,32 @@ def test_from_pipe_optional_components(self):
pipe = AutoPipelineForText2Image.from_pipe(pipe, image_encoder=None)
assert pipe.image_encoder is None
+ def test_from_pretrained_text2video(self):
+ repo = "hf-internal-testing/tiny-cogvideox-pipe"
+ pipe = AutoPipelineForText2Video.from_pretrained(repo)
+ assert pipe.__class__.__name__ == "CogVideoXPipeline"
+
+ def test_from_pretrained_image2video(self):
+ repo = "hf-internal-testing/tiny-cogvideox-pipe"
+ pipe = AutoPipelineForImage2Video.from_pretrained(repo)
+ assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline"
+
+ def test_from_pipe_text2video_image2video(self):
+ repo = "hf-internal-testing/tiny-cogvideox-pipe"
+ pipe = AutoPipelineForText2Video.from_pretrained(repo)
+ assert pipe.__class__.__name__ == "CogVideoXPipeline"
+
+ pipe = AutoPipelineForImage2Video.from_pipe(pipe)
+ assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline"
+
+ def test_from_pipe_image2video_text2video(self):
+ repo = "hf-internal-testing/tiny-cogvideox-pipe"
+ pipe = AutoPipelineForImage2Video.from_pretrained(repo)
+ assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline"
+
+ pipe = AutoPipelineForText2Video.from_pipe(pipe)
+ assert pipe.__class__.__name__ == "CogVideoXPipeline"
+
@slow
class AutoPipelineIntegrationTest(unittest.TestCase):