From dc1d7222a9c088e55a99a9b0fb2415a774a84479 Mon Sep 17 00:00:00 2001 From: hlky Date: Tue, 15 Apr 2025 08:00:37 +0100 Subject: [PATCH 1/3] Introduce AutoPipelineForText2Video and AutoPipelineForImage2Video --- src/diffusers/__init__.py | 4 + src/diffusers/pipelines/__init__.py | 4 + src/diffusers/pipelines/auto_pipeline.py | 560 +++++++++++++++++++++++ tests/pipelines/test_pipelines_auto.py | 28 ++ 4 files changed, 596 insertions(+) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index c4a3a5bffcbc..817631a8c6b6 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -229,6 +229,8 @@ "AutoPipelineForImage2Image", "AutoPipelineForInpainting", "AutoPipelineForText2Image", + "AutoPipelineForText2Video", + "AutoPipelineForImage2Video", "ConsistencyModelPipeline", "DanceDiffusionPipeline", "DDIMPipeline", @@ -818,8 +820,10 @@ from .pipelines import ( AudioPipelineOutput, AutoPipelineForImage2Image, + AutoPipelineForImage2Video, AutoPipelineForInpainting, AutoPipelineForText2Image, + AutoPipelineForText2Video, BlipDiffusionControlNetPipeline, BlipDiffusionPipeline, CLIPImageProjection, diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index 011f23ed371c..abf1b150f581 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -46,6 +46,8 @@ "AutoPipelineForImage2Image", "AutoPipelineForInpainting", "AutoPipelineForText2Image", + "AutoPipelineForText2Video", + "AutoPipelineForImage2Video", ] _import_structure["consistency_models"] = ["ConsistencyModelPipeline"] _import_structure["dance_diffusion"] = ["DanceDiffusionPipeline"] @@ -468,8 +470,10 @@ else: from .auto_pipeline import ( AutoPipelineForImage2Image, + AutoPipelineForImage2Video, AutoPipelineForInpainting, AutoPipelineForText2Image, + AutoPipelineForText2Video, ) from .consistency_models import ConsistencyModelPipeline from .dance_diffusion import DanceDiffusionPipeline diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 6a5f6098b6fb..e188d1742a55 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -20,9 +20,12 @@ from ..configuration_utils import ConfigMixin from ..models.controlnets import ControlNetUnionModel from ..utils import is_sentencepiece_available +from .allegro import AllegroPipeline from .aura_flow import AuraFlowPipeline +from .cogvideo import CogVideoXImageToVideoPipeline, CogVideoXPipeline from .cogview3 import CogView3PlusPipeline from .cogview4 import CogView4ControlPipeline, CogView4Pipeline +from .consisid import ConsisIDPipeline from .controlnet import ( StableDiffusionControlNetImg2ImgPipeline, StableDiffusionControlNetInpaintPipeline, @@ -39,6 +42,7 @@ StableDiffusion3ControlNetPipeline, ) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline +from .easyanimate import EasyAnimatePipeline from .flux import ( FluxControlImg2ImgPipeline, FluxControlInpaintPipeline, @@ -50,6 +54,7 @@ FluxInpaintPipeline, FluxPipeline, ) +from .hunyuan_video import HunyuanVideoImageToVideoPipeline, HunyuanVideoPipeline from .hunyuandit import HunyuanDiTPipeline from .kandinsky import ( KandinskyCombinedPipeline, @@ -69,8 +74,10 @@ ) from .kandinsky3 import Kandinsky3Img2ImgPipeline, Kandinsky3Pipeline from .latent_consistency_models import LatentConsistencyModelImg2ImgPipeline, LatentConsistencyModelPipeline +from .ltx import LTXImageToVideoPipeline, LTXPipeline from .lumina import LuminaPipeline from .lumina2 import Lumina2Pipeline +from .mochi import MochiPipeline from .pag import ( HunyuanDiTPAGPipeline, PixArtSigmaPAGPipeline, @@ -106,6 +113,9 @@ StableDiffusionXLInpaintPipeline, StableDiffusionXLPipeline, ) +from .stable_video_diffusion import StableVideoDiffusionPipeline +from .text_to_video_synthesis import TextToVideoSDPipeline, VideoToVideoSDPipeline +from .wan import WanImageToVideoPipeline, WanPipeline from .wuerstchen import WuerstchenCombinedPipeline, WuerstchenDecoderPipeline @@ -214,6 +224,33 @@ ] ) + +AUTO_TEXT2VIDEO_PIPELINES_MAPPING = OrderedDict( + [ + ("allegro", AllegroPipeline), + ("cogvideox", CogVideoXPipeline), + ("consisid", ConsisIDPipeline), + ("easyanimate", EasyAnimatePipeline), + ("hunyuan-video", HunyuanVideoPipeline), + ("ltx", LTXPipeline), + ("mochi", MochiPipeline), + ("svd", StableVideoDiffusionPipeline), + ("t2v-sd", TextToVideoSDPipeline), + ("wan", WanPipeline), + ] +) + +AUTO_IMAGE2VIDEO_PIPELINES_MAPPING = OrderedDict( + [ + ("cogvideox", CogVideoXImageToVideoPipeline), + ("hunyuan-video", HunyuanVideoImageToVideoPipeline), + ("ltx", LTXImageToVideoPipeline), + ("t2v-sd", VideoToVideoSDPipeline), + ("wan", WanImageToVideoPipeline), + ] +) + + if is_sentencepiece_available(): from .kolors import KolorsImg2ImgPipeline, KolorsPipeline from .pag import KolorsPAGPipeline @@ -226,6 +263,8 @@ AUTO_TEXT2IMAGE_PIPELINES_MAPPING, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, AUTO_INPAINT_PIPELINES_MAPPING, + AUTO_TEXT2VIDEO_PIPELINES_MAPPING, + AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, _AUTO_TEXT2IMAGE_DECODER_PIPELINES_MAPPING, _AUTO_IMAGE2IMAGE_DECODER_PIPELINES_MAPPING, _AUTO_INPAINT_DECODER_PIPELINES_MAPPING, @@ -244,6 +283,14 @@ def _get_connected_pipeline(pipeline_cls): ) if pipeline_cls in _AUTO_INPAINT_DECODER_PIPELINES_MAPPING.values(): return _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False) + if pipeline_cls in AUTO_TEXT2VIDEO_PIPELINES_MAPPING.values(): + return _get_task_class( + AUTO_TEXT2VIDEO_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False + ) + if pipeline_cls in AUTO_IMAGE2VIDEO_PIPELINES_MAPPING.values(): + return _get_task_class( + AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False + ) def _get_task_class(mapping, pipeline_class_name, throw_error_if_not_exist: bool = True): @@ -1174,3 +1221,516 @@ def from_pipe(cls, pipeline, **kwargs): model.register_to_config(**unused_original_config) return model + + +class AutoPipelineForText2Video(ConfigMixin): + r""" + + [`AutoPipelineForText2Video`] is a generic pipeline class that instantiates a text-to-video pipeline class. The + specific underlying pipeline class is automatically selected from either the + [`~AutoPipelineForText2Video.from_pretrained`] or [`~AutoPipelineForText2Video.from_pipe`] methods. + + This class cannot be instantiated using `__init__()` (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ + + config_name = "model_index.json" + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a text-to-video Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the text-to-video pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetPipeline`] object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at stable-diffusion-v1-5/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2vm-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForText2Video + + >>> pipeline = AutoPipelineForText2Video.from_pretrained("Lightricks/LTX-Video") + >>> video = pipeline(prompt).videos[0] + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + + config = cls.load_config(pretrained_model_or_path, **load_config_kwargs) + orig_class_name = config["_class_name"] + + text_2_video_cls = _get_task_class(AUTO_TEXT2VIDEO_PIPELINES_MAPPING, orig_class_name) + + kwargs = {**load_config_kwargs, **kwargs} + return text_2_video_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a text-to-video Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by finding the text-to-video + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memory. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + ```py + >>> from diffusers import AutoPipelineForText2Video, AutoPipelineForImage2Video + + >>> pipe_i2v = AutoPipelineForImage2Video.from_pretrained( + ... "Lightricks/LTX-Video" + ... ) + + >>> pipe_t2v = AutoPipelineForText2Video.from_pipe(pipe_i2v) + >>> video = pipe_t2v(prompt).videos[0] + ``` + """ + + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + text_2_video_cls = _get_task_class(AUTO_TEXT2VIDEO_PIPELINES_MAPPING, original_cls_name) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = text_2_video_cls._get_signature_keys(text_2_video_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to override the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to override the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + # config that were not expected by original pipeline is stored as private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + text_2_video_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in text_2_video_kwargs + } + + missing_modules = ( + set(expected_modules) - set(text_2_video_cls._optional_components) - set(text_2_video_kwargs.keys()) + ) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {text_2_video_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed" + ) + + model = text_2_video_cls(**text_2_video_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) + + return model + + +class AutoPipelineForImage2Video(ConfigMixin): + r""" + + [`AutoPipelineForImage2Video`] is a generic pipeline class that instantiates an image-to-video pipeline class. The + specific underlying pipeline class is automatically selected from either the + [`~AutoPipelineForImage2Video.from_pretrained`] or [`~AutoPipelineForImage2Video.from_pipe`] methods. + + This class cannot be instantiated using `__init__()` (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ + + config_name = "model_index.json" + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a image-to-video Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the image-to-video pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetImg2ImgPipeline`] + object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at stable-diffusion-v1-5/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2vm-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForImage2Video + + >>> pipeline = AutoPipelineForImage2Video.from_pretrained("Lightricks/LTX-Video") + >>> video = pipeline(prompt, image).videos[0] + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + + config = cls.load_config(pretrained_model_or_path, **load_config_kwargs) + orig_class_name = config["_class_name"] + + image_2_video_cls = _get_task_class(AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, orig_class_name) + + kwargs = {**load_config_kwargs, **kwargs} + return image_2_video_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a image-to-video Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by finding the + image-to-video pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memory. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + Examples: + + ```py + >>> from diffusers import AutoPipelineForText2vmage, AutoPipelineForImage2Video + + >>> pipe_t2v = AutoPipelineForText2vmage.from_pretrained( + ... "Lightricks/LTX-Video" + ... ) + + >>> pipe_i2v = AutoPipelineForImage2Video.from_pipe(pipe_t2v) + >>> video = pipe_i2v(prompt, image).videos[0] + ``` + """ + + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + image_2_video_cls = _get_task_class(AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, original_cls_name) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = image_2_video_cls._get_signature_keys(image_2_video_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to override the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to override the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + # config attribute that were not expected by original pipeline is stored as its private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + image_2_video_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in image_2_video_kwargs + } + + missing_modules = ( + set(expected_modules) - set(image_2_video_cls._optional_components) - set(image_2_video_kwargs.keys()) + ) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {image_2_video_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed" + ) + + model = image_2_video_cls(**image_2_video_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) + + return model diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index 561a9011c6ae..84327a19c440 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -25,8 +25,10 @@ from diffusers import ( AutoPipelineForImage2Image, + AutoPipelineForImage2Video, AutoPipelineForInpainting, AutoPipelineForText2Image, + AutoPipelineForText2Video, ControlNetModel, DiffusionPipeline, ) @@ -454,6 +456,32 @@ def test_from_pipe_optional_components(self): pipe = AutoPipelineForText2Image.from_pipe(pipe, image_encoder=None) assert pipe.image_encoder is None + def test_from_pretrained_text2video(self): + repo = "hf-internal-testing/tiny-cogvideox-pipe" + pipe = AutoPipelineForText2Video.from_pretrained(repo) + assert pipe.__class__.__name__ == "CogVideoXPipeline" + + def test_from_pretrained_image2video(self): + repo = "hf-internal-testing/tiny-cogvideox-pipe" + pipe = AutoPipelineForImage2Video.from_pretrained(repo) + assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline" + + def test_from_pipe_text2video_image2video(self): + repo = "hf-internal-testing/tiny-cogvideox-pipe" + pipe = AutoPipelineForText2Video.from_pretrained(repo) + assert pipe.__class__.__name__ == "CogVideoXPipeline" + + pipe = AutoPipelineForImage2Video.from_pipe(pipe) + assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline" + + def test_from_pipe_image2video_text2video(self): + repo = "hf-internal-testing/tiny-cogvideox-pipe" + pipe = AutoPipelineForImage2Video.from_pretrained(repo) + assert pipe.__class__.__name__ == "CogVideoXImageToVideoPipeline" + + pipe = AutoPipelineForText2Video.from_pipe(pipe) + assert pipe.__class__.__name__ == "CogVideoXPipeline" + @slow class AutoPipelineIntegrationTest(unittest.TestCase): From 6cf40e1ceaf79b7889ff8ce655750f382d290569 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Apr 2025 07:10:11 +0000 Subject: [PATCH 2/3] Apply style fixes --- src/diffusers/__init__.py | 2 +- src/diffusers/pipelines/auto_pipeline.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 817631a8c6b6..dad85f2b4382 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -227,10 +227,10 @@ [ "AudioPipelineOutput", "AutoPipelineForImage2Image", + "AutoPipelineForImage2Video", "AutoPipelineForInpainting", "AutoPipelineForText2Image", "AutoPipelineForText2Video", - "AutoPipelineForImage2Video", "ConsistencyModelPipeline", "DanceDiffusionPipeline", "DDIMPipeline", diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index e188d1742a55..428580f6adcd 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -1407,9 +1407,7 @@ def from_pipe(cls, pipeline, **kwargs): ```py >>> from diffusers import AutoPipelineForText2Video, AutoPipelineForImage2Video - >>> pipe_i2v = AutoPipelineForImage2Video.from_pretrained( - ... "Lightricks/LTX-Video" - ... ) + >>> pipe_i2v = AutoPipelineForImage2Video.from_pretrained("Lightricks/LTX-Video") >>> pipe_t2v = AutoPipelineForText2Video.from_pipe(pipe_i2v) >>> video = pipe_t2v(prompt).videos[0] @@ -1665,9 +1663,7 @@ def from_pipe(cls, pipeline, **kwargs): ```py >>> from diffusers import AutoPipelineForText2vmage, AutoPipelineForImage2Video - >>> pipe_t2v = AutoPipelineForText2vmage.from_pretrained( - ... "Lightricks/LTX-Video" - ... ) + >>> pipe_t2v = AutoPipelineForText2vmage.from_pretrained("Lightricks/LTX-Video") >>> pipe_i2v = AutoPipelineForImage2Video.from_pipe(pipe_t2v) >>> video = pipe_i2v(prompt, image).videos[0] From a62cd940131833f7e60b074223f4ead707045baa Mon Sep 17 00:00:00 2001 From: hlky Date: Tue, 15 Apr 2025 08:13:51 +0100 Subject: [PATCH 3/3] fix-copies --- src/diffusers/utils/dummy_pt_objects.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/diffusers/utils/dummy_pt_objects.py b/src/diffusers/utils/dummy_pt_objects.py index bf2f19ee2d26..3cc6b41a67dc 100644 --- a/src/diffusers/utils/dummy_pt_objects.py +++ b/src/diffusers/utils/dummy_pt_objects.py @@ -1163,6 +1163,21 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class AutoPipelineForImage2Video(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class AutoPipelineForInpainting(metaclass=DummyObject): _backends = ["torch"] @@ -1193,6 +1208,21 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class AutoPipelineForText2Video(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class BlipDiffusionControlNetPipeline(metaclass=DummyObject): _backends = ["torch"]