[bugfix] fix bug in diffuers/pipeline/_init_ (mindspore-lab#982)

* bugfix

* bugfix

Author: LiTingyu1997

mindone/diffusers/__init__.py

@@ -525,6 +525,7 @@
         StableUnCLIPPipeline,
         StableVideoDiffusionPipeline,
         TextToVideoSDPipeline,
+        TextToVideoZeroPipeline,
         TextToVideoZeroSDXLPipeline,
         UnCLIPImageVariationPipeline,
         UnCLIPPipeline,
@@ -535,7 +536,6 @@
         WuerstchenCombinedPipeline,
         WuerstchenDecoderPipeline,
         WuerstchenPriorPipeline,
-        TextToVideoZeroPipeline,
     )
     from .schedulers import (
         AmusedScheduler,

mindone/diffusers/pipelines/__init__.py

@@ -199,9 +199,10 @@
         "TextToVideoSDPipeline",
         "TextToVideoZeroSDXLPipeline",
         "VideoToVideoSDPipeline",
+        "TextToVideoZeroPipeline",
     ],
     "unclip": ["UnCLIPImageVariationPipeline", "UnCLIPPipeline"],
-    ["unidiffuser"]: [
+    "unidiffuser": [
         "ImageTextPipelineOutput",
         "UniDiffuserModel",
         "UniDiffuserPipeline",
@@ -212,7 +213,6 @@
         "WuerstchenDecoderPipeline",
         "WuerstchenPriorPipeline",
     ],
-    "text_to_video_synthesis" : ["TextToVideoZeroPipeline"],
     "pipeline_utils": [
         "AudioPipelineOutput",
         "DiffusionPipeline",
@@ -383,11 +383,15 @@
     )
     from .stable_video_diffusion import StableVideoDiffusionPipeline
     from .t2i_adapter import StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline
-    from .text_to_video_synthesis import TextToVideoSDPipeline, TextToVideoZeroSDXLPipeline, VideoToVideoSDPipeline
+    from .text_to_video_synthesis import (
+        TextToVideoSDPipeline,
+        TextToVideoZeroPipeline,
+        TextToVideoZeroSDXLPipeline,
+        VideoToVideoSDPipeline,
+    )
     from .unclip import UnCLIPImageVariationPipeline, UnCLIPPipeline
     from .unidiffuser import ImageTextPipelineOutput, UniDiffuserModel, UniDiffuserPipeline, UniDiffuserTextDecoder
     from .wuerstchen import WuerstchenCombinedPipeline, WuerstchenDecoderPipeline, WuerstchenPriorPipeline
-    from .text_to_video_synthesis import TextToVideoZeroPipeline
 else:
     import sys
 

mindone/diffusers/pipelines/text_to_video_synthesis/__init__.py

@@ -2,7 +2,6 @@
 
 from ...utils import _LazyModule
 
-
 _import_structure = {}
 
 _import_structure["pipeline_output"] = ["TextToVideoSDPipelineOutput"]

mindone/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py

@@ -3,11 +3,10 @@
 
 import numpy as np
 import PIL
+
 import mindspore as ms
 
-from ...utils import (
-    BaseOutput,
-)
+from ...utils import BaseOutput
 
 
 @dataclass

mindone/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py

@@ -1,13 +1,14 @@
 import copy
 import inspect
 from dataclasses import dataclass
-from typing import Callable, List, Optional, Union, Tuple
+from typing import Callable, List, Optional, Union
 
 import numpy as np
 import PIL.Image
+from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
+
 import mindspore as ms
 from mindspore import mint, ops
-from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
 
 from ...image_processor import VaeImageProcessor
 from ...loaders import StableDiffusionLoraLoaderMixin, TextualInversionLoaderMixin
@@ -18,7 +19,6 @@
 from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
 from ..stable_diffusion import StableDiffusionSafetyChecker
 
-
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 
 
@@ -226,7 +226,8 @@ def warp_single_latent(latent, reference_flow):
     # mint.nn.functional.grid_sample not support dtype float16.
     if latent.dtype == ms.float16:
         warped = mint.nn.functional.grid_sample(
-            latent.to(ms.float32), coords_t0.to(ms.float32), mode="nearest", padding_mode="reflection").to(ms.float16)
+            latent.to(ms.float32), coords_t0.to(ms.float32), mode="nearest", padding_mode="reflection"
+        ).to(ms.float16)
     else:
         warped = mint.nn.functional.grid_sample(latent, coords_t0, mode="nearest", padding_mode="reflection")
     return warped
@@ -466,7 +467,8 @@ def check_inputs(
             k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
         ):
             raise ValueError(
-                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found \
+                    {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
             )
 
         if prompt is not None and prompt_embeds is not None:
@@ -712,7 +714,7 @@ def __call__(
 
         self.scheduler = scheduler_copy
         x_1k_0 = self.backward_loop(
-            timesteps=timesteps[-t1 - 1:],
+            timesteps=timesteps[-t1 - 1 :],
             prompt_embeds=prompt_embeds,
             latents=x_1k_t1,
             guidance_scale=guidance_scale,
@@ -846,9 +848,7 @@ def encode_prompt(
             if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not mint.equal(
                 text_input_ids, untruncated_ids
             ):
-                removed_text = self.tokenizer.batch_decode(
-                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
-                )
+                removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1])
                 logger.warning(
                     "The following part of your input was truncated because CLIP can only handle sequences up to"
                     f" {self.tokenizer.model_max_length} tokens: {removed_text}"

tests/diffusers_tests/pipelines/text_to_video_synthesis/test_text_to_video_zero.py

@@ -22,15 +22,15 @@
 @ddt
 @slow
 class StableTextToVideoZeroPipelineIntegrationTests(PipelineTesterMixin, unittest.TestCase):
-
     @data(*test_cases)
     @unpack
     def test_text_to_video_zero(self, mode, dtype):
         ms.set_context(mode=mode)
         ms_dtype = getattr(ms, dtype)
 
-        pipe = TextToVideoZeroPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5",
-                                                       mindspore_dtype=ms_dtype)
+        pipe = TextToVideoZeroPipeline.from_pretrained(
+            "stable-diffusion-v1-5/stable-diffusion-v1-5", mindspore_dtype=ms_dtype
+        )
         pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
 
         prompt = "A bear is playing a guitar on Times Square"