|
| 1 | +# Copyright 2024 The HuggingFace Team. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +import unittest |
| 16 | + |
| 17 | +import numpy as np |
| 18 | +import torch |
| 19 | +from PIL import Image |
| 20 | +from transformers import AutoTokenizer, T5EncoderModel, CLIPVisionConfig, CLIPVisionModel, CLIPImageProcessor |
| 21 | + |
| 22 | +from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanI2VPipeline, WanTransformer3DModel |
| 23 | +from diffusers.utils.testing_utils import enable_full_determinism |
| 24 | + |
| 25 | +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS |
| 26 | +from ..test_pipelines_common import PipelineTesterMixin |
| 27 | + |
| 28 | + |
| 29 | +enable_full_determinism() |
| 30 | + |
| 31 | + |
| 32 | +class WanImageToVideoPipelineFastTests(PipelineTesterMixin, unittest.TestCase): |
| 33 | + pipeline_class = WanI2VPipeline |
| 34 | + params = TEXT_TO_IMAGE_PARAMS - {"cross_attention_kwargs", "height", "width"} |
| 35 | + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS |
| 36 | + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS |
| 37 | + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS |
| 38 | + required_optional_params = frozenset( |
| 39 | + [ |
| 40 | + "num_inference_steps", |
| 41 | + "generator", |
| 42 | + "latents", |
| 43 | + "return_dict", |
| 44 | + "callback_on_step_end", |
| 45 | + "callback_on_step_end_tensor_inputs", |
| 46 | + ] |
| 47 | + ) |
| 48 | + test_xformers_attention = False |
| 49 | + supports_dduf = False |
| 50 | + |
| 51 | + def get_dummy_components(self): |
| 52 | + torch.manual_seed(0) |
| 53 | + vae = AutoencoderKLWan( |
| 54 | + base_dim=3, |
| 55 | + z_dim=16, |
| 56 | + dim_mult=[1, 1, 1, 1], |
| 57 | + num_res_blocks=1, |
| 58 | + temperal_downsample=[False, True, True], |
| 59 | + ) |
| 60 | + |
| 61 | + torch.manual_seed(0) |
| 62 | + # TODO: impl FlowDPMSolverMultistepScheduler |
| 63 | + scheduler = FlowMatchEulerDiscreteScheduler(shift=7.0) |
| 64 | + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") |
| 65 | + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") |
| 66 | + |
| 67 | + torch.manual_seed(0) |
| 68 | + transformer = WanTransformer3DModel( |
| 69 | + patch_size=(1, 2, 2), |
| 70 | + num_attention_heads=2, |
| 71 | + attention_head_dim=12, |
| 72 | + in_channels=36, |
| 73 | + out_channels=16, |
| 74 | + text_dim=32, |
| 75 | + freq_dim=256, |
| 76 | + ffn_dim=32, |
| 77 | + num_layers=2, |
| 78 | + cross_attn_norm=True, |
| 79 | + qk_norm="rms_norm_across_heads", |
| 80 | + rope_max_seq_len=32, |
| 81 | + image_embedding_dim=4, |
| 82 | + ) |
| 83 | + |
| 84 | + torch.manual_seed(0) |
| 85 | + image_encoder_config = CLIPVisionConfig( |
| 86 | + hidden_size=4, |
| 87 | + projection_dim=4, |
| 88 | + num_hidden_layers=2, |
| 89 | + num_attention_heads=2, |
| 90 | + image_size=32, |
| 91 | + intermediate_size=16, |
| 92 | + patch_size=1, |
| 93 | + ) |
| 94 | + image_encoder = CLIPVisionModel(image_encoder_config) |
| 95 | + |
| 96 | + torch.manual_seed(0) |
| 97 | + image_processor = CLIPImageProcessor(crop_size=32, size=32) |
| 98 | + |
| 99 | + components = { |
| 100 | + "transformer": transformer, |
| 101 | + "vae": vae, |
| 102 | + "scheduler": scheduler, |
| 103 | + "text_encoder": text_encoder, |
| 104 | + "tokenizer": tokenizer, |
| 105 | + "image_encoder": image_encoder, |
| 106 | + "image_processor": image_processor, |
| 107 | + } |
| 108 | + return components |
| 109 | + |
| 110 | + def get_dummy_inputs(self, device, seed=0): |
| 111 | + if str(device).startswith("mps"): |
| 112 | + generator = torch.manual_seed(seed) |
| 113 | + else: |
| 114 | + generator = torch.Generator(device=device).manual_seed(seed) |
| 115 | + image_height = 16 |
| 116 | + image_width = 16 |
| 117 | + image = Image.new("RGB", (image_width, image_height)) |
| 118 | + inputs = { |
| 119 | + "image": image, |
| 120 | + "prompt": "dance monkey", |
| 121 | + "negative_prompt": "negative", # TODO |
| 122 | + "max_area": 1024, |
| 123 | + "generator": generator, |
| 124 | + "num_inference_steps": 2, |
| 125 | + "guidance_scale": 6.0, |
| 126 | + "num_frames": 9, |
| 127 | + "max_sequence_length": 16, |
| 128 | + "output_type": "pt", |
| 129 | + } |
| 130 | + return inputs |
| 131 | + |
| 132 | + def test_inference(self): |
| 133 | + device = "cpu" |
| 134 | + |
| 135 | + components = self.get_dummy_components() |
| 136 | + pipe = self.pipeline_class(**components) |
| 137 | + pipe.to(device) |
| 138 | + pipe.set_progress_bar_config(disable=None) |
| 139 | + |
| 140 | + inputs = self.get_dummy_inputs(device) |
| 141 | + video = pipe(**inputs).frames |
| 142 | + generated_video = video[0] |
| 143 | + |
| 144 | + self.assertEqual(generated_video.shape, (9, 3, 32, 32)) |
| 145 | + expected_video = torch.randn(9, 3, 32, 32) |
| 146 | + max_diff = np.abs(generated_video - expected_video).max() |
| 147 | + self.assertLessEqual(max_diff, 1e10) |
| 148 | + |
| 149 | + @unittest.skip("Test not supported") |
| 150 | + def test_attention_slicing_forward_pass(self): |
| 151 | + pass |
0 commit comments