Enabled cpu offload

Author: guiyrt

src/diffusers/loaders/ip_adapter.py

@@ -378,8 +378,8 @@ def is_ip_adapter_active(self) -> bool:
     def load_ip_adapter(
         self,
         pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]],
-        subfolder: str,
         weight_name: str = "ip-adapter.safetensors",
+        subfolder: Optional[str] = None,
         image_encoder_folder: Optional[str] = "image_encoder",
         **kwargs,
     ) -> None:
@@ -393,12 +393,12 @@ def load_ip_adapter(
                       with [`ModelMixin.save_pretrained`].
                     - A [torch state
                       dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).
-            subfolder (`str`):
-                The subfolder location of a model file within a larger model repository on the Hub or locally. If a
-                list is passed, it should have the same length as `weight_name`.
             weight_name (`str`, defaults to "ip-adapter.safetensors"):
                 The name of the weight file to load. If a list is passed, it should have the same length as
                 `subfolder`.
+            subfolder (`str`, *optional*):
+                The subfolder location of a model file within a larger model repository on the Hub or locally. If a
+                list is passed, it should have the same length as `weight_name`.
             image_encoder_folder (`str`, *optional*, defaults to `image_encoder`):
                 The subfolder location of the image encoder within a larger model repository on the Hub or locally.
                 Pass `None` to not load the image encoder. If the image encoder is located in a folder inside

src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py

@@ -183,6 +183,7 @@ class StableDiffusion3Pipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingle
     """
 
     model_cpu_offload_seq = "text_encoder->text_encoder_2->text_encoder_3->transformer->vae"
+    _exclude_from_cpu_offload = ["image_encoder"]
     _optional_components = ["image_encoder", "feature_extractor"]
     _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds", "negative_pooled_prompt_embeds"]
 
@@ -694,20 +695,22 @@ def interrupt(self):
         return self._interrupt
 
     # Adapted from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_image
-    def encode_image(self, image: PipelineImageInput) -> torch.Tensor:
+    def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch.Tensor:
         """Encodes the given image into a feature representation using a pre-trained image encoder.
 
         Args:
             image (`PipelineImageInput`):
                 Input image to be encoded.
+            device: (`torch.device`):
+                Torch device.
 
         Returns:
             `torch.Tensor`: The encoded image feature representation.
         """
         if not isinstance(image, torch.Tensor):
             image = self.feature_extractor(image, return_tensors="pt").pixel_values
 
-        image = image.to(device=self.device, dtype=self.dtype)
+        image = image.to(device=device, dtype=self.dtype)
 
         return self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
 
@@ -744,7 +747,7 @@ def prepare_ip_adapter_image_embeds(
             else:
                 single_image_embeds = ip_adapter_image_embeds
         elif ip_adapter_image is not None:
-            single_image_embeds = self.encode_image(ip_adapter_image)
+            single_image_embeds = self.encode_image(ip_adapter_image, device)
             if do_classifier_free_guidance:
                 single_negative_image_embeds = torch.zeros_like(single_image_embeds)
         else: