update to to not assume pipeline has hf_device_map

Author: yiyixuxu

src/diffusers/pipelines/modular_pipeline.py

@@ -966,7 +966,7 @@ def module_is_offloaded(module):
                 "It seems like you have activated sequential model offloading by calling `enable_sequential_cpu_offload`, but are now attempting to move the pipeline to GPU. This is not compatible with offloading. Please, move your pipeline `.to('cpu')` or consider removing the move altogether if you use sequential offloading."
             )
 
-        is_pipeline_device_mapped = self.hf_device_map is not None and len(self.hf_device_map) > 1
+        is_pipeline_device_mapped = hasattr(self, "hf_device_map") and self.hf_device_map is not None and len(self.hf_device_map) > 1
         if is_pipeline_device_mapped:
             raise ValueError(
                 "It seems like you have activated a device mapping strategy on the pipeline which doesn't allow explicit device placement using `to()`. You can call `reset_device_map()` first and then call `to()`."

src/diffusers/pipelines/pipeline_utils.py

@@ -422,7 +422,7 @@ def module_is_offloaded(module):
                     "You are trying to call `.to('cuda')` on a pipeline that has models quantized with `bitsandbytes`. Your current `accelerate` installation does not support it. Please upgrade the installation."
                 )
 
-        is_pipeline_device_mapped = self.hf_device_map is not None and len(self.hf_device_map) > 1
+        is_pipeline_device_mapped = hasattr(self, "hf_device_map") and self.hf_device_map is not None and len(self.hf_device_map) > 1
         if is_pipeline_device_mapped:
             raise ValueError(
                 "It seems like you have activated a device mapping strategy on the pipeline which doesn't allow explicit device placement using `to()`. You can call `reset_device_map()` first and then call `to()`."
@@ -1030,7 +1030,7 @@ def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[t
                 The PyTorch device type of the accelerator that shall be used in inference. If not specified, it will
                 default to "cuda".
         """
-        is_pipeline_device_mapped = self.hf_device_map is not None and len(self.hf_device_map) > 1
+        is_pipeline_device_mapped = hasattr(self, "hf_device_map") and self.hf_device_map is not None and len(self.hf_device_map) > 1
         if is_pipeline_device_mapped:
             raise ValueError(
                 "It seems like you have activated a device mapping strategy on the pipeline so calling `enable_model_cpu_offload() isn't allowed. You can call `reset_device_map()` first and then call `enable_model_cpu_offload()`."
@@ -1138,7 +1138,7 @@ def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Un
             raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher")
         self.remove_all_hooks()
 
-        is_pipeline_device_mapped = self.hf_device_map is not None and len(self.hf_device_map) > 1
+        is_pipeline_device_mapped = hasattr(self, "hf_device_map") and self.hf_device_map is not None and len(self.hf_device_map) > 1
         if is_pipeline_device_mapped:
             raise ValueError(
                 "It seems like you have activated a device mapping strategy on the pipeline so calling `enable_sequential_cpu_offload() isn't allowed. You can call `reset_device_map()` first and then call `enable_sequential_cpu_offload()`."

src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_modular.py

@@ -544,80 +544,88 @@ def __call__(self, pipeline, state: PipelineState) -> PipelineState:
         return pipeline, state
 
 
-class StableDiffusionXLInpaintPrepareLatentsStep(PipelineBlock):
-    expected_components = ["vae", "scheduler"]
-    model_name = "stable-diffusion-xl"
-
-    @property
-    def inputs(self) -> List[Tuple[str, Any]]:
-        return [
-            ("height", None),
-            ("width", None),
-            ("generator", None),
-            ("latents", None),
-            ("num_images_per_prompt", 1),
-            ("device", None),
-            ("dtype", None),
-            ("image", None),
-            ("denoising_start", None),
-        ]
-
-    @property
-    def intermediates_inputs(self) -> List[str]:
-        return ["batch_size", "latent_timestep", "prompt_embeds"]
-
-    @property
-    def intermediates_outputs(self) -> List[str]:
-        return ["latents"]
-
-    def __init__(self):
-        super().__init__()
-        self.auxiliaries["image_processor"] = VaeImageProcessor()
-        self.components["vae"] = None
-        self.components["scheduler"] = None
-
-    @torch.no_grad()
-    def __call__(self, pipeline: DiffusionPipeline, state: PipelineState) -> PipelineState:
-        latents = state.get_input("latents")
-        num_images_per_prompt = state.get_input("num_images_per_prompt")
-        generator = state.get_input("generator")
-        device = state.get_input("device")
-        dtype = state.get_input("dtype")
-
-        # image to image only
-        image = state.get_input("image")
-        denoising_start = state.get_input("denoising_start")
-
-        batch_size = state.get_intermediate("batch_size")
-        prompt_embeds = state.get_intermediate("prompt_embeds")
-        # image to image only
-        latent_timestep = state.get_intermediate("latent_timestep")
-
-        if dtype is None and prompt_embeds is not None:
-            dtype = prompt_embeds.dtype
-        elif dtype is None:
-            dtype = pipeline.vae.dtype
-
-        if device is None:
-            device = pipeline._execution_device
-
-        image = pipeline.image_processor.preprocess(image)
-        add_noise = True if denoising_start is None else False
-        if latents is None:
-            latents = pipeline.prepare_latents_img2img(
-                image,
-                latent_timestep,
-                batch_size,
-                num_images_per_prompt,
-                dtype,
-                device,
-                generator,
-                add_noise,
-            )
-
-        state.add_intermediate("latents", latents)
-
-        return pipeline, state
+# class StableDiffusionXLInpaintPrepareLatentsStep(PipelineBlock):
+#     expected_components = ["vae", "scheduler"]
+#     model_name = "stable-diffusion-xl"
+
+#     @property
+#     def inputs(self) -> List[Tuple[str, Any]]:
+#         return [
+#             ("height", None),
+#             ("width", None),
+#             ("generator", None),
+#             ("latents", None),
+#             ("num_images_per_prompt", 1),
+#             ("device", None),
+#             ("dtype", None),
+#             ("image", None),
+#             ("denoising_start", None),
+#         ]
+
+#     @property
+#     def intermediates_inputs(self) -> List[str]:
+#         return ["batch_size", "latent_timestep", "prompt_embeds"]
+
+#     @property
+#     def intermediates_outputs(self) -> List[str]:
+#         return ["latents"]
+
+#     def __init__(self):
+#         super().__init__()
+#         self.auxiliaries["image_processor"] = VaeImageProcessor()
+#         self.components["vae"] = None
+#         self.components["scheduler"] = None
+
+#     @torch.no_grad()
+#     def __call__(self, pipeline: DiffusionPipeline, state: PipelineState) -> PipelineState:
+#         latents = state.get_input("latents")
+#         num_images_per_prompt = state.get_input("num_images_per_prompt")
+#         generator = state.get_input("generator")
+#         device = state.get_input("device")
+#         dtype = state.get_input("dtype")
+
+#         # image to image only
+#         image = state.get_input("image")
+#         denoising_start = state.get_input("denoising_start")
+
+#         # inpaint only
+#         strength = state.get_input("strength")
+#         padding_mask_crop = state.get_input("padding_mask_crop")
+#         mask_image = state.get_input("mask_image")
+#         masked_image_latents = state.get_input("masked_image_latents")
+
+
+
+#         batch_size = state.get_intermediate("batch_size")
+#         prompt_embeds = state.get_intermediate("prompt_embeds")
+#         # image to image only
+#         latent_timestep = state.get_intermediate("latent_timestep")
+
+#         if dtype is None and prompt_embeds is not None:
+#             dtype = prompt_embeds.dtype
+#         elif dtype is None:
+#             dtype = pipeline.vae.dtype
+
+#         if device is None:
+#             device = pipeline._execution_device
+
+#         image = pipeline.image_processor.preprocess(image)
+#         add_noise = True if denoising_start is None else False
+#         if latents is None:
+#             latents = pipeline.prepare_latents_img2img(
+#                 image,
+#                 latent_timestep,
+#                 batch_size,
+#                 num_images_per_prompt,
+#                 dtype,
+#                 device,
+#                 generator,
+#                 add_noise,
+#             )
+
+#         state.add_intermediate("latents", latents)
+
+#         return pipeline, state
 
 
 class StableDiffusionXLImg2ImgPrepareLatentsStep(PipelineBlock):