handle dtype casting.

Author: sayakpaul

src/diffusers/models/modeling_utils.py

@@ -963,7 +963,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
             raise ValueError(
                 f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}."
             )
-        elif torch_dtype is not None and hf_quantizer is None:
+        # When using `use_keep_in_fp32_modules` if we do a global `to()` here, then we will
+        # completely lose the effectivity of `use_keep_in_fp32_modules`. `transformers` does
+        # a global dtype setting (see: https://github.com/huggingface/transformers/blob/fa3f2db5c7405a742fcb8f686d3754f70db00977/src/transformers/modeling_utils.py#L4021),
+        # but this would prevent us from doing things like https://github.com/huggingface/diffusers/pull/9177/.
+        elif torch_dtype is not None and hf_quantizer is None and not use_keep_in_fp32_modules:
             model = model.to(torch_dtype)
 
         if hf_quantizer is not None: