Don't use torch_dtype when quantization_config is set

Author: hlky

src/diffusers/loaders/single_file_model.py

@@ -266,6 +266,9 @@ def from_single_file(cls, pretrained_model_link_or_path_or_dict: Optional[str] =
                 f"Passed `torch_dtype` {torch_dtype} is not a `torch.dtype`. Defaulting to `torch.float32`."
             )
 
+        if quantization_config is not None and torch_dtype is not None:
+            torch_dtype = None
+
         if isinstance(pretrained_model_link_or_path_or_dict, dict):
             checkpoint = pretrained_model_link_or_path_or_dict
         else:

src/diffusers/models/modeling_utils.py

@@ -889,6 +889,9 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
                 f"Passed `torch_dtype` {torch_dtype} is not a `torch.dtype`. Defaulting to `torch.float32`."
             )
 
+        if quantization_config is not None and torch_dtype is not None:
+            torch_dtype = None
+
         allow_pickle = False
         if use_safetensors is None:
             use_safetensors = True