Apply suggestions from code review

Author: hlky

src/diffusers/loaders/single_file_model.py

@@ -266,9 +266,6 @@ def from_single_file(cls, pretrained_model_link_or_path_or_dict: Optional[str] =
                 f"Passed `torch_dtype` {torch_dtype} is not a `torch.dtype`. Defaulting to `torch.float32`."
             )
 
-        if quantization_config is not None and torch_dtype is not None:
-            torch_dtype = None
-
         if isinstance(pretrained_model_link_or_path_or_dict, dict):
             checkpoint = pretrained_model_link_or_path_or_dict
         else:

src/diffusers/models/modeling_utils.py

@@ -889,9 +889,6 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
                 f"Passed `torch_dtype` {torch_dtype} is not a `torch.dtype`. Defaulting to `torch.float32`."
             )
 
-        if quantization_config is not None and torch_dtype is not None:
-            torch_dtype = None
-
         allow_pickle = False
         if use_safetensors is None:
             use_safetensors = True