Merge remote-tracking branch 'origin/dduf' into dduf

Author: SunMarc

src/diffusers/models/model_loading_utils.py

@@ -281,7 +281,7 @@ def _fetch_index_file(
     revision,
     user_agent,
     commit_hash,
-    dduf_entries=None,
+    dduf_entries: Optional[Dict[str, DDUFEntry]] = None,
 ):
     if is_local:
         index_file = Path(
@@ -359,7 +359,7 @@ def _fetch_index_file_legacy(
     revision,
     user_agent,
     commit_hash,
-    dduf_entries=None,
+    dduf_entries: Optional[Dict[str, DDUFEntry]] = None,
 ):
     if is_local:
         index_file = Path(

src/diffusers/models/modeling_utils.py

@@ -586,7 +586,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
         variant = kwargs.pop("variant", None)
         use_safetensors = kwargs.pop("use_safetensors", None)
         quantization_config = kwargs.pop("quantization_config", None)
-        dduf_entries = kwargs.pop("dduf_entries", None)
+        dduf_entries: Optional[Dict[str, DDUFEntry]] = kwargs.pop("dduf_entries", None)
 
         allow_pickle = False
         if use_safetensors is None:

src/diffusers/pipelines/pipeline_utils.py

@@ -1290,7 +1290,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]:
         use_onnx = kwargs.pop("use_onnx", None)
         load_connected_pipeline = kwargs.pop("load_connected_pipeline", False)
         trust_remote_code = kwargs.pop("trust_remote_code", False)
-        dduf_file = kwargs.pop("dduf_file", None)
+        dduf_file: Optional[Dict[str, DDUFEntry]] = kwargs.pop("dduf_file", None)
 
         allow_pickle = False
         if use_safetensors is None:
@@ -1309,11 +1309,11 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]:
                 local_files_only = True
                 model_info_call_error = e  # save error to reraise it if model is not cached locally
 
-        if dduf_file is not None and not local_files_only:
-            dduf_available = False
-            for sibling in info.siblings:
-                dduf_available = dduf_file in sibling.rfilename
-            if not dduf_available:
+        if (
+            not local_files_only
+            and dduf_file is not None
+            and dduf_file not in (sibling.rfilename for sibling in info.siblings)
+            ):  
                 raise ValueError(f"Requested {dduf_file} file is not available in {pretrained_model_name}.")
 
         if not local_files_only and not dduf_file:
@@ -1477,27 +1477,29 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]:
             )
 
             # retrieve pipeline class from local file
-            if not dduf_file:
-                cls_name = cls.load_config(os.path.join(cached_folder, "model_index.json")).get("_class_name", None)
-                cls_name = cls_name[4:] if isinstance(cls_name, str) and cls_name.startswith("Flax") else cls_name
-
-                diffusers_module = importlib.import_module(__name__.split(".")[0])
-                pipeline_class = getattr(diffusers_module, cls_name, None) if isinstance(cls_name, str) else None
-
-                if pipeline_class is not None and pipeline_class._load_connected_pipes:
-                    modelcard = ModelCard.load(os.path.join(cached_folder, "README.md"))
-                    connected_pipes = sum([getattr(modelcard.data, k, []) for k in CONNECTED_PIPES_KEYS], [])
-                    for connected_pipe_repo_id in connected_pipes:
-                        download_kwargs = {
-                            "cache_dir": cache_dir,
-                            "force_download": force_download,
-                            "proxies": proxies,
-                            "local_files_only": local_files_only,
-                            "token": token,
-                            "variant": variant,
-                            "use_safetensors": use_safetensors,
-                        }
-                        DiffusionPipeline.download(connected_pipe_repo_id, **download_kwargs)
+            if dduf_file:
+                return cached_folder
+
+            cls_name = cls.load_config(os.path.join(cached_folder, "model_index.json")).get("_class_name", None)
+            cls_name = cls_name[4:] if isinstance(cls_name, str) and cls_name.startswith("Flax") else cls_name
+
+            diffusers_module = importlib.import_module(__name__.split(".")[0])
+            pipeline_class = getattr(diffusers_module, cls_name, None) if isinstance(cls_name, str) else None
+
+            if pipeline_class is not None and pipeline_class._load_connected_pipes:
+                modelcard = ModelCard.load(os.path.join(cached_folder, "README.md"))
+                connected_pipes = sum([getattr(modelcard.data, k, []) for k in CONNECTED_PIPES_KEYS], [])
+                for connected_pipe_repo_id in connected_pipes:
+                    download_kwargs = {
+                        "cache_dir": cache_dir,
+                        "force_download": force_download,
+                        "proxies": proxies,
+                        "local_files_only": local_files_only,
+                        "token": token,
+                        "variant": variant,
+                        "use_safetensors": use_safetensors,
+                    }
+                    DiffusionPipeline.download(connected_pipe_repo_id, **download_kwargs)
 
             return cached_folder
 

src/diffusers/utils/__init__.py

@@ -35,7 +35,6 @@
     WEIGHTS_INDEX_NAME,
     WEIGHTS_NAME,
 )
-from .dduf import DDUFReader
 from .deprecation_utils import deprecate
 from .doc_utils import replace_example_docstring
 from .dynamic_modules_utils import get_class_from_dynamic_module
@@ -68,7 +67,6 @@
     is_flax_available,
     is_ftfy_available,
     is_google_colab,
-    is_huggingface_hub_version,
     is_inflect_available,
     is_invisible_watermark_available,
     is_k_diffusion_available,

src/diffusers/utils/import_utils.py

@@ -317,15 +317,6 @@
         _timm_available = False
 
 
-_huggingface_hub_available = importlib.util.find_spec("huggingface_hub") is not None
-if _huggingface_hub_available:
-    try:
-        _huggingface_hub_version = importlib_metadata.version("huggingface_hub")
-        logger.info(f"huggingface_hub version {_huggingface_hub_version} available.")
-    except importlib_metadata.PackageNotFoundError:
-        _huggingface_hub_available = False
-
-
 def is_timm_available():
     return _timm_available
 
@@ -798,21 +789,6 @@ def is_k_diffusion_version(operation: str, version: str):
     return compare_versions(parse(_k_diffusion_version), operation, version)
 
 
-def is_huggingface_hub_version(operation: str, version: str):
-    """
-    Compares the current huggingface_hub version to a given reference with an operation.
-
-    Args:
-        operation (`str`):
-            A string representation of an operator, such as `">"` or `"<="`
-        version (`str`):
-            A version string
-    """
-    if not _huggingface_hub_available:
-        return False
-    return compare_versions(parse(_huggingface_hub_version), operation, version)
-
-
 def get_objects_from_module(module):
     """
     Returns a dict of object names and values in a module, while skipping private/internal objects