diff --git a/.github/workflows/pr_dependency_test.yml b/.github/workflows/pr_dependency_test.yml index b914d1076190..b297cba9dc8a 100644 --- a/.github/workflows/pr_dependency_test.yml +++ b/.github/workflows/pr_dependency_test.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install -e . diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 3306ebe43ef7..b84d2f8f6d34 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -35,7 +35,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install --upgrade pip @@ -55,7 +55,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install --upgrade pip diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index 6c208ad7cac7..f40f33f29dca 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -36,7 +36,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install --upgrade pip @@ -56,7 +56,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install --upgrade pip diff --git a/.github/workflows/pr_torch_dependency_test.yml b/.github/workflows/pr_torch_dependency_test.yml index 4b6160ff71e2..0870da70fb7b 100644 --- a/.github/workflows/pr_torch_dependency_test.yml +++ b/.github/workflows/pr_torch_dependency_test.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install -e . diff --git a/.github/workflows/pypi_publish.yaml b/.github/workflows/pypi_publish.yaml index dc36b6b024c5..a03db58d51c3 100644 --- a/.github/workflows/pypi_publish.yaml +++ b/.github/workflows/pypi_publish.yaml @@ -47,7 +47,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | diff --git a/scripts/remove_typing_builtin_imports.py b/scripts/remove_typing_builtin_imports.py new file mode 100644 index 000000000000..fcbbbafd555d --- /dev/null +++ b/scripts/remove_typing_builtin_imports.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Remove lower-case built-in generics imported from `typing`. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Iterable, Iterator, Sequence + + +try: + import libcst as cst +except ImportError as exc: # pragma: no cover - dependency guard + raise SystemExit("This script requires `libcst`. Install it via `pip install libcst` and retry.") from exc + + +BUILTIN_TYPING_NAMES = frozenset({"callable", "dict", "frozenset", "list", "set", "tuple", "type"}) + + +class TypingBuiltinImportRemover(cst.CSTTransformer): + def __init__(self) -> None: + self.changed = False + self.removed: list[str] = [] + self.warnings: list[str] = [] + + def leave_ImportFrom(self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom) -> cst.BaseStatement: + module_name = self._module_name(updated_node.module) + if module_name != "typing": + return updated_node + + names = updated_node.names + if isinstance(names, cst.ImportStar): + self.warnings.append("encountered `from typing import *` (skipped)") + return updated_node + + new_aliases = [] + removed_here: list[str] = [] + for alias in names: + if isinstance(alias, cst.ImportStar): + self.warnings.append("encountered `from typing import *` (skipped)") + return updated_node + if not isinstance(alias.name, cst.Name): + new_aliases.append(alias) + continue + imported_name = alias.name.value + if imported_name in BUILTIN_TYPING_NAMES: + removed_here.append(imported_name) + continue + new_aliases.append(alias) + + if not removed_here: + return updated_node + + self.changed = True + self.removed.extend(removed_here) + + if not new_aliases: + return cst.RemoveFromParent() + # Ensure trailing commas are removed. + formatted_aliases = [] + for alias in new_aliases: + if alias.comma is not None and alias is new_aliases[-1]: + formatted_aliases.append(alias.with_changes(comma=None)) + else: + formatted_aliases.append(alias) + + return updated_node.with_changes(names=tuple(formatted_aliases)) + + def _module_name(self, node: cst.BaseExpression | None) -> str | None: + if node is None: + return None + if isinstance(node, cst.Name): + return node.value + if isinstance(node, cst.Attribute): + prefix = self._module_name(node.value) + if prefix is None: + return node.attr.value + return f"{prefix}.{node.attr.value}" + return None + + +def iter_python_files(paths: Iterable[Path]) -> Iterator[Path]: + for path in paths: + if path.is_dir(): + yield from (p for p in path.rglob("*.py") if not p.name.startswith(".")) + yield from (p for p in path.rglob("*.pyi") if not p.name.startswith(".")) + elif path.suffix in {".py", ".pyi"}: + yield path + + +def process_file(path: Path, dry_run: bool) -> tuple[bool, TypingBuiltinImportRemover]: + source = path.read_text(encoding="utf-8") + module = cst.parse_module(source) + transformer = TypingBuiltinImportRemover() + updated = module.visit(transformer) + + if not transformer.changed or source == updated.code: + return False, transformer + + if not dry_run: + path.write_text(updated.code, encoding="utf-8") + return True, transformer + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Remove lower-case built-in generics imported from typing.") + parser.add_argument( + "paths", + nargs="*", + type=Path, + default=[Path("src")], + help="Files or directories to rewrite (default: src).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Only report files that would change without writing them.", + ) + args = parser.parse_args(argv) + + files = sorted(set(iter_python_files(args.paths))) + if not files: + print("No Python files matched the provided paths.", file=sys.stderr) + return 1 + + changed_any = False + for path in files: + changed, transformer = process_file(path, dry_run=args.dry_run) + if changed: + changed_any = True + action = "Would update" if args.dry_run else "Updated" + removed = ", ".join(sorted(set(transformer.removed))) + print(f"{action}: {path} (removed typing imports: {removed})") + for warning in transformer.warnings: + print(f"Warning: {path}: {warning}", file=sys.stderr) + + if not changed_any: + print("No changes needed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.py b/setup.py index 8d346ddfecca..53379b9a188c 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ "pytest", "pytest-timeout", "pytest-xdist", - "python>=3.8.0", + "python>=3.9.0", "ruff==0.9.10", "safetensors>=0.3.1", "sentencepiece>=0.1.91,!=0.1.92", @@ -287,7 +287,7 @@ def run(self): packages=find_packages("src"), package_data={"diffusers": ["py.typed"]}, include_package_data=True, - python_requires=">=3.8.0", + python_requires=">=3.9.0", install_requires=list(install_requires), extras_require=extras, entry_points={"console_scripts": ["diffusers-cli=diffusers.commands.diffusers_cli:main"]}, diff --git a/src/diffusers/callbacks.py b/src/diffusers/callbacks.py index 2a08f091d9f3..087a6b7fee56 100644 --- a/src/diffusers/callbacks.py +++ b/src/diffusers/callbacks.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any from .configuration_utils import ConfigMixin, register_to_config from .utils import CONFIG_NAME @@ -33,13 +33,13 @@ def __init__(self, cutoff_step_ratio=1.0, cutoff_step_index=None): raise ValueError("cutoff_step_ratio must be a float between 0.0 and 1.0.") @property - def tensor_inputs(self) -> List[str]: + def tensor_inputs(self) -> list[str]: raise NotImplementedError(f"You need to set the attribute `tensor_inputs` for {self.__class__}") - def callback_fn(self, pipeline, step_index, timesteps, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timesteps, callback_kwargs) -> dict[str, Any]: raise NotImplementedError(f"You need to implement the method `callback_fn` for {self.__class__}") - def __call__(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def __call__(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: return self.callback_fn(pipeline, step_index, timestep, callback_kwargs) @@ -49,14 +49,14 @@ class MultiPipelineCallbacks: provides a unified interface for calling all of them. """ - def __init__(self, callbacks: List[PipelineCallback]): + def __init__(self, callbacks: list[PipelineCallback]): self.callbacks = callbacks @property - def tensor_inputs(self) -> List[str]: + def tensor_inputs(self) -> list[str]: return [input for callback in self.callbacks for input in callback.tensor_inputs] - def __call__(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def __call__(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: """ Calls all the callbacks in order with the given arguments and returns the final callback_kwargs. """ @@ -76,7 +76,7 @@ class SDCFGCutoffCallback(PipelineCallback): tensor_inputs = ["prompt_embeds"] - def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: cutoff_step_ratio = self.config.cutoff_step_ratio cutoff_step_index = self.config.cutoff_step_index @@ -109,7 +109,7 @@ class SDXLCFGCutoffCallback(PipelineCallback): "add_time_ids", ] - def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: cutoff_step_ratio = self.config.cutoff_step_ratio cutoff_step_index = self.config.cutoff_step_index @@ -152,7 +152,7 @@ class SDXLControlnetCFGCutoffCallback(PipelineCallback): "image", ] - def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: cutoff_step_ratio = self.config.cutoff_step_ratio cutoff_step_index = self.config.cutoff_step_index @@ -195,7 +195,7 @@ class IPAdapterScaleCutoffCallback(PipelineCallback): tensor_inputs = [] - def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: cutoff_step_ratio = self.config.cutoff_step_ratio cutoff_step_index = self.config.cutoff_step_index @@ -219,7 +219,7 @@ class SD3CFGCutoffCallback(PipelineCallback): tensor_inputs = ["prompt_embeds", "pooled_prompt_embeds"] - def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> Dict[str, Any]: + def callback_fn(self, pipeline, step_index, timestep, callback_kwargs) -> dict[str, Any]: cutoff_step_ratio = self.config.cutoff_step_ratio cutoff_step_index = self.config.cutoff_step_index diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 1c4ee33acbfd..269d52ee1b7a 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -24,7 +24,7 @@ import re from collections import OrderedDict from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import numpy as np from huggingface_hub import DDUFEntry, create_repo, hf_hub_download @@ -94,10 +94,10 @@ class ConfigMixin: Class attributes: - **config_name** (`str`) -- A filename under which the config should stored when calling [`~ConfigMixin.save_config`] (should be overridden by parent class). - - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + - **ignore_for_config** (`list[str]`) -- A list of attributes that should not be saved in the config (should be overridden by subclass). - **has_compatibles** (`bool`) -- Whether the class has compatible classes (should be overridden by subclass). - - **_deprecated_kwargs** (`List[str]`) -- Keyword arguments that are deprecated. Note that the `init` function + - **_deprecated_kwargs** (`list[str]`) -- Keyword arguments that are deprecated. Note that the `init` function should only have a `kwargs` argument if at least one argument is deprecated (should be overridden by subclass). """ @@ -143,7 +143,7 @@ def __getattr__(self, name: str) -> Any: raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") - def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_config(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save a configuration object to the directory specified in `save_directory` so that it can be reloaded using the [`~ConfigMixin.from_config`] class method. @@ -155,7 +155,7 @@ def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -189,13 +189,13 @@ def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool @classmethod def from_config( - cls, config: Union[FrozenDict, Dict[str, Any]] = None, return_unused_kwargs=False, **kwargs - ) -> Union[Self, Tuple[Self, Dict[str, Any]]]: + cls, config: FrozenDict | dict[str, Any] = None, return_unused_kwargs=False, **kwargs + ) -> Self | tuple[Self, dict[str, Any]]: r""" Instantiate a Python class from a config dictionary. Parameters: - config (`Dict[str, Any]`): + config (`dict[str, Any]`): A config dictionary from which the Python class is instantiated. Make sure to only load configuration files of compatible classes. return_unused_kwargs (`bool`, *optional*, defaults to `False`): @@ -292,11 +292,11 @@ def get_config_dict(cls, *args, **kwargs): @validate_hf_hub_args def load_config( cls, - pretrained_model_name_or_path: Union[str, os.PathLike], + pretrained_model_name_or_path: str | os.PathLike, return_unused_kwargs=False, return_commit_hash=False, **kwargs, - ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + ) -> tuple[dict[str, Any], dict[str, Any]]: r""" Load a model or scheduler configuration. @@ -315,7 +315,7 @@ def load_config( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -352,7 +352,7 @@ def load_config( _ = kwargs.pop("mirror", None) subfolder = kwargs.pop("subfolder", None) user_agent = kwargs.pop("user_agent", {}) - dduf_entries: Optional[Dict[str, DDUFEntry]] = kwargs.pop("dduf_entries", None) + dduf_entries: Optional[dict[str, DDUFEntry]] = kwargs.pop("dduf_entries", None) user_agent = {**user_agent, "file_type": "config"} user_agent = http_user_agent(user_agent) @@ -563,9 +563,7 @@ def extract_init_dict(cls, config_dict, **kwargs): return init_dict, unused_kwargs, hidden_config_dict @classmethod - def _dict_from_json_file( - cls, json_file: Union[str, os.PathLike], dduf_entries: Optional[Dict[str, DDUFEntry]] = None - ): + def _dict_from_json_file(cls, json_file: str | os.PathLike, dduf_entries: Optional[dict[str, DDUFEntry]] = None): if dduf_entries: text = dduf_entries[json_file].read_text() else: @@ -577,12 +575,12 @@ def __repr__(self): return f"{self.__class__.__name__} {self.to_json_string()}" @property - def config(self) -> Dict[str, Any]: + def config(self) -> dict[str, Any]: """ Returns the config of the class as a frozen dictionary Returns: - `Dict[str, Any]`: Config of the class. + `dict[str, Any]`: Config of the class. """ return self._internal_dict @@ -625,7 +623,7 @@ def to_json_saveable(value): return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" - def to_json_file(self, json_file_path: Union[str, os.PathLike]): + def to_json_file(self, json_file_path: str | os.PathLike): """ Save the configuration instance's parameters to a JSON file. @@ -637,7 +635,7 @@ def to_json_file(self, json_file_path: Union[str, os.PathLike]): writer.write(self.to_json_string()) @classmethod - def _get_config_file_from_dduf(cls, pretrained_model_name_or_path: str, dduf_entries: Dict[str, DDUFEntry]): + def _get_config_file_from_dduf(cls, pretrained_model_name_or_path: str, dduf_entries: dict[str, DDUFEntry]): # paths inside a DDUF file must always be "/" config_file = ( cls.config_name @@ -756,7 +754,7 @@ class LegacyConfigMixin(ConfigMixin): """ @classmethod - def from_config(cls, config: Union[FrozenDict, Dict[str, Any]] = None, return_unused_kwargs=False, **kwargs): + def from_config(cls, config: FrozenDict | dict[str, Any] = None, return_unused_kwargs=False, **kwargs): # To prevent dependency import problem. from .models.model_loading_utils import _fetch_remapped_cls_from_config diff --git a/src/diffusers/dependency_versions_table.py b/src/diffusers/dependency_versions_table.py index 6e5ac630ab08..ec0d98ec399a 100644 --- a/src/diffusers/dependency_versions_table.py +++ b/src/diffusers/dependency_versions_table.py @@ -29,7 +29,7 @@ "pytest": "pytest", "pytest-timeout": "pytest-timeout", "pytest-xdist": "pytest-xdist", - "python": "python>=3.8.0", + "python": "python>=3.9.0", "ruff": "ruff==0.9.10", "safetensors": "safetensors>=0.3.1", "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", diff --git a/src/diffusers/guiders/__init__.py b/src/diffusers/guiders/__init__.py index 23cb7a0a7157..5f04a13d0eea 100644 --- a/src/diffusers/guiders/__init__.py +++ b/src/diffusers/guiders/__init__.py @@ -28,14 +28,14 @@ from .smoothed_energy_guidance import SmoothedEnergyGuidance from .tangential_classifier_free_guidance import TangentialClassifierFreeGuidance - GuiderType = Union[ - AdaptiveProjectedGuidance, - AutoGuidance, - ClassifierFreeGuidance, - ClassifierFreeZeroStarGuidance, - FrequencyDecoupledGuidance, - PerturbedAttentionGuidance, - SkipLayerGuidance, - SmoothedEnergyGuidance, - TangentialClassifierFreeGuidance, - ] + GuiderType = ( + AdaptiveProjectedGuidance + | AutoGuidance + | ClassifierFreeGuidance + | ClassifierFreeZeroStarGuidance + | FrequencyDecoupledGuidance + | PerturbedAttentionGuidance + | SkipLayerGuidance + | SmoothedEnergyGuidance + | TangentialClassifierFreeGuidance + ) diff --git a/src/diffusers/guiders/adaptive_projected_guidance.py b/src/diffusers/guiders/adaptive_projected_guidance.py index 92b1fd5a1c2c..afd15ac2666a 100644 --- a/src/diffusers/guiders/adaptive_projected_guidance.py +++ b/src/diffusers/guiders/adaptive_projected_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -77,8 +77,8 @@ def __init__( self.momentum_buffer = None def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/auto_guidance.py b/src/diffusers/guiders/auto_guidance.py index 5271a530ea7a..7cfedf83c991 100644 --- a/src/diffusers/guiders/auto_guidance.py +++ b/src/diffusers/guiders/auto_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional import torch @@ -36,10 +36,10 @@ class AutoGuidance(BaseGuidance): The scale parameter for classifier-free guidance. Higher values result in stronger conditioning on the text prompt, while lower values allow for more freedom in generation. Higher values may lead to saturation and deterioration of image quality. - auto_guidance_layers (`int` or `List[int]`, *optional*): + auto_guidance_layers (`int` or `list[int]`, *optional*): The layer indices to apply skip layer guidance to. Can be a single integer or a list of integers. If not provided, `skip_layer_config` must be provided. - auto_guidance_config (`LayerSkipConfig` or `List[LayerSkipConfig]`, *optional*): + auto_guidance_config (`LayerSkipConfig` or `list[LayerSkipConfig]`, *optional*): The configuration for the skip layer guidance. Can be a single `LayerSkipConfig` or a list of `LayerSkipConfig`. If not provided, `skip_layer_guidance_layers` must be provided. dropout (`float`, *optional*): @@ -65,8 +65,8 @@ class AutoGuidance(BaseGuidance): def __init__( self, guidance_scale: float = 7.5, - auto_guidance_layers: Optional[Union[int, List[int]]] = None, - auto_guidance_config: Union[LayerSkipConfig, List[LayerSkipConfig], Dict[str, Any]] = None, + auto_guidance_layers: Optional[int | list[int]] = None, + auto_guidance_config: LayerSkipConfig | list[LayerSkipConfig] | dict[str, Any] = None, dropout: Optional[float] = None, guidance_rescale: float = 0.0, use_original_formulation: bool = False, @@ -133,8 +133,8 @@ def cleanup_models(self, denoiser: torch.nn.Module) -> None: registry.remove_hook(name, recurse=True) def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/classifier_free_guidance.py b/src/diffusers/guiders/classifier_free_guidance.py index 050590336ffb..d4bb97a4d62b 100644 --- a/src/diffusers/guiders/classifier_free_guidance.py +++ b/src/diffusers/guiders/classifier_free_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -84,8 +84,8 @@ def __init__( self.use_original_formulation = use_original_formulation def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/classifier_free_zero_star_guidance.py b/src/diffusers/guiders/classifier_free_zero_star_guidance.py index b64e35633114..06603232a4b1 100644 --- a/src/diffusers/guiders/classifier_free_zero_star_guidance.py +++ b/src/diffusers/guiders/classifier_free_zero_star_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -77,8 +77,8 @@ def __init__( self.use_original_formulation = use_original_formulation def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/frequency_decoupled_guidance.py b/src/diffusers/guiders/frequency_decoupled_guidance.py index 93822a180e9d..e8cb8e0829f9 100644 --- a/src/diffusers/guiders/frequency_decoupled_guidance.py +++ b/src/diffusers/guiders/frequency_decoupled_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -37,7 +37,7 @@ build_laplacian_pyramid_func = None -def project(v0: torch.Tensor, v1: torch.Tensor, upcast_to_double: bool = True) -> Tuple[torch.Tensor, torch.Tensor]: +def project(v0: torch.Tensor, v1: torch.Tensor, upcast_to_double: bool = True) -> tuple[torch.Tensor, torch.Tensor]: """ Project vector v0 onto vector v1, returning the parallel and orthogonal components of v0. Implementation from paper (Algorithm 2). @@ -58,7 +58,7 @@ def project(v0: torch.Tensor, v1: torch.Tensor, upcast_to_double: bool = True) - return v0_parallel, v0_orthogonal -def build_image_from_pyramid(pyramid: List[torch.Tensor]) -> torch.Tensor: +def build_image_from_pyramid(pyramid: list[torch.Tensor]) -> torch.Tensor: """ Recovers the data space latents from the Laplacian pyramid frequency space. Implementation from the paper (Algorithm 2). @@ -99,19 +99,19 @@ class FrequencyDecoupledGuidance(BaseGuidance): paper. By default, we use the diffusers-native implementation that has been in the codebase for a long time. Args: - guidance_scales (`List[float]`, defaults to `[10.0, 5.0]`): + guidance_scales (`list[float]`, defaults to `[10.0, 5.0]`): The scale parameter for frequency-decoupled guidance for each frequency component, listed from highest frequency level to lowest. Higher values result in stronger conditioning on the text prompt, while lower values allow for more freedom in generation. Higher values may lead to saturation and deterioration of image quality. The FDG authors recommend using higher guidance scales for higher frequency components and lower guidance scales for lower frequency components (so `guidance_scales` should typically be sorted in descending order). - guidance_rescale (`float` or `List[float]`, defaults to `0.0`): + guidance_rescale (`float` or `list[float]`, defaults to `0.0`): The rescale factor applied to the noise predictions. This is used to improve image quality and fix overexposure. Based on Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). If a list is supplied, it should be the same length as `guidance_scales`. - parallel_weights (`float` or `List[float]`, *optional*): + parallel_weights (`float` or `list[float]`, *optional*): Optional weights for the parallel component of each frequency component of the projected CFG shift. If not set, the weights will default to `1.0` for all components, which corresponds to using the normal CFG shift (that is, equal weights for the parallel and orthogonal components). If set, a value in `[0, 1]` is @@ -120,10 +120,10 @@ class FrequencyDecoupledGuidance(BaseGuidance): Whether to use the original formulation of classifier-free guidance as proposed in the paper. By default, we use the diffusers-native implementation that has been in the codebase for a long time. See [~guiders.classifier_free_guidance.ClassifierFreeGuidance] for more details. - start (`float` or `List[float]`, defaults to `0.0`): + start (`float` or `list[float]`, defaults to `0.0`): The fraction of the total number of denoising steps after which guidance starts. If a list is supplied, it should be the same length as `guidance_scales`. - stop (`float` or `List[float]`, defaults to `1.0`): + stop (`float` or `list[float]`, defaults to `1.0`): The fraction of the total number of denoising steps after which guidance stops. If a list is supplied, it should be the same length as `guidance_scales`. guidance_rescale_space (`str`, defaults to `"data"`): @@ -141,12 +141,12 @@ class FrequencyDecoupledGuidance(BaseGuidance): @register_to_config def __init__( self, - guidance_scales: Union[List[float], Tuple[float]] = [10.0, 5.0], - guidance_rescale: Union[float, List[float], Tuple[float]] = 0.0, - parallel_weights: Optional[Union[float, List[float], Tuple[float]]] = None, + guidance_scales: list[float] | tuple[float] = [10.0, 5.0], + guidance_rescale: float | list[float] | tuple[float] = 0.0, + parallel_weights: Optional[float | list[float] | tuple[float]] = None, use_original_formulation: bool = False, - start: Union[float, List[float], Tuple[float]] = 0.0, - stop: Union[float, List[float], Tuple[float]] = 1.0, + start: float | list[float] | tuple[float] = 0.0, + stop: float | list[float] | tuple[float] = 1.0, guidance_rescale_space: str = "data", upcast_to_double: bool = True, ): @@ -218,8 +218,8 @@ def __init__( ) def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/guider_utils.py b/src/diffusers/guiders/guider_utils.py index 7524b5a3eacc..f50f31a4487e 100644 --- a/src/diffusers/guiders/guider_utils.py +++ b/src/diffusers/guiders/guider_utils.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional import torch from huggingface_hub.utils import validate_hf_hub_args @@ -47,7 +47,7 @@ def __init__(self, start: float = 0.0, stop: float = 1.0): self._num_inference_steps: int = None self._timestep: torch.LongTensor = None self._count_prepared = 0 - self._input_fields: Dict[str, Union[str, Tuple[str, str]]] = None + self._input_fields: dict[str, str | tuple[str, str]] = None self._enabled = True if not (0.0 <= start < 1.0): @@ -72,14 +72,14 @@ def set_state(self, step: int, num_inference_steps: int, timestep: torch.LongTen self._timestep = timestep self._count_prepared = 0 - def set_input_fields(self, **kwargs: Dict[str, Union[str, Tuple[str, str]]]) -> None: + def set_input_fields(self, **kwargs: dict[str, str | tuple[str, str]]) -> None: """ Set the input fields for the guidance technique. The input fields are used to specify the names of the returned attributes containing the prepared data after `prepare_inputs` is called. The prepared data is obtained from the values of the provided keyword arguments to this method. Args: - **kwargs (`Dict[str, Union[str, Tuple[str, str]]]`): + **kwargs (`dict[str, Union[str, tuple[str, str]]]`): A dictionary where the keys are the names of the fields that will be used to store the data once it is prepared with `prepare_inputs`. The values can be either a string or a tuple of length 2, which is used to look up the required data provided for preparation. @@ -124,10 +124,10 @@ def cleanup_models(self, denoiser: torch.nn.Module) -> None: """ pass - def prepare_inputs(self, data: "BlockState") -> List["BlockState"]: + def prepare_inputs(self, data: "BlockState") -> list["BlockState"]: raise NotImplementedError("BaseGuidance::prepare_inputs must be implemented in subclasses.") - def __call__(self, data: List["BlockState"]) -> Any: + def __call__(self, data: list["BlockState"]) -> Any: if not all(hasattr(d, "noise_pred") for d in data): raise ValueError("Expected all data to have `noise_pred` attribute.") if len(data) != self.num_conditions: @@ -155,7 +155,7 @@ def num_conditions(self) -> int: @classmethod def _prepare_batch( cls, - input_fields: Dict[str, Union[str, Tuple[str, str]]], + input_fields: dict[str, str | tuple[str, str]], data: "BlockState", tuple_index: int, identifier: str, @@ -165,7 +165,7 @@ def _prepare_batch( `BaseGuidance` class. It prepares the batch based on the provided tuple index. Args: - input_fields (`Dict[str, Union[str, Tuple[str, str]]]`): + input_fields (`dict[str, Union[str, tuple[str, str]]]`): A dictionary where the keys are the names of the fields that will be used to store the data once it is prepared with `prepare_inputs`. The values can be either a string or a tuple of length 2, which is used to look up the required data provided for preparation. If a string is provided, it will be used as the @@ -205,7 +205,7 @@ def _prepare_batch( @validate_hf_hub_args def from_pretrained( cls, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: Optional[str | os.PathLike] = None, subfolder: Optional[str] = None, return_unused_kwargs=False, **kwargs, @@ -232,7 +232,7 @@ def from_pretrained( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -262,7 +262,7 @@ def from_pretrained( ) return cls.from_config(config, return_unused_kwargs=return_unused_kwargs, **kwargs) - def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save a guider configuration object to a directory so that it can be reloaded using the [`~BaseGuidance.from_pretrained`] class method. @@ -274,7 +274,7 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ self.save_config(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) diff --git a/src/diffusers/guiders/perturbed_attention_guidance.py b/src/diffusers/guiders/perturbed_attention_guidance.py index e294e8d0db59..42484d43f666 100644 --- a/src/diffusers/guiders/perturbed_attention_guidance.py +++ b/src/diffusers/guiders/perturbed_attention_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional import torch @@ -58,10 +58,10 @@ class PerturbedAttentionGuidance(BaseGuidance): The fraction of the total number of denoising steps after which perturbed attention guidance starts. perturbed_guidance_stop (`float`, defaults to `0.2`): The fraction of the total number of denoising steps after which perturbed attention guidance stops. - perturbed_guidance_layers (`int` or `List[int]`, *optional*): + perturbed_guidance_layers (`int` or `list[int]`, *optional*): The layer indices to apply perturbed attention guidance to. Can be a single integer or a list of integers. If not provided, `perturbed_guidance_config` must be provided. - perturbed_guidance_config (`LayerSkipConfig` or `List[LayerSkipConfig]`, *optional*): + perturbed_guidance_config (`LayerSkipConfig` or `list[LayerSkipConfig]`, *optional*): The configuration for the perturbed attention guidance. Can be a single `LayerSkipConfig` or a list of `LayerSkipConfig`. If not provided, `perturbed_guidance_layers` must be provided. guidance_rescale (`float`, defaults to `0.0`): @@ -92,8 +92,8 @@ def __init__( perturbed_guidance_scale: float = 2.8, perturbed_guidance_start: float = 0.01, perturbed_guidance_stop: float = 0.2, - perturbed_guidance_layers: Optional[Union[int, List[int]]] = None, - perturbed_guidance_config: Union[LayerSkipConfig, List[LayerSkipConfig], Dict[str, Any]] = None, + perturbed_guidance_layers: Optional[int | list[int]] = None, + perturbed_guidance_config: LayerSkipConfig | list[LayerSkipConfig] | dict[str, Any] = None, guidance_rescale: float = 0.0, use_original_formulation: bool = False, start: float = 0.0, @@ -169,8 +169,8 @@ def cleanup_models(self, denoiser: torch.nn.Module) -> None: # Copied from diffusers.guiders.skip_layer_guidance.SkipLayerGuidance.prepare_inputs def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/skip_layer_guidance.py b/src/diffusers/guiders/skip_layer_guidance.py index 3530df8b0a18..f1a1c774cf35 100644 --- a/src/diffusers/guiders/skip_layer_guidance.py +++ b/src/diffusers/guiders/skip_layer_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional import torch @@ -64,11 +64,11 @@ class SkipLayerGuidance(BaseGuidance): The fraction of the total number of denoising steps after which skip layer guidance starts. skip_layer_guidance_stop (`float`, defaults to `0.2`): The fraction of the total number of denoising steps after which skip layer guidance stops. - skip_layer_guidance_layers (`int` or `List[int]`, *optional*): + skip_layer_guidance_layers (`int` or `list[int]`, *optional*): The layer indices to apply skip layer guidance to. Can be a single integer or a list of integers. If not provided, `skip_layer_config` must be provided. The recommended values are `[7, 8, 9]` for Stable Diffusion 3.5 Medium. - skip_layer_config (`LayerSkipConfig` or `List[LayerSkipConfig]`, *optional*): + skip_layer_config (`LayerSkipConfig` or `list[LayerSkipConfig]`, *optional*): The configuration for the skip layer guidance. Can be a single `LayerSkipConfig` or a list of `LayerSkipConfig`. If not provided, `skip_layer_guidance_layers` must be provided. guidance_rescale (`float`, defaults to `0.0`): @@ -94,8 +94,8 @@ def __init__( skip_layer_guidance_scale: float = 2.8, skip_layer_guidance_start: float = 0.01, skip_layer_guidance_stop: float = 0.2, - skip_layer_guidance_layers: Optional[Union[int, List[int]]] = None, - skip_layer_config: Union[LayerSkipConfig, List[LayerSkipConfig], Dict[str, Any]] = None, + skip_layer_guidance_layers: Optional[int | list[int]] = None, + skip_layer_config: LayerSkipConfig | list[LayerSkipConfig] | dict[str, Any] = None, guidance_rescale: float = 0.0, use_original_formulation: bool = False, start: float = 0.0, @@ -165,8 +165,8 @@ def cleanup_models(self, denoiser: torch.nn.Module) -> None: registry.remove_hook(hook_name, recurse=True) def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/smoothed_energy_guidance.py b/src/diffusers/guiders/smoothed_energy_guidance.py index 767d20b62f85..17bb159eda8d 100644 --- a/src/diffusers/guiders/smoothed_energy_guidance.py +++ b/src/diffusers/guiders/smoothed_energy_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -54,11 +54,11 @@ class SmoothedEnergyGuidance(BaseGuidance): The fraction of the total number of denoising steps after which smoothed energy guidance starts. seg_guidance_stop (`float`, defaults to `1.0`): The fraction of the total number of denoising steps after which smoothed energy guidance stops. - seg_guidance_layers (`int` or `List[int]`, *optional*): + seg_guidance_layers (`int` or `list[int]`, *optional*): The layer indices to apply smoothed energy guidance to. Can be a single integer or a list of integers. If not provided, `seg_guidance_config` must be provided. The recommended values are `[7, 8, 9]` for Stable Diffusion 3.5 Medium. - seg_guidance_config (`SmoothedEnergyGuidanceConfig` or `List[SmoothedEnergyGuidanceConfig]`, *optional*): + seg_guidance_config (`SmoothedEnergyGuidanceConfig` or `list[SmoothedEnergyGuidanceConfig]`, *optional*): The configuration for the smoothed energy layer guidance. Can be a single `SmoothedEnergyGuidanceConfig` or a list of `SmoothedEnergyGuidanceConfig`. If not provided, `seg_guidance_layers` must be provided. guidance_rescale (`float`, defaults to `0.0`): @@ -86,8 +86,8 @@ def __init__( seg_blur_threshold_inf: float = 9999.0, seg_guidance_start: float = 0.0, seg_guidance_stop: float = 1.0, - seg_guidance_layers: Optional[Union[int, List[int]]] = None, - seg_guidance_config: Union[SmoothedEnergyGuidanceConfig, List[SmoothedEnergyGuidanceConfig]] = None, + seg_guidance_layers: Optional[int | list[int]] = None, + seg_guidance_config: SmoothedEnergyGuidanceConfig | list[SmoothedEnergyGuidanceConfig] = None, guidance_rescale: float = 0.0, use_original_formulation: bool = False, start: float = 0.0, @@ -154,8 +154,8 @@ def cleanup_models(self, denoiser: torch.nn.Module): registry.remove_hook(hook_name, recurse=True) def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/guiders/tangential_classifier_free_guidance.py b/src/diffusers/guiders/tangential_classifier_free_guidance.py index df1e69fe71f5..9f540f31c9b5 100644 --- a/src/diffusers/guiders/tangential_classifier_free_guidance.py +++ b/src/diffusers/guiders/tangential_classifier_free_guidance.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional import torch @@ -66,8 +66,8 @@ def __init__( self.use_original_formulation = use_original_formulation def prepare_inputs( - self, data: "BlockState", input_fields: Optional[Dict[str, Union[str, Tuple[str, str]]]] = None - ) -> List["BlockState"]: + self, data: "BlockState", input_fields: Optional[dict[str, str | tuple[str, str]]] = None + ) -> list["BlockState"]: if input_fields is None: input_fields = self._input_fields diff --git a/src/diffusers/hooks/_helpers.py b/src/diffusers/hooks/_helpers.py index f6e5bdd52d1f..0cd5380f2343 100644 --- a/src/diffusers/hooks/_helpers.py +++ b/src/diffusers/hooks/_helpers.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, Type +from typing import Any, Callable, Type @dataclass @@ -28,7 +28,7 @@ class TransformerBlockMetadata: return_encoder_hidden_states_index: int = None _cls: Type = None - _cached_parameter_indices: Dict[str, int] = None + _cached_parameter_indices: dict[str, int] = None def _get_parameter_from_args_kwargs(self, identifier: str, args=(), kwargs=None): kwargs = kwargs or {} diff --git a/src/diffusers/hooks/context_parallel.py b/src/diffusers/hooks/context_parallel.py index 915fe453b90b..a3a66e21cb10 100644 --- a/src/diffusers/hooks/context_parallel.py +++ b/src/diffusers/hooks/context_parallel.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Dict, List, Type, Union +from typing import Type import torch @@ -42,7 +42,7 @@ # TODO(aryan): consolidate with ._helpers.TransformerBlockMetadata @dataclass class ModuleForwardMetadata: - cached_parameter_indices: Dict[str, int] = None + cached_parameter_indices: dict[str, int] = None _cls: Type = None def _get_parameter_from_args_kwargs(self, identifier: str, args=(), kwargs=None): @@ -78,7 +78,7 @@ def _get_parameter_from_args_kwargs(self, identifier: str, args=(), kwargs=None) def apply_context_parallel( module: torch.nn.Module, parallel_config: ContextParallelConfig, - plan: Dict[str, ContextParallelModelPlan], + plan: dict[str, ContextParallelModelPlan], ) -> None: """Apply context parallel on a model.""" logger.debug(f"Applying context parallel with CP mesh: {parallel_config._mesh} and plan: {plan}") @@ -107,7 +107,7 @@ def apply_context_parallel( registry.register_hook(hook, hook_name) -def remove_context_parallel(module: torch.nn.Module, plan: Dict[str, ContextParallelModelPlan]) -> None: +def remove_context_parallel(module: torch.nn.Module, plan: dict[str, ContextParallelModelPlan]) -> None: for module_id, cp_model_plan in plan.items(): submodule = _get_submodule_by_name(module, module_id) if not isinstance(submodule, list): @@ -272,13 +272,13 @@ def unshard(cls, tensor: torch.Tensor, dim: int, mesh: torch.distributed.device_ return tensor -def _get_submodule_by_name(model: torch.nn.Module, name: str) -> Union[torch.nn.Module, List[torch.nn.Module]]: +def _get_submodule_by_name(model: torch.nn.Module, name: str) -> torch.nn.Module | list[torch.nn.Module]: if name.count("*") > 1: raise ValueError("Wildcard '*' can only be used once in the name") return _find_submodule_by_name(model, name) -def _find_submodule_by_name(model: torch.nn.Module, name: str) -> Union[torch.nn.Module, List[torch.nn.Module]]: +def _find_submodule_by_name(model: torch.nn.Module, name: str) -> torch.nn.Module | list[torch.nn.Module]: if name == "": return model first_atom, remaining_name = name.split(".", 1) if "." in name else (name, "") diff --git a/src/diffusers/hooks/faster_cache.py b/src/diffusers/hooks/faster_cache.py index a01afeffdb95..f789894f7c5e 100644 --- a/src/diffusers/hooks/faster_cache.py +++ b/src/diffusers/hooks/faster_cache.py @@ -14,7 +14,7 @@ import re from dataclasses import dataclass -from typing import Any, Callable, List, Optional, Tuple +from typing import Any, Callable, Optional import torch @@ -60,7 +60,7 @@ class FasterCacheConfig: Calculate the attention states every `N` iterations. If this is set to `N`, the attention computation will be skipped `N - 1` times (i.e., cached attention states will be reused) before computing the new attention states again. - spatial_attention_timestep_skip_range (`Tuple[float, float]`, defaults to `(-1, 681)`): + spatial_attention_timestep_skip_range (`tuple[float, float]`, defaults to `(-1, 681)`): The timestep range within which the spatial attention computation can be skipped without a significant loss in quality. This is to be determined by the user based on the underlying model. The first value in the tuple is the lower bound and the second value is the upper bound. Typically, diffusion timesteps for @@ -68,17 +68,17 @@ class FasterCacheConfig: timestep 0). For the default values, this would mean that the spatial attention computation skipping will be applicable only after denoising timestep 681 is reached, and continue until the end of the denoising process. - temporal_attention_timestep_skip_range (`Tuple[float, float]`, *optional*, defaults to `None`): + temporal_attention_timestep_skip_range (`tuple[float, float]`, *optional*, defaults to `None`): The timestep range within which the temporal attention computation can be skipped without a significant loss in quality. This is to be determined by the user based on the underlying model. The first value in the tuple is the lower bound and the second value is the upper bound. Typically, diffusion timesteps for denoising are in the reversed range of 0 to 1000 (i.e. denoising starts at timestep 1000 and ends at timestep 0). - low_frequency_weight_update_timestep_range (`Tuple[int, int]`, defaults to `(99, 901)`): + low_frequency_weight_update_timestep_range (`tuple[int, int]`, defaults to `(99, 901)`): The timestep range within which the low frequency weight scaling update is applied. The first value in the tuple is the lower bound and the second value is the upper bound of the timestep range. The callback function for the update is called only within this range. - high_frequency_weight_update_timestep_range (`Tuple[int, int]`, defaults to `(-1, 301)`): + high_frequency_weight_update_timestep_range (`tuple[int, int]`, defaults to `(-1, 301)`): The timestep range within which the high frequency weight scaling update is applied. The first value in the tuple is the lower bound and the second value is the upper bound of the timestep range. The callback function for the update is called only within this range. @@ -92,15 +92,15 @@ class FasterCacheConfig: Process the unconditional branch every `N` iterations. If this is set to `N`, the unconditional branch computation will be skipped `N - 1` times (i.e., cached unconditional branch states will be reused) before computing the new unconditional branch states again. - unconditional_batch_timestep_skip_range (`Tuple[float, float]`, defaults to `(-1, 641)`): + unconditional_batch_timestep_skip_range (`tuple[float, float]`, defaults to `(-1, 641)`): The timestep range within which the unconditional branch computation can be skipped without a significant loss in quality. This is to be determined by the user based on the underlying model. The first value in the tuple is the lower bound and the second value is the upper bound. - spatial_attention_block_identifiers (`Tuple[str, ...]`, defaults to `("blocks.*attn1", "transformer_blocks.*attn1", "single_transformer_blocks.*attn1")`): + spatial_attention_block_identifiers (`tuple[str, ...]`, defaults to `("blocks.*attn1", "transformer_blocks.*attn1", "single_transformer_blocks.*attn1")`): The identifiers to match the spatial attention blocks in the model. If the name of the block contains any of these identifiers, FasterCache will be applied to that block. This can either be the full layer names, partial layer names, or regex patterns. Matching will always be done using a regex match. - temporal_attention_block_identifiers (`Tuple[str, ...]`, defaults to `("temporal_transformer_blocks.*attn1",)`): + temporal_attention_block_identifiers (`tuple[str, ...]`, defaults to `("temporal_transformer_blocks.*attn1",)`): The identifiers to match the temporal attention blocks in the model. If the name of the block contains any of these identifiers, FasterCache will be applied to that block. This can either be the full layer names, partial layer names, or regex patterns. Matching will always be done using a regex match. @@ -123,7 +123,7 @@ class FasterCacheConfig: is_guidance_distilled (`bool`, defaults to `False`): Whether the model is guidance distilled or not. If the model is guidance distilled, FasterCache will not be applied at the denoiser-level to skip the unconditional branch computation (as there is none). - _unconditional_conditional_input_kwargs_identifiers (`List[str]`, defaults to `("hidden_states", "encoder_hidden_states", "timestep", "attention_mask", "encoder_attention_mask")`): + _unconditional_conditional_input_kwargs_identifiers (`list[str]`, defaults to `("hidden_states", "encoder_hidden_states", "timestep", "attention_mask", "encoder_attention_mask")`): The identifiers to match the input kwargs that contain the batchwise-concatenated unconditional and conditional inputs. If the name of the input kwargs contains any of these identifiers, FasterCache will split the inputs into unconditional and conditional branches. This must be a list of exact input kwargs @@ -135,12 +135,12 @@ class FasterCacheConfig: spatial_attention_block_skip_range: int = 2 temporal_attention_block_skip_range: Optional[int] = None - spatial_attention_timestep_skip_range: Tuple[int, int] = (-1, 681) - temporal_attention_timestep_skip_range: Tuple[int, int] = (-1, 681) + spatial_attention_timestep_skip_range: tuple[int, int] = (-1, 681) + temporal_attention_timestep_skip_range: tuple[int, int] = (-1, 681) # Indicator functions for low/high frequency as mentioned in Equation 11 of the paper - low_frequency_weight_update_timestep_range: Tuple[int, int] = (99, 901) - high_frequency_weight_update_timestep_range: Tuple[int, int] = (-1, 301) + low_frequency_weight_update_timestep_range: tuple[int, int] = (99, 901) + high_frequency_weight_update_timestep_range: tuple[int, int] = (-1, 301) # ⍺1 and ⍺2 as mentioned in Equation 11 of the paper alpha_low_frequency: float = 1.1 @@ -148,10 +148,10 @@ class FasterCacheConfig: # n as described in CFG-Cache explanation in the paper - dependent on the model unconditional_batch_skip_range: int = 5 - unconditional_batch_timestep_skip_range: Tuple[int, int] = (-1, 641) + unconditional_batch_timestep_skip_range: tuple[int, int] = (-1, 641) - spatial_attention_block_identifiers: Tuple[str, ...] = _SPATIAL_ATTENTION_BLOCK_IDENTIFIERS - temporal_attention_block_identifiers: Tuple[str, ...] = _TEMPORAL_ATTENTION_BLOCK_IDENTIFIERS + spatial_attention_block_identifiers: tuple[str, ...] = _SPATIAL_ATTENTION_BLOCK_IDENTIFIERS + temporal_attention_block_identifiers: tuple[str, ...] = _TEMPORAL_ATTENTION_BLOCK_IDENTIFIERS attention_weight_callback: Callable[[torch.nn.Module], float] = None low_frequency_weight_callback: Callable[[torch.nn.Module], float] = None @@ -162,7 +162,7 @@ class FasterCacheConfig: current_timestep_callback: Callable[[], int] = None - _unconditional_conditional_input_kwargs_identifiers: List[str] = _UNCOND_COND_INPUT_KWARGS_IDENTIFIERS + _unconditional_conditional_input_kwargs_identifiers: list[str] = _UNCOND_COND_INPUT_KWARGS_IDENTIFIERS def __repr__(self) -> str: return ( @@ -209,7 +209,7 @@ class FasterCacheBlockState: def __init__(self) -> None: self.iteration: int = 0 self.batch_size: int = None - self.cache: Tuple[torch.Tensor, torch.Tensor] = None + self.cache: tuple[torch.Tensor, torch.Tensor] = None def reset(self): self.iteration = 0 @@ -223,10 +223,10 @@ class FasterCacheDenoiserHook(ModelHook): def __init__( self, unconditional_batch_skip_range: int, - unconditional_batch_timestep_skip_range: Tuple[int, int], + unconditional_batch_timestep_skip_range: tuple[int, int], tensor_format: str, is_guidance_distilled: bool, - uncond_cond_input_kwargs_identifiers: List[str], + uncond_cond_input_kwargs_identifiers: list[str], current_timestep_callback: Callable[[], int], low_frequency_weight_callback: Callable[[torch.nn.Module], torch.Tensor], high_frequency_weight_callback: Callable[[torch.nn.Module], torch.Tensor], @@ -252,7 +252,7 @@ def initialize_hook(self, module): return module @staticmethod - def _get_cond_input(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + def _get_cond_input(input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: # Note: this method assumes that the input tensor is batchwise-concatenated with unconditional inputs # followed by conditional inputs. _, cond = input.chunk(2, dim=0) @@ -371,7 +371,7 @@ class FasterCacheBlockHook(ModelHook): def __init__( self, block_skip_range: int, - timestep_skip_range: Tuple[int, int], + timestep_skip_range: tuple[int, int], is_guidance_distilled: bool, weight_callback: Callable[[torch.nn.Module], float], current_timestep_callback: Callable[[], int], diff --git a/src/diffusers/hooks/first_block_cache.py b/src/diffusers/hooks/first_block_cache.py index 862d44059301..685ccd383674 100644 --- a/src/diffusers/hooks/first_block_cache.py +++ b/src/diffusers/hooks/first_block_cache.py @@ -13,7 +13,6 @@ # limitations under the License. from dataclasses import dataclass -from typing import Tuple, Union import torch @@ -53,9 +52,9 @@ class FBCSharedBlockState(BaseState): def __init__(self) -> None: super().__init__() - self.head_block_output: Union[torch.Tensor, Tuple[torch.Tensor, ...]] = None + self.head_block_output: torch.Tensor | tuple[torch.Tensor, ...] = None self.head_block_residual: torch.Tensor = None - self.tail_block_residuals: Union[torch.Tensor, Tuple[torch.Tensor, ...]] = None + self.tail_block_residuals: torch.Tensor | tuple[torch.Tensor, ...] = None self.should_compute: bool = True def reset(self): diff --git a/src/diffusers/hooks/group_offloading.py b/src/diffusers/hooks/group_offloading.py index 38f291f5203c..26fd5c93e815 100644 --- a/src/diffusers/hooks/group_offloading.py +++ b/src/diffusers/hooks/group_offloading.py @@ -17,7 +17,7 @@ from contextlib import contextmanager, nullcontext from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Optional, Set import safetensors.torch import torch @@ -58,21 +58,21 @@ class GroupOffloadingConfig: low_cpu_mem_usage: bool num_blocks_per_group: Optional[int] = None offload_to_disk_path: Optional[str] = None - stream: Optional[Union[torch.cuda.Stream, torch.Stream]] = None + stream: Optional[torch.cuda.Stream | torch.Stream] = None class ModuleGroup: def __init__( self, - modules: List[torch.nn.Module], + modules: list[torch.nn.Module], offload_device: torch.device, onload_device: torch.device, offload_leader: torch.nn.Module, onload_leader: Optional[torch.nn.Module] = None, - parameters: Optional[List[torch.nn.Parameter]] = None, - buffers: Optional[List[torch.Tensor]] = None, + parameters: Optional[list[torch.nn.Parameter]] = None, + buffers: Optional[list[torch.Tensor]] = None, non_blocking: bool = False, - stream: Union[torch.cuda.Stream, torch.Stream, None] = None, + stream: torch.cuda.Stream | torch.Stream | None = None, record_stream: Optional[bool] = False, low_cpu_mem_usage: bool = False, onload_self: bool = True, @@ -340,7 +340,7 @@ class LazyPrefetchGroupOffloadingHook(ModelHook): _is_stateful = False def __init__(self): - self.execution_order: List[Tuple[str, torch.nn.Module]] = [] + self.execution_order: list[tuple[str, torch.nn.Module]] = [] self._layer_execution_tracker_module_names = set() def initialize_hook(self, module): @@ -444,9 +444,9 @@ def pre_forward(self, module, *args, **kwargs): def apply_group_offloading( module: torch.nn.Module, - onload_device: Union[str, torch.device], - offload_device: Union[str, torch.device] = torch.device("cpu"), - offload_type: Union[str, GroupOffloadingType] = "block_level", + onload_device: str | torch.device, + offload_device: str | torch.device = torch.device("cpu"), + offload_type: str | GroupOffloadingType = "block_level", num_blocks_per_group: Optional[int] = None, non_blocking: bool = False, use_stream: bool = False, @@ -787,7 +787,7 @@ def _apply_lazy_group_offloading_hook( def _gather_parameters_with_no_group_offloading_parent( module: torch.nn.Module, modules_with_group_offloading: Set[str] -) -> List[torch.nn.Parameter]: +) -> list[torch.nn.Parameter]: parameters = [] for name, parameter in module.named_parameters(): has_parent_with_group_offloading = False @@ -805,7 +805,7 @@ def _gather_parameters_with_no_group_offloading_parent( def _gather_buffers_with_no_group_offloading_parent( module: torch.nn.Module, modules_with_group_offloading: Set[str] -) -> List[torch.Tensor]: +) -> list[torch.Tensor]: buffers = [] for name, buffer in module.named_buffers(): has_parent_with_group_offloading = False @@ -821,7 +821,7 @@ def _gather_buffers_with_no_group_offloading_parent( return buffers -def _find_parent_module_in_module_dict(name: str, module_dict: Dict[str, torch.nn.Module]) -> str: +def _find_parent_module_in_module_dict(name: str, module_dict: dict[str, torch.nn.Module]) -> str: atoms = name.split(".") while len(atoms) > 0: parent_name = ".".join(atoms) diff --git a/src/diffusers/hooks/hooks.py b/src/diffusers/hooks/hooks.py index 6e097e5882a0..dcf877ebe5e5 100644 --- a/src/diffusers/hooks/hooks.py +++ b/src/diffusers/hooks/hooks.py @@ -13,7 +13,7 @@ # limitations under the License. import functools -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional import torch @@ -86,19 +86,19 @@ def deinitalize_hook(self, module: torch.nn.Module) -> torch.nn.Module: """ return module - def pre_forward(self, module: torch.nn.Module, *args, **kwargs) -> Tuple[Tuple[Any], Dict[str, Any]]: + def pre_forward(self, module: torch.nn.Module, *args, **kwargs) -> tuple[tuple[Any], dict[str, Any]]: r""" Hook that is executed just before the forward method of the model. Args: module (`torch.nn.Module`): The module whose forward pass will be executed just after this event. - args (`Tuple[Any]`): + args (`tuple[Any]`): The positional arguments passed to the module. - kwargs (`Dict[Str, Any]`): + kwargs (`dict[Str, Any]`): The keyword arguments passed to the module. Returns: - `Tuple[Tuple[Any], Dict[Str, Any]]`: + `tuple[tuple[Any], dict[Str, Any]]`: A tuple with the treated `args` and `kwargs`. """ return args, kwargs @@ -168,7 +168,7 @@ class HookRegistry: def __init__(self, module_ref: torch.nn.Module) -> None: super().__init__() - self.hooks: Dict[str, ModelHook] = {} + self.hooks: dict[str, ModelHook] = {} self._module_ref = module_ref self._hook_order = [] diff --git a/src/diffusers/hooks/layer_skip.py b/src/diffusers/hooks/layer_skip.py index 0ce02e987d09..964b0a64b330 100644 --- a/src/diffusers/hooks/layer_skip.py +++ b/src/diffusers/hooks/layer_skip.py @@ -14,7 +14,7 @@ import math from dataclasses import asdict, dataclass -from typing import Callable, List, Optional +from typing import Callable, Optional import torch @@ -43,7 +43,7 @@ class LayerSkipConfig: Configuration for skipping internal transformer blocks when executing a transformer model. Args: - indices (`List[int]`): + indices (`list[int]`): The indices of the layer to skip. This is typically the first layer in the transformer block. fqn (`str`, defaults to `"auto"`): The fully qualified name identifying the stack of transformer blocks. Typically, this is @@ -63,7 +63,7 @@ class LayerSkipConfig: skipped layers are fully retained, which is equivalent to not skipping any layers. """ - indices: List[int] + indices: list[int] fqn: str = "auto" skip_attention: bool = True skip_attention_scores: bool = False diff --git a/src/diffusers/hooks/layerwise_casting.py b/src/diffusers/hooks/layerwise_casting.py index a036ad37dc2f..1d8b5eaaa8be 100644 --- a/src/diffusers/hooks/layerwise_casting.py +++ b/src/diffusers/hooks/layerwise_casting.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Optional, Tuple, Type, Union +from typing import Optional, Type import torch @@ -102,8 +102,8 @@ def apply_layerwise_casting( module: torch.nn.Module, storage_dtype: torch.dtype, compute_dtype: torch.dtype, - skip_modules_pattern: Union[str, Tuple[str, ...]] = "auto", - skip_modules_classes: Optional[Tuple[Type[torch.nn.Module], ...]] = None, + skip_modules_pattern: str | tuple[str, ...] = "auto", + skip_modules_classes: Optional[tuple[Type[torch.nn.Module], ...]] = None, non_blocking: bool = False, ) -> None: r""" @@ -137,12 +137,12 @@ def apply_layerwise_casting( The dtype to cast the module to before/after the forward pass for storage. compute_dtype (`torch.dtype`): The dtype to cast the module to during the forward pass for computation. - skip_modules_pattern (`Tuple[str, ...]`, defaults to `"auto"`): + skip_modules_pattern (`tuple[str, ...]`, defaults to `"auto"`): A list of patterns to match the names of the modules to skip during the layerwise casting process. If set to `"auto"`, the default patterns are used. If set to `None`, no modules are skipped. If set to `None` alongside `skip_modules_classes` being `None`, the layerwise casting is applied directly to the module instead of its internal submodules. - skip_modules_classes (`Tuple[Type[torch.nn.Module], ...]`, defaults to `None`): + skip_modules_classes (`tuple[Type[torch.nn.Module], ...]`, defaults to `None`): A list of module classes to skip during the layerwise casting process. non_blocking (`bool`, defaults to `False`): If `True`, the weight casting operations are non-blocking. @@ -169,8 +169,8 @@ def _apply_layerwise_casting( module: torch.nn.Module, storage_dtype: torch.dtype, compute_dtype: torch.dtype, - skip_modules_pattern: Optional[Tuple[str, ...]] = None, - skip_modules_classes: Optional[Tuple[Type[torch.nn.Module], ...]] = None, + skip_modules_pattern: Optional[tuple[str, ...]] = None, + skip_modules_classes: Optional[tuple[Type[torch.nn.Module], ...]] = None, non_blocking: bool = False, _prefix: str = "", ) -> None: diff --git a/src/diffusers/hooks/pyramid_attention_broadcast.py b/src/diffusers/hooks/pyramid_attention_broadcast.py index 12d6aa0616e9..b5e8d050bfb8 100644 --- a/src/diffusers/hooks/pyramid_attention_broadcast.py +++ b/src/diffusers/hooks/pyramid_attention_broadcast.py @@ -14,7 +14,7 @@ import re from dataclasses import dataclass -from typing import Any, Callable, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch @@ -54,20 +54,20 @@ class PyramidAttentionBroadcastConfig: The number of times a specific cross-attention broadcast is skipped before computing the attention states to re-use. If this is set to the value `N`, the attention computation will be skipped `N - 1` times (i.e., old attention states will be reused) before computing the new attention states again. - spatial_attention_timestep_skip_range (`Tuple[int, int]`, defaults to `(100, 800)`): + spatial_attention_timestep_skip_range (`tuple[int, int]`, defaults to `(100, 800)`): The range of timesteps to skip in the spatial attention layer. The attention computations will be conditionally skipped if the current timestep is within the specified range. - temporal_attention_timestep_skip_range (`Tuple[int, int]`, defaults to `(100, 800)`): + temporal_attention_timestep_skip_range (`tuple[int, int]`, defaults to `(100, 800)`): The range of timesteps to skip in the temporal attention layer. The attention computations will be conditionally skipped if the current timestep is within the specified range. - cross_attention_timestep_skip_range (`Tuple[int, int]`, defaults to `(100, 800)`): + cross_attention_timestep_skip_range (`tuple[int, int]`, defaults to `(100, 800)`): The range of timesteps to skip in the cross-attention layer. The attention computations will be conditionally skipped if the current timestep is within the specified range. - spatial_attention_block_identifiers (`Tuple[str, ...]`): + spatial_attention_block_identifiers (`tuple[str, ...]`): The identifiers to match against the layer names to determine if the layer is a spatial attention layer. - temporal_attention_block_identifiers (`Tuple[str, ...]`): + temporal_attention_block_identifiers (`tuple[str, ...]`): The identifiers to match against the layer names to determine if the layer is a temporal attention layer. - cross_attention_block_identifiers (`Tuple[str, ...]`): + cross_attention_block_identifiers (`tuple[str, ...]`): The identifiers to match against the layer names to determine if the layer is a cross-attention layer. """ @@ -75,13 +75,13 @@ class PyramidAttentionBroadcastConfig: temporal_attention_block_skip_range: Optional[int] = None cross_attention_block_skip_range: Optional[int] = None - spatial_attention_timestep_skip_range: Tuple[int, int] = (100, 800) - temporal_attention_timestep_skip_range: Tuple[int, int] = (100, 800) - cross_attention_timestep_skip_range: Tuple[int, int] = (100, 800) + spatial_attention_timestep_skip_range: tuple[int, int] = (100, 800) + temporal_attention_timestep_skip_range: tuple[int, int] = (100, 800) + cross_attention_timestep_skip_range: tuple[int, int] = (100, 800) - spatial_attention_block_identifiers: Tuple[str, ...] = _SPATIAL_TRANSFORMER_BLOCK_IDENTIFIERS - temporal_attention_block_identifiers: Tuple[str, ...] = _TEMPORAL_TRANSFORMER_BLOCK_IDENTIFIERS - cross_attention_block_identifiers: Tuple[str, ...] = _CROSS_TRANSFORMER_BLOCK_IDENTIFIERS + spatial_attention_block_identifiers: tuple[str, ...] = _SPATIAL_TRANSFORMER_BLOCK_IDENTIFIERS + temporal_attention_block_identifiers: tuple[str, ...] = _TEMPORAL_TRANSFORMER_BLOCK_IDENTIFIERS + cross_attention_block_identifiers: tuple[str, ...] = _CROSS_TRANSFORMER_BLOCK_IDENTIFIERS current_timestep_callback: Callable[[], int] = None @@ -141,7 +141,7 @@ class PyramidAttentionBroadcastHook(ModelHook): _is_stateful = True def __init__( - self, timestep_skip_range: Tuple[int, int], block_skip_range: int, current_timestep_callback: Callable[[], int] + self, timestep_skip_range: tuple[int, int], block_skip_range: int, current_timestep_callback: Callable[[], int] ) -> None: super().__init__() @@ -288,8 +288,8 @@ def _apply_pyramid_attention_broadcast_on_attention_class( def _apply_pyramid_attention_broadcast_hook( - module: Union[Attention, MochiAttention], - timestep_skip_range: Tuple[int, int], + module: Attention | MochiAttention, + timestep_skip_range: tuple[int, int], block_skip_range: int, current_timestep_callback: Callable[[], int], ): @@ -299,7 +299,7 @@ def _apply_pyramid_attention_broadcast_hook( Args: module (`torch.nn.Module`): The module to apply Pyramid Attention Broadcast to. - timestep_skip_range (`Tuple[int, int]`): + timestep_skip_range (`tuple[int, int]`): The range of timesteps to skip in the attention layer. The attention computations will be conditionally skipped if the current timestep is within the specified range. block_skip_range (`int`): diff --git a/src/diffusers/hooks/smoothed_energy_guidance_utils.py b/src/diffusers/hooks/smoothed_energy_guidance_utils.py index 622f60764762..1063078d22f6 100644 --- a/src/diffusers/hooks/smoothed_energy_guidance_utils.py +++ b/src/diffusers/hooks/smoothed_energy_guidance_utils.py @@ -14,7 +14,7 @@ import math from dataclasses import asdict, dataclass -from typing import List, Optional +from typing import Optional import torch import torch.nn.functional as F @@ -35,21 +35,21 @@ class SmoothedEnergyGuidanceConfig: Configuration for skipping internal transformer blocks when executing a transformer model. Args: - indices (`List[int]`): + indices (`list[int]`): The indices of the layer to skip. This is typically the first layer in the transformer block. fqn (`str`, defaults to `"auto"`): The fully qualified name identifying the stack of transformer blocks. Typically, this is `transformer_blocks`, `single_transformer_blocks`, `blocks`, `layers`, or `temporal_transformer_blocks`. For automatic detection, set this to `"auto"`. "auto" only works on DiT models. For UNet models, you must provide the correct fqn. - _query_proj_identifiers (`List[str]`, defaults to `None`): + _query_proj_identifiers (`list[str]`, defaults to `None`): The identifiers for the query projection layers. Typically, these are `to_q`, `query`, or `q_proj`. If `None`, `to_q` is used by default. """ - indices: List[int] + indices: list[int] fqn: str = "auto" - _query_proj_identifiers: List[str] = None + _query_proj_identifiers: list[str] = None def to_dict(self): return asdict(self) diff --git a/src/diffusers/hooks/utils.py b/src/diffusers/hooks/utils.py index c5260eeebe1f..da9e398fdf29 100644 --- a/src/diffusers/hooks/utils.py +++ b/src/diffusers/hooks/utils.py @@ -21,8 +21,8 @@ def _get_identifiable_transformer_blocks_in_module(module: torch.nn.Module): module_list_with_transformer_blocks = [] for name, submodule in module.named_modules(): name_endswith_identifier = any(name.endswith(identifier) for identifier in _ALL_TRANSFORMER_BLOCK_IDENTIFIERS) - is_modulelist = isinstance(submodule, torch.nn.ModuleList) - if name_endswith_identifier and is_modulelist: + is_ModuleList = isinstance(submodule, torch.nn.ModuleList) + if name_endswith_identifier and is_ModuleList: module_list_with_transformer_blocks.append((name, submodule)) return module_list_with_transformer_blocks diff --git a/src/diffusers/image_processor.py b/src/diffusers/image_processor.py index 0e3082eada8a..5c3756a6d173 100644 --- a/src/diffusers/image_processor.py +++ b/src/diffusers/image_processor.py @@ -14,7 +14,7 @@ import math import warnings -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import PIL.Image @@ -26,14 +26,9 @@ from .utils import CONFIG_NAME, PIL_INTERPOLATION, deprecate -PipelineImageInput = Union[ - PIL.Image.Image, - np.ndarray, - torch.Tensor, - List[PIL.Image.Image], - List[np.ndarray], - List[torch.Tensor], -] +PipelineImageInput = ( + PIL.Image.Image | np.ndarray | torch.Tensor | list[PIL.Image.Image] | list[np.ndarray] | list[torch.Tensor] +) PipelineDepthInput = PipelineImageInput @@ -68,7 +63,7 @@ def is_valid_image_imagelist(images): - A list of valid images. Args: - images (`Union[np.ndarray, torch.Tensor, PIL.Image.Image, List]`): + images (`Union[np.ndarray, torch.Tensor, PIL.Image.Image, list]`): The image(s) to check. Can be a batch of images (4D tensor/array), a single image, or a list of valid images. @@ -131,7 +126,7 @@ def __init__( ) @staticmethod - def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: + def numpy_to_pil(images: np.ndarray) -> list[PIL.Image.Image]: r""" Convert a numpy image or a batch of images to a PIL image. @@ -140,7 +135,7 @@ def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: The image array to convert to PIL format. Returns: - `List[PIL.Image.Image]`: + `list[PIL.Image.Image]`: A list of PIL images. """ if images.ndim == 3: @@ -155,12 +150,12 @@ def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: return pil_images @staticmethod - def pil_to_numpy(images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> np.ndarray: + def pil_to_numpy(images: list[PIL.Image.Image] | PIL.Image.Image) -> np.ndarray: r""" Convert a PIL image or a list of PIL images to NumPy arrays. Args: - images (`PIL.Image.Image` or `List[PIL.Image.Image]`): + images (`PIL.Image.Image` or `list[PIL.Image.Image]`): The PIL image or list of images to convert to NumPy format. Returns: @@ -210,7 +205,7 @@ def pt_to_numpy(images: torch.Tensor) -> np.ndarray: return images @staticmethod - def normalize(images: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + def normalize(images: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor: r""" Normalize an image array to [-1,1]. @@ -225,7 +220,7 @@ def normalize(images: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torc return 2.0 * images - 1.0 @staticmethod - def denormalize(images: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + def denormalize(images: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor: r""" Denormalize an image array to [0,1]. @@ -467,11 +462,11 @@ def _resize_and_crop( def resize( self, - image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + image: PIL.Image.Image | np.ndarray | torch.Tensor, height: int, width: int, resize_mode: str = "default", # "default", "fill", "crop" - ) -> Union[PIL.Image.Image, np.ndarray, torch.Tensor]: + ) -> PIL.Image.Image | np.ndarray | torch.Tensor: """ Resize image. @@ -544,7 +539,7 @@ def binarize(self, image: PIL.Image.Image) -> PIL.Image.Image: return image def _denormalize_conditionally( - self, images: torch.Tensor, do_denormalize: Optional[List[bool]] = None + self, images: torch.Tensor, do_denormalize: Optional[list[bool]] = None ) -> torch.Tensor: r""" Denormalize a batch of images based on a condition list. @@ -552,7 +547,7 @@ def _denormalize_conditionally( Args: images (`torch.Tensor`): The input image tensor. - do_denormalize (`Optional[List[bool]`, *optional*, defaults to `None`): + do_denormalize (`Optional[list[bool]`, *optional*, defaults to `None`): A list of booleans indicating whether to denormalize each image in the batch. If `None`, will use the value of `do_normalize` in the `VaeImageProcessor` config. """ @@ -565,10 +560,10 @@ def _denormalize_conditionally( def get_default_height_width( self, - image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + image: PIL.Image.Image | np.ndarray | torch.Tensor, height: Optional[int] = None, width: Optional[int] = None, - ) -> Tuple[int, int]: + ) -> tuple[int, int]: r""" Returns the height and width of the image, downscaled to the next integer multiple of `vae_scale_factor`. @@ -583,7 +578,7 @@ def get_default_height_width( The width of the preprocessed image. If `None`, the width of the `image` input will be used. Returns: - `Tuple[int, int]`: + `tuple[int, int]`: A tuple containing the height and width, both resized to the nearest integer multiple of `vae_scale_factor`. """ @@ -616,7 +611,7 @@ def preprocess( height: Optional[int] = None, width: Optional[int] = None, resize_mode: str = "default", # "default", "fill", "crop" - crops_coords: Optional[Tuple[int, int, int, int]] = None, + crops_coords: Optional[tuple[int, int, int, int]] = None, ) -> torch.Tensor: """ Preprocess the image input. @@ -638,7 +633,7 @@ def preprocess( image to fit within the specified width and height, maintaining the aspect ratio, and then center the image within the dimensions, cropping the excess. Note that resize_mode `fill` and `crop` are only supported for PIL image input. - crops_coords (`List[Tuple[int, int, int, int]]`, *optional*, defaults to `None`): + crops_coords (`list[tuple[int, int, int, int]]`, *optional*, defaults to `None`): The crop coordinates for each image in the batch. If `None`, will not crop the image. Returns: @@ -745,8 +740,8 @@ def postprocess( self, image: torch.Tensor, output_type: str = "pil", - do_denormalize: Optional[List[bool]] = None, - ) -> Union[PIL.Image.Image, np.ndarray, torch.Tensor]: + do_denormalize: Optional[list[bool]] = None, + ) -> PIL.Image.Image | np.ndarray | torch.Tensor: """ Postprocess the image output from tensor to `output_type`. @@ -755,7 +750,7 @@ def postprocess( The image input, should be a pytorch tensor with shape `B x C x H x W`. output_type (`str`, *optional*, defaults to `pil`): The output type of the image, can be one of `pil`, `np`, `pt`, `latent`. - do_denormalize (`List[bool]`, *optional*, defaults to `None`): + do_denormalize (`list[bool]`, *optional*, defaults to `None`): Whether to denormalize the image to [0,1]. If `None`, will use the value of `do_normalize` in the `VaeImageProcessor` config. @@ -796,7 +791,7 @@ def apply_overlay( mask: PIL.Image.Image, init_image: PIL.Image.Image, image: PIL.Image.Image, - crop_coords: Optional[Tuple[int, int, int, int]] = None, + crop_coords: Optional[tuple[int, int, int, int]] = None, ) -> PIL.Image.Image: r""" Applies an overlay of the mask and the inpainted image on the original image. @@ -808,7 +803,7 @@ def apply_overlay( The original image to which the overlay is applied. image (`PIL.Image.Image`): The image to overlay onto the original. - crop_coords (`Tuple[int, int, int, int]`, *optional*): + crop_coords (`tuple[int, int, int, int]`, *optional*): Coordinates to crop the image. If provided, the image will be cropped accordingly. Returns: @@ -891,7 +886,7 @@ def preprocess( height: int = None, width: int = None, padding_mask_crop: Optional[int] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: """ Preprocess the image and mask. """ @@ -946,8 +941,8 @@ def postprocess( output_type: str = "pil", original_image: Optional[PIL.Image.Image] = None, original_mask: Optional[PIL.Image.Image] = None, - crops_coords: Optional[Tuple[int, int, int, int]] = None, - ) -> Tuple[PIL.Image.Image, PIL.Image.Image]: + crops_coords: Optional[tuple[int, int, int, int]] = None, + ) -> tuple[PIL.Image.Image, PIL.Image.Image]: """ Postprocess the image, optionally apply mask overlay """ @@ -998,7 +993,7 @@ def __init__( super().__init__() @staticmethod - def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: + def numpy_to_pil(images: np.ndarray) -> list[PIL.Image.Image]: r""" Convert a NumPy image or a batch of images to a list of PIL images. @@ -1007,7 +1002,7 @@ def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: The input NumPy array of images, which can be a single image or a batch. Returns: - `List[PIL.Image.Image]`: + `list[PIL.Image.Image]`: A list of PIL images converted from the input NumPy array. """ if images.ndim == 3: @@ -1022,12 +1017,12 @@ def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: return pil_images @staticmethod - def depth_pil_to_numpy(images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> np.ndarray: + def depth_pil_to_numpy(images: list[PIL.Image.Image] | PIL.Image.Image) -> np.ndarray: r""" Convert a PIL image or a list of PIL images to NumPy arrays. Args: - images (`Union[List[PIL.Image.Image], PIL.Image.Image]`): + images (`Union[list[PIL.Image.Image], PIL.Image.Image]`): The input image or list of images to be converted. Returns: @@ -1042,7 +1037,7 @@ def depth_pil_to_numpy(images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> return images @staticmethod - def rgblike_to_depthmap(image: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + def rgblike_to_depthmap(image: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor: r""" Convert an RGB-like depth image to a depth map. @@ -1056,7 +1051,7 @@ def rgblike_to_depthmap(image: Union[np.ndarray, torch.Tensor]) -> Union[np.ndar """ return image[:, :, 1] * 2**8 + image[:, :, 2] - def numpy_to_depth(self, images: np.ndarray) -> List[PIL.Image.Image]: + def numpy_to_depth(self, images: np.ndarray) -> list[PIL.Image.Image]: r""" Convert a NumPy depth image or a batch of images to a list of PIL images. @@ -1065,7 +1060,7 @@ def numpy_to_depth(self, images: np.ndarray) -> List[PIL.Image.Image]: The input NumPy array of depth images, which can be a single image or a batch. Returns: - `List[PIL.Image.Image]`: + `list[PIL.Image.Image]`: A list of PIL images converted from the input NumPy depth images. """ if images.ndim == 3: @@ -1088,8 +1083,8 @@ def postprocess( self, image: torch.Tensor, output_type: str = "pil", - do_denormalize: Optional[List[bool]] = None, - ) -> Union[PIL.Image.Image, np.ndarray, torch.Tensor]: + do_denormalize: Optional[list[bool]] = None, + ) -> PIL.Image.Image | np.ndarray | torch.Tensor: """ Postprocess the image output from tensor to `output_type`. @@ -1098,7 +1093,7 @@ def postprocess( The image input, should be a pytorch tensor with shape `B x C x H x W`. output_type (`str`, *optional*, defaults to `pil`): The output type of the image, can be one of `pil`, `np`, `pt`, `latent`. - do_denormalize (`List[bool]`, *optional*, defaults to `None`): + do_denormalize (`list[bool]`, *optional*, defaults to `None`): Whether to denormalize the image to [0,1]. If `None`, will use the value of `do_normalize` in the `VaeImageProcessor` config. @@ -1136,8 +1131,8 @@ def postprocess( def preprocess( self, - rgb: Union[torch.Tensor, PIL.Image.Image, np.ndarray], - depth: Union[torch.Tensor, PIL.Image.Image, np.ndarray], + rgb: torch.Tensor | PIL.Image.Image | np.ndarray, + depth: torch.Tensor | PIL.Image.Image | np.ndarray, height: Optional[int] = None, width: Optional[int] = None, target_res: Optional[int] = None, @@ -1158,7 +1153,7 @@ def preprocess( Target resolution for resizing the images. If specified, overrides height and width. Returns: - `Tuple[torch.Tensor, torch.Tensor]`: + `tuple[torch.Tensor, torch.Tensor]`: A tuple containing the processed RGB and depth images as PyTorch tensors. """ supported_formats = (PIL.Image.Image, np.ndarray, torch.Tensor) @@ -1396,7 +1391,7 @@ def __init__( ) @staticmethod - def classify_height_width_bin(height: int, width: int, ratios: dict) -> Tuple[int, int]: + def classify_height_width_bin(height: int, width: int, ratios: dict) -> tuple[int, int]: r""" Returns the binned height and width based on the aspect ratio. @@ -1406,7 +1401,7 @@ def classify_height_width_bin(height: int, width: int, ratios: dict) -> Tuple[in ratios (`dict`): A dictionary where keys are aspect ratios and values are tuples of (height, width). Returns: - `Tuple[int, int]`: The closest binned height and width. + `tuple[int, int]`: The closest binned height and width. """ ar = float(height / width) closest_ratio = min(ratios.keys(), key=lambda ratio: abs(float(ratio) - ar)) diff --git a/src/diffusers/loaders/ip_adapter.py b/src/diffusers/loaders/ip_adapter.py index dca4758ba038..fc76aeb7326b 100644 --- a/src/diffusers/loaders/ip_adapter.py +++ b/src/diffusers/loaders/ip_adapter.py @@ -13,7 +13,7 @@ # limitations under the License. from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Optional import torch import torch.nn.functional as F @@ -57,15 +57,15 @@ class IPAdapterMixin: @validate_hf_hub_args def load_ip_adapter( self, - pretrained_model_name_or_path_or_dict: Union[str, List[str], Dict[str, torch.Tensor]], - subfolder: Union[str, List[str]], - weight_name: Union[str, List[str]], + pretrained_model_name_or_path_or_dict: str | list[str] | dict[str, torch.Tensor], + subfolder: str | list[str], + weight_name: str | list[str], image_encoder_folder: Optional[str] = "image_encoder", **kwargs, ): """ Parameters: - pretrained_model_name_or_path_or_dict (`str` or `List[str]` or `os.PathLike` or `List[os.PathLike]` or `dict` or `List[dict]`): + pretrained_model_name_or_path_or_dict (`str` or `list[str]` or `os.PathLike` or `list[os.PathLike]` or `dict` or `list[dict]`): Can be either: - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on @@ -74,10 +74,10 @@ def load_ip_adapter( with [`ModelMixin.save_pretrained`]. - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). - subfolder (`str` or `List[str]`): + subfolder (`str` or `list[str]`): The subfolder location of a model file within a larger model repository on the Hub or locally. If a list is passed, it should have the same length as `weight_name`. - weight_name (`str` or `List[str]`): + weight_name (`str` or `list[str]`): The name of the weight file to load. If a list is passed, it should have the same length as `subfolder`. image_encoder_folder (`str`, *optional*, defaults to `image_encoder`): @@ -94,7 +94,7 @@ def load_ip_adapter( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -358,14 +358,14 @@ class ModularIPAdapterMixin: @validate_hf_hub_args def load_ip_adapter( self, - pretrained_model_name_or_path_or_dict: Union[str, List[str], Dict[str, torch.Tensor]], - subfolder: Union[str, List[str]], - weight_name: Union[str, List[str]], + pretrained_model_name_or_path_or_dict: str | list[str] | dict[str, torch.Tensor], + subfolder: str | list[str], + weight_name: str | list[str], **kwargs, ): """ Parameters: - pretrained_model_name_or_path_or_dict (`str` or `List[str]` or `os.PathLike` or `List[os.PathLike]` or `dict` or `List[dict]`): + pretrained_model_name_or_path_or_dict (`str` or `list[str]` or `os.PathLike` or `list[os.PathLike]` or `dict` or `list[dict]`): Can be either: - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on @@ -374,10 +374,10 @@ def load_ip_adapter( with [`ModelMixin.save_pretrained`]. - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). - subfolder (`str` or `List[str]`): + subfolder (`str` or `list[str]`): The subfolder location of a model file within a larger model repository on the Hub or locally. If a list is passed, it should have the same length as `weight_name`. - weight_name (`str` or `List[str]`): + weight_name (`str` or `list[str]`): The name of the weight file to load. If a list is passed, it should have the same length as `subfolder`. cache_dir (`Union[str, os.PathLike]`, *optional*): @@ -387,7 +387,7 @@ def load_ip_adapter( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -608,9 +608,9 @@ class FluxIPAdapterMixin: @validate_hf_hub_args def load_ip_adapter( self, - pretrained_model_name_or_path_or_dict: Union[str, List[str], Dict[str, torch.Tensor]], - weight_name: Union[str, List[str]], - subfolder: Optional[Union[str, List[str]]] = "", + pretrained_model_name_or_path_or_dict: str | list[str] | dict[str, torch.Tensor], + weight_name: str | list[str], + subfolder: Optional[str | list[str]] = "", image_encoder_pretrained_model_name_or_path: Optional[str] = "image_encoder", image_encoder_subfolder: Optional[str] = "", image_encoder_dtype: torch.dtype = torch.float16, @@ -618,7 +618,7 @@ def load_ip_adapter( ): """ Parameters: - pretrained_model_name_or_path_or_dict (`str` or `List[str]` or `os.PathLike` or `List[os.PathLike]` or `dict` or `List[dict]`): + pretrained_model_name_or_path_or_dict (`str` or `list[str]` or `os.PathLike` or `list[os.PathLike]` or `dict` or `list[dict]`): Can be either: - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on @@ -627,10 +627,10 @@ def load_ip_adapter( with [`ModelMixin.save_pretrained`]. - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). - subfolder (`str` or `List[str]`): + subfolder (`str` or `list[str]`): The subfolder location of a model file within a larger model repository on the Hub or locally. If a list is passed, it should have the same length as `weight_name`. - weight_name (`str` or `List[str]`): + weight_name (`str` or `list[str]`): The name of the weight file to load. If a list is passed, it should have the same length as `weight_name`. image_encoder_pretrained_model_name_or_path (`str`, *optional*, defaults to `./image_encoder`): @@ -647,7 +647,7 @@ def load_ip_adapter( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -797,13 +797,13 @@ def load_ip_adapter( # load ip-adapter into transformer self.transformer._load_ip_adapter_weights(state_dicts, low_cpu_mem_usage=low_cpu_mem_usage) - def set_ip_adapter_scale(self, scale: Union[float, List[float], List[List[float]]]): + def set_ip_adapter_scale(self, scale: float | list[float] | list[list[float]]): """ Set IP-Adapter scales per-transformer block. Input `scale` could be a single config or a list of configs for granular control over each IP-Adapter behavior. A config can be a float or a list. - `float` is converted to list and repeated for the number of blocks and the number of IP adapters. `List[float]` - length match the number of blocks, it is repeated for each IP adapter. `List[List[float]]` must match the + `float` is converted to list and repeated for the number of blocks and the number of IP adapters. `list[float]` + length match the number of blocks, it is repeated for each IP adapter. `list[list[float]]` must match the number of IP adapters and each must match the number of blocks. Example: @@ -823,18 +823,18 @@ def LinearStrengthModel(start, finish, size): ``` """ - scale_type = Union[int, float] + scale_type = int | float num_ip_adapters = self.transformer.encoder_hid_proj.num_ip_adapters num_layers = self.transformer.config.num_layers # Single value for all layers of all IP-Adapters if isinstance(scale, scale_type): scale = [scale for _ in range(num_ip_adapters)] - # List of per-layer scales for a single IP-Adapter - elif _is_valid_type(scale, List[scale_type]) and num_ip_adapters == 1: + # list of per-layer scales for a single IP-Adapter + elif _is_valid_type(scale, list[scale_type]) and num_ip_adapters == 1: scale = [scale] # Invalid scale type - elif not _is_valid_type(scale, List[Union[scale_type, List[scale_type]]]): + elif not _is_valid_type(scale, list[scale_type | list[scale_type]]): raise TypeError(f"Unexpected type {_get_detailed_type(scale)} for scale.") if len(scale) != num_ip_adapters: @@ -918,7 +918,7 @@ def is_ip_adapter_active(self) -> bool: @validate_hf_hub_args def load_ip_adapter( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], weight_name: str = "ip-adapter.safetensors", subfolder: Optional[str] = None, image_encoder_folder: Optional[str] = "image_encoder", @@ -953,7 +953,7 @@ def load_ip_adapter( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 3d75a7d875a4..005298a0fe3f 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -17,7 +17,7 @@ import json import os from pathlib import Path -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import safetensors import torch @@ -77,7 +77,7 @@ def fuse_text_encoder_lora(text_encoder, lora_scale=1.0, safe_fusing=False, adap Controls how much to influence the outputs with the LoRA parameters. safe_fusing (`bool`, defaults to `False`): Whether to check fused weights for NaN values before fusing and if values are NaN not fusing them. - adapter_names (`List[str]` or `str`): + adapter_names (`list[str]` or `str`): The names of the adapters to use. """ merge_kwargs = {"safe_merge": safe_fusing} @@ -116,20 +116,20 @@ def unfuse_text_encoder_lora(text_encoder): def set_adapters_for_text_encoder( - adapter_names: Union[List[str], str], + adapter_names: list[str] | str, text_encoder: Optional["PreTrainedModel"] = None, # noqa: F821 - text_encoder_weights: Optional[Union[float, List[float], List[None]]] = None, + text_encoder_weights: Optional[float | list[float] | list[None]] = None, ): """ Sets the adapter layers for the text encoder. Args: - adapter_names (`List[str]` or `str`): + adapter_names (`list[str]` or `str`): The names of the adapters to use. text_encoder (`torch.nn.Module`, *optional*): The text encoder module to set the adapter layers for. If `None`, it will try to get the `text_encoder` attribute. - text_encoder_weights (`List[float]`, *optional*): + text_encoder_weights (`list[float]`, *optional*): The weights to use for the text encoder. If `None`, the weights are set to `1.0` for all the adapters. """ if text_encoder is None: @@ -535,10 +535,10 @@ def unload_lora_weights(self): def fuse_lora( self, - components: List[str] = [], + components: list[str] = [], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -547,12 +547,12 @@ def fuse_lora( > [!WARNING] > This is an experimental API. Args: - components: (`List[str]`): List of LoRA-injectable components to fuse the LoRAs into. + components: (`list[str]`): list of LoRA-injectable components to fuse the LoRAs into. lora_scale (`float`, defaults to 1.0): Controls how much to influence the outputs with the LoRA parameters. safe_fusing (`bool`, defaults to `False`): Whether to check fused weights for NaN values before fusing and if values are NaN not fusing them. - adapter_names (`List[str]`, *optional*): + adapter_names (`list[str]`, *optional*): Adapter names to be used for fusing. If nothing is passed, all active adapters will be fused. Example: @@ -619,7 +619,7 @@ def fuse_lora( self._merged_adapters = self._merged_adapters | merged_adapter_names - def unfuse_lora(self, components: List[str] = [], **kwargs): + def unfuse_lora(self, components: list[str] = [], **kwargs): r""" Reverses the effect of [`pipe.fuse_lora()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraBaseMixin.fuse_lora). @@ -627,7 +627,7 @@ def unfuse_lora(self, components: List[str] = [], **kwargs): > [!WARNING] > This is an experimental API. Args: - components (`List[str]`): List of LoRA-injectable components to unfuse LoRA from. + components (`list[str]`): list of LoRA-injectable components to unfuse LoRA from. unfuse_unet (`bool`, defaults to `True`): Whether to unfuse the UNet LoRA parameters. unfuse_text_encoder (`bool`, defaults to `True`): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the @@ -674,16 +674,16 @@ def unfuse_lora(self, components: List[str] = [], **kwargs): def set_adapters( self, - adapter_names: Union[List[str], str], - adapter_weights: Optional[Union[float, Dict, List[float], List[Dict]]] = None, + adapter_names: list[str] | str, + adapter_weights: Optional[float | Dict | list[float] | list[Dict]] = None, ): """ Set the currently active adapters for use in the pipeline. Args: - adapter_names (`List[str]` or `str`): + adapter_names (`list[str]` or `str`): The names of the adapters to use. - adapter_weights (`Union[List[float], float]`, *optional*): + adapter_weights (`Union[list[float], float]`, *optional*): The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the adapters. @@ -835,12 +835,12 @@ def enable_lora(self): elif issubclass(model.__class__, PreTrainedModel): enable_lora_for_text_encoder(model) - def delete_adapters(self, adapter_names: Union[List[str], str]): + def delete_adapters(self, adapter_names: list[str] | str): """ Delete an adapter's LoRA layers from the pipeline. Args: - adapter_names (`Union[List[str], str]`): + adapter_names (`Union[list[str], str]`): The names of the adapters to delete. Example: @@ -873,7 +873,7 @@ def delete_adapters(self, adapter_names: Union[List[str], str]): for adapter_name in adapter_names: delete_adapter_layers(model, adapter_name) - def get_active_adapters(self) -> List[str]: + def get_active_adapters(self) -> list[str]: """ Gets the list of the current active adapters. @@ -906,7 +906,7 @@ def get_active_adapters(self) -> List[str]: return active_adapters - def get_list_adapters(self) -> Dict[str, List[str]]: + def get_list_adapters(self) -> dict[str, list[str]]: """ Gets the current list of all available adapters in the pipeline. """ @@ -928,7 +928,7 @@ def get_list_adapters(self) -> Dict[str, List[str]]: return set_adapters - def set_lora_device(self, adapter_names: List[str], device: Union[torch.device, str, int]) -> None: + def set_lora_device(self, adapter_names: list[str], device: torch.device | str | int) -> None: """ Moves the LoRAs listed in `adapter_names` to a target device. Useful for offloading the LoRA to the CPU in case you want to load multiple adapters and free some GPU memory. @@ -955,8 +955,8 @@ def set_lora_device(self, adapter_names: List[str], device: Union[torch.device, ``` Args: - adapter_names (`List[str]`): - List of adapters to send device to. + adapter_names (`list[str]`): + list of adapters to send device to. device (`Union[torch.device, str, int]`): Device to send the adapters to. Can be either a torch device, a str or an integer. """ @@ -1007,7 +1007,7 @@ def pack_weights(layers, prefix): @staticmethod def write_lora_layers( - state_dict: Dict[str, torch.Tensor], + state_dict: dict[str, torch.Tensor], save_directory: str, is_main_process: bool, weight_name: str, @@ -1059,9 +1059,9 @@ def save_function(weights, filename): @classmethod def _save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - lora_layers: Dict[str, Dict[str, Union[torch.nn.Module, torch.Tensor]]], - lora_metadata: Dict[str, Optional[dict]], + save_directory: str | os.PathLike, + lora_layers: dict[str, dict[str, torch.nn.Module | torch.Tensor]], + lora_metadata: dict[str, Optional[dict]], is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, diff --git a/src/diffusers/loaders/lora_conversion_utils.py b/src/diffusers/loaders/lora_conversion_utils.py index 89afb6529a50..cedfd8de2f19 100644 --- a/src/diffusers/loaders/lora_conversion_utils.py +++ b/src/diffusers/loaders/lora_conversion_utils.py @@ -13,7 +13,6 @@ # limitations under the License. import re -from typing import List import torch @@ -1021,7 +1020,7 @@ def handle_qkv(sds_sd, ait_sd, sds_key, ait_keys, dims=None): return new_state_dict -def _custom_replace(key: str, substrings: List[str]) -> str: +def _custom_replace(key: str, substrings: list[str]) -> str: # Replaces the "."s with "_"s upto the `substrings`. # Example: # lora_unet.foo.bar.lora_A.weight -> lora_unet_foo_bar.lora_A.weight diff --git a/src/diffusers/loaders/lora_pipeline.py b/src/diffusers/loaders/lora_pipeline.py index 2bb6c0ea026e..d7e38369865b 100644 --- a/src/diffusers/loaders/lora_pipeline.py +++ b/src/diffusers/loaders/lora_pipeline.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Optional import torch from huggingface_hub.utils import validate_hf_hub_args @@ -137,7 +137,7 @@ class StableDiffusionLoraLoaderMixin(LoraBaseMixin): def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -240,7 +240,7 @@ def load_lora_weights( @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -267,7 +267,7 @@ def lora_state_dict( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -367,7 +367,7 @@ def load_lora_into_unet( A standard state dict containing the lora layer parameters. The keys can either be indexed directly into the unet or prefixed with an additional `unet` which can be used to distinguish between text encoder lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -429,7 +429,7 @@ def load_lora_into_text_encoder( state_dict (`dict`): A standard state dict containing the lora layer parameters. The key should be prefixed with an additional `text_encoder` to distinguish between unet lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -469,9 +469,9 @@ def load_lora_into_text_encoder( @classmethod def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_lora_layers: Dict[str, torch.nn.Module] = None, + save_directory: str | os.PathLike, + unet_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_lora_layers: dict[str, torch.nn.Module] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -485,9 +485,9 @@ def save_lora_weights( Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. Will be created if it doesn't exist. - unet_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + unet_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `unet`. - text_encoder_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + text_encoder_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `text_encoder`. Must explicitly pass the text encoder LoRA state dict because it comes from 🤗 Transformers. is_main_process (`bool`, *optional*, defaults to `True`): @@ -531,10 +531,10 @@ def save_lora_weights( def fuse_lora( self, - components: List[str] = ["unet", "text_encoder"], + components: list[str] = ["unet", "text_encoder"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -543,12 +543,12 @@ def fuse_lora( > [!WARNING] > This is an experimental API. Args: - components: (`List[str]`): List of LoRA-injectable components to fuse the LoRAs into. + components: (`list[str]`): list of LoRA-injectable components to fuse the LoRAs into. lora_scale (`float`, defaults to 1.0): Controls how much to influence the outputs with the LoRA parameters. safe_fusing (`bool`, defaults to `False`): Whether to check fused weights for NaN values before fusing and if values are NaN not fusing them. - adapter_names (`List[str]`, *optional*): + adapter_names (`list[str]`, *optional*): Adapter names to be used for fusing. If nothing is passed, all active adapters will be fused. Example: @@ -572,7 +572,7 @@ def fuse_lora( **kwargs, ) - def unfuse_lora(self, components: List[str] = ["unet", "text_encoder"], **kwargs): + def unfuse_lora(self, components: list[str] = ["unet", "text_encoder"], **kwargs): r""" Reverses the effect of [`pipe.fuse_lora()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraBaseMixin.fuse_lora). @@ -580,7 +580,7 @@ def unfuse_lora(self, components: List[str] = ["unet", "text_encoder"], **kwargs > [!WARNING] > This is an experimental API. Args: - components (`List[str]`): List of LoRA-injectable components to unfuse LoRA from. + components (`list[str]`): list of LoRA-injectable components to unfuse LoRA from. unfuse_unet (`bool`, defaults to `True`): Whether to unfuse the UNet LoRA parameters. unfuse_text_encoder (`bool`, defaults to `True`): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the @@ -602,7 +602,7 @@ class StableDiffusionXLLoraLoaderMixin(LoraBaseMixin): def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -679,7 +679,7 @@ def load_lora_weights( # Copied from diffusers.loaders.lora_pipeline.StableDiffusionLoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -706,7 +706,7 @@ def lora_state_dict( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -807,7 +807,7 @@ def load_lora_into_unet( A standard state dict containing the lora layer parameters. The keys can either be indexed directly into the unet or prefixed with an additional `unet` which can be used to distinguish between text encoder lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -870,7 +870,7 @@ def load_lora_into_text_encoder( state_dict (`dict`): A standard state dict containing the lora layer parameters. The key should be prefixed with an additional `text_encoder` to distinguish between unet lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -910,10 +910,10 @@ def load_lora_into_text_encoder( @classmethod def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + unet_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_2_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -957,10 +957,10 @@ def save_lora_weights( def fuse_lora( self, - components: List[str] = ["unet", "text_encoder", "text_encoder_2"], + components: list[str] = ["unet", "text_encoder", "text_encoder_2"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -974,7 +974,7 @@ def fuse_lora( **kwargs, ) - def unfuse_lora(self, components: List[str] = ["unet", "text_encoder", "text_encoder_2"], **kwargs): + def unfuse_lora(self, components: list[str] = ["unet", "text_encoder", "text_encoder_2"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -998,7 +998,7 @@ class SD3LoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -1050,7 +1050,7 @@ def lora_state_dict( def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name=None, hotswap: bool = False, **kwargs, @@ -1166,7 +1166,7 @@ def load_lora_into_text_encoder( state_dict (`dict`): A standard state dict containing the lora layer parameters. The key should be prefixed with an additional `text_encoder` to distinguish between unet lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -1207,10 +1207,10 @@ def load_lora_into_text_encoder( # Copied from diffusers.loaders.lora_pipeline.StableDiffusionXLLoraLoaderMixin.save_lora_weights with unet->transformer def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_2_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -1255,10 +1255,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.StableDiffusionXLLoraLoaderMixin.fuse_lora with unet->transformer def fuse_lora( self, - components: List[str] = ["transformer", "text_encoder", "text_encoder_2"], + components: list[str] = ["transformer", "text_encoder", "text_encoder_2"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -1273,7 +1273,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.StableDiffusionXLLoraLoaderMixin.unfuse_lora with unet->transformer - def unfuse_lora(self, components: List[str] = ["transformer", "text_encoder", "text_encoder_2"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer", "text_encoder", "text_encoder_2"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -1293,7 +1293,7 @@ class AuraFlowLoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -1346,7 +1346,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -1421,8 +1421,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -1455,10 +1455,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -1473,7 +1473,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer", "text_encoder"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer", "text_encoder"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -1497,7 +1497,7 @@ class FluxLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], return_alphas: bool = False, **kwargs, ): @@ -1620,7 +1620,7 @@ def lora_state_dict( def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -1782,7 +1782,7 @@ def _load_norm_into_transformer( transformer, prefix=None, discard_original_layers=False, - ) -> Dict[str, torch.Tensor]: + ) -> dict[str, torch.Tensor]: # Remove prefix if present prefix = prefix or cls.transformer_name for key in list(state_dict.keys()): @@ -1851,7 +1851,7 @@ def load_lora_into_text_encoder( state_dict (`dict`): A standard state dict containing the lora layer parameters. The key should be prefixed with an additional `text_encoder` to distinguish between unet lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -1892,9 +1892,9 @@ def load_lora_into_text_encoder( # Copied from diffusers.loaders.lora_pipeline.StableDiffusionLoraLoaderMixin.save_lora_weights with unet->transformer def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_lora_layers: Dict[str, torch.nn.Module] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, + text_encoder_lora_layers: dict[str, torch.nn.Module] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -1908,9 +1908,9 @@ def save_lora_weights( Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. Will be created if it doesn't exist. - transformer_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + transformer_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `transformer`. - text_encoder_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + text_encoder_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `text_encoder`. Must explicitly pass the text encoder LoRA state dict because it comes from 🤗 Transformers. is_main_process (`bool`, *optional*, defaults to `True`): @@ -1954,10 +1954,10 @@ def save_lora_weights( def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -1984,7 +1984,7 @@ def fuse_lora( **kwargs, ) - def unfuse_lora(self, components: List[str] = ["transformer", "text_encoder"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer", "text_encoder"], **kwargs): r""" Reverses the effect of [`pipe.fuse_lora()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraBaseMixin.fuse_lora). @@ -1992,7 +1992,7 @@ def unfuse_lora(self, components: List[str] = ["transformer", "text_encoder"], * > [!WARNING] > This is an experimental API. Args: - components (`List[str]`): List of LoRA-injectable components to unfuse LoRA from. + components (`list[str]`): list of LoRA-injectable components to unfuse LoRA from. """ transformer = getattr(self, self.transformer_name) if not hasattr(self, "transformer") else self.transformer if hasattr(transformer, "_transformer_norm_layers") and transformer._transformer_norm_layers: @@ -2341,7 +2341,7 @@ def load_lora_into_text_encoder( state_dict (`dict`): A standard state dict containing the lora layer parameters. The key should be prefixed with an additional `text_encoder` to distinguish between unet lora layers. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -2381,9 +2381,9 @@ def load_lora_into_text_encoder( @classmethod def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - text_encoder_lora_layers: Dict[str, torch.nn.Module] = None, - transformer_lora_layers: Dict[str, torch.nn.Module] = None, + save_directory: str | os.PathLike, + text_encoder_lora_layers: dict[str, torch.nn.Module] = None, + transformer_lora_layers: dict[str, torch.nn.Module] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -2395,9 +2395,9 @@ def save_lora_weights( Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. Will be created if it doesn't exist. - unet_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + unet_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `unet`. - text_encoder_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + text_encoder_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `text_encoder`. Must explicitly pass the text encoder LoRA state dict because it comes from 🤗 Transformers. is_main_process (`bool`, *optional*, defaults to `True`): @@ -2446,7 +2446,7 @@ class CogVideoXLoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.SD3LoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -2498,7 +2498,7 @@ def lora_state_dict( def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -2572,8 +2572,8 @@ def load_lora_into_transformer( @classmethod def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -2605,10 +2605,10 @@ def save_lora_weights( def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -2622,7 +2622,7 @@ def fuse_lora( **kwargs, ) - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -2642,7 +2642,7 @@ class Mochi1LoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.SD3LoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -2695,7 +2695,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -2770,8 +2770,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -2804,10 +2804,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -2822,7 +2822,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -2841,7 +2841,7 @@ class LTXVideoLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -2898,7 +2898,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -2973,8 +2973,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -3007,10 +3007,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -3025,7 +3025,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -3045,7 +3045,7 @@ class SanaLoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.SD3LoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -3098,7 +3098,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -3173,8 +3173,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -3207,10 +3207,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -3225,7 +3225,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -3244,7 +3244,7 @@ class HunyuanVideoLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -3301,7 +3301,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -3376,8 +3376,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -3410,10 +3410,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -3428,7 +3428,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -3447,7 +3447,7 @@ class Lumina2LoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -3505,7 +3505,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -3580,8 +3580,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -3614,10 +3614,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -3632,7 +3632,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -3651,7 +3651,7 @@ class KandinskyLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -3669,7 +3669,7 @@ def lora_state_dict( Path to a directory where a downloaded pretrained model configuration is cached. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint. local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files. @@ -3731,7 +3731,7 @@ def lora_state_dict( def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -3832,8 +3832,8 @@ def load_lora_into_transformer( @classmethod def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -3846,7 +3846,7 @@ def save_lora_weights( Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. - transformer_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + transformer_lora_layers (`dict[str, torch.nn.Module]` or `dict[str, torch.Tensor]`): State dict of the LoRA layers corresponding to the `transformer`. is_main_process (`bool`, *optional*, defaults to `True`): Whether the process calling this is the main process. @@ -3879,22 +3879,22 @@ def save_lora_weights( def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" Fuses the LoRA parameters into the original parameters of the corresponding blocks. Args: - components: (`List[str]`): List of LoRA-injectable components to fuse the LoRAs into. + components: (`list[str]`): list of LoRA-injectable components to fuse the LoRAs into. lora_scale (`float`, defaults to 1.0): Controls how much to influence the outputs with the LoRA parameters. safe_fusing (`bool`, defaults to `False`): Whether to check fused weights for NaN values before fusing. - adapter_names (`List[str]`, *optional*): + adapter_names (`list[str]`, *optional*): Adapter names to be used for fusing. Example: @@ -3914,12 +3914,12 @@ def fuse_lora( **kwargs, ) - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" Reverses the effect of [`pipe.fuse_lora()`]. Args: - components (`List[str]`): List of LoRA-injectable components to unfuse LoRA from. + components (`list[str]`): list of LoRA-injectable components to unfuse LoRA from. """ super().unfuse_lora(components=components, **kwargs) @@ -3936,7 +3936,7 @@ class WanLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -4040,7 +4040,7 @@ def _maybe_expand_t2v_lora_for_i2v( def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -4139,8 +4139,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -4173,10 +4173,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -4191,7 +4191,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -4211,7 +4211,7 @@ class SkyReelsV2LoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.WanLoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -4317,7 +4317,7 @@ def _maybe_expand_t2v_lora_for_i2v( # Copied from diffusers.loaders.lora_pipeline.WanLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -4416,8 +4416,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -4450,10 +4450,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -4468,7 +4468,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -4488,7 +4488,7 @@ class CogView4LoraLoaderMixin(LoraBaseMixin): # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.lora_state_dict def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -4541,7 +4541,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -4616,8 +4616,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -4650,10 +4650,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -4668,7 +4668,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -4687,7 +4687,7 @@ class HiDreamImageLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -4744,7 +4744,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -4819,8 +4819,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -4853,10 +4853,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -4871,7 +4871,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.SanaLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ @@ -4890,7 +4890,7 @@ class QwenImageLoraLoaderMixin(LoraBaseMixin): @validate_hf_hub_args def lora_state_dict( cls, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs, ): r""" @@ -4949,7 +4949,7 @@ def lora_state_dict( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.load_lora_weights def load_lora_weights( self, - pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], adapter_name: Optional[str] = None, hotswap: bool = False, **kwargs, @@ -5024,8 +5024,8 @@ def load_lora_into_transformer( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.save_lora_weights def save_lora_weights( cls, - save_directory: Union[str, os.PathLike], - transformer_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + save_directory: str | os.PathLike, + transformer_lora_layers: dict[str, torch.nn.Module | torch.Tensor] = None, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, @@ -5058,10 +5058,10 @@ def save_lora_weights( # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.fuse_lora def fuse_lora( self, - components: List[str] = ["transformer"], + components: list[str] = ["transformer"], lora_scale: float = 1.0, safe_fusing: bool = False, - adapter_names: Optional[List[str]] = None, + adapter_names: Optional[list[str]] = None, **kwargs, ): r""" @@ -5076,7 +5076,7 @@ def fuse_lora( ) # Copied from diffusers.loaders.lora_pipeline.CogVideoXLoraLoaderMixin.unfuse_lora - def unfuse_lora(self, components: List[str] = ["transformer"], **kwargs): + def unfuse_lora(self, components: list[str] = ["transformer"], **kwargs): r""" See [`~loaders.StableDiffusionLoraLoaderMixin.unfuse_lora`] for more details. """ diff --git a/src/diffusers/loaders/peft.py b/src/diffusers/loaders/peft.py index 7d65b30659fb..729bbabf2500 100644 --- a/src/diffusers/loaders/peft.py +++ b/src/diffusers/loaders/peft.py @@ -17,7 +17,7 @@ import os from functools import partial from pathlib import Path -from typing import Dict, List, Literal, Optional, Union +from typing import Dict, Literal, Optional import safetensors import torch @@ -113,7 +113,7 @@ def load_lora_adapter( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -127,7 +127,7 @@ def load_lora_adapter( allowed by Git. subfolder (`str`, *optional*, defaults to `""`): The subfolder location of a model file within a larger model repository on the Hub or locally. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -447,16 +447,16 @@ def save_function(weights, filename): def set_adapters( self, - adapter_names: Union[List[str], str], - weights: Optional[Union[float, Dict, List[float], List[Dict], List[None]]] = None, + adapter_names: list[str] | str, + weights: Optional[float | Dict | list[float] | list[Dict] | list[None]] = None, ): """ Set the currently active adapters for use in the diffusion network (e.g. unet, transformer, etc.). Args: - adapter_names (`List[str]` or `str`): + adapter_names (`list[str]` or `str`): The names of the adapters to use. - adapter_weights (`Union[List[float], float]`, *optional*): + adapter_weights (`Union[list[float], float]`, *optional*): The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the adapters. @@ -539,7 +539,7 @@ def add_adapter(self, adapter_config, adapter_name: str = "default") -> None: inject_adapter_in_model(adapter_config, self, adapter_name) self.set_adapter(adapter_name) - def set_adapter(self, adapter_name: Union[str, List[str]]) -> None: + def set_adapter(self, adapter_name: str | list[str]) -> None: """ Sets a specific adapter by forcing the model to only use that adapter and disables the other adapters. @@ -547,7 +547,7 @@ def set_adapter(self, adapter_name: Union[str, List[str]]) -> None: [documentation](https://huggingface.co/docs/peft). Args: - adapter_name (Union[str, List[str]])): + adapter_name (Union[str, list[str]])): The list of adapters to set or the adapter name in the case of a single adapter. """ check_peft_version(min_version=MIN_PEFT_VERSION) @@ -633,7 +633,7 @@ def enable_adapters(self) -> None: # support for older PEFT versions module.disable_adapters = False - def active_adapters(self) -> List[str]: + def active_adapters(self) -> list[str]: """ Gets the current list of active adapters of the model. @@ -756,12 +756,12 @@ def enable_lora(self): raise ValueError("PEFT backend is required for this method.") set_adapter_layers(self, enabled=True) - def delete_adapters(self, adapter_names: Union[List[str], str]): + def delete_adapters(self, adapter_names: list[str] | str): """ Delete an adapter's LoRA layers from the underlying model. Args: - adapter_names (`Union[List[str], str]`): + adapter_names (`Union[list[str], str]`): The names (single string or list of strings) of the adapter to delete. Example: diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py index 667f79437985..0dcb5b3010fa 100644 --- a/src/diffusers/loaders/single_file.py +++ b/src/diffusers/loaders/single_file.py @@ -290,7 +290,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs) -> Self: Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): diff --git a/src/diffusers/loaders/single_file_model.py b/src/diffusers/loaders/single_file_model.py index b53647d47630..3275215bfdf7 100644 --- a/src/diffusers/loaders/single_file_model.py +++ b/src/diffusers/loaders/single_file_model.py @@ -229,7 +229,7 @@ def from_single_file(cls, pretrained_model_link_or_path_or_dict: Optional[str] = Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 63fc97ed431f..1ba6a1742284 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Union +from typing import Optional import safetensors import torch @@ -112,7 +112,7 @@ class TextualInversionLoaderMixin: Load Textual Inversion tokens and embeddings to the tokenizer and text encoder. """ - def maybe_convert_prompt(self, prompt: Union[str, List[str]], tokenizer: "PreTrainedTokenizer"): # noqa: F821 + def maybe_convert_prompt(self, prompt: str | list[str], tokenizer: "PreTrainedTokenizer"): # noqa: F821 r""" Processes prompts that include a special token corresponding to a multi-vector textual inversion embedding to be replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual @@ -127,14 +127,14 @@ def maybe_convert_prompt(self, prompt: Union[str, List[str]], tokenizer: "PreTra Returns: `str` or list of `str`: The converted prompt """ - if not isinstance(prompt, List): + if not isinstance(prompt, list): prompts = [prompt] else: prompts = prompt prompts = [self._maybe_convert_prompt(p, tokenizer) for p in prompts] - if not isinstance(prompt, List): + if not isinstance(prompt, list): return prompts[0] return prompts @@ -263,8 +263,8 @@ def _extend_tokens_and_embeddings(tokens, embeddings, tokenizer): @validate_hf_hub_args def load_textual_inversion( self, - pretrained_model_name_or_path: Union[str, List[str], Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]], - token: Optional[Union[str, List[str]]] = None, + pretrained_model_name_or_path: str | list[str] | dict[str, torch.Tensor] | list[dict[str, torch.Tensor]], + token: Optional[str | list[str]] = None, tokenizer: Optional["PreTrainedTokenizer"] = None, # noqa: F821 text_encoder: Optional["PreTrainedModel"] = None, # noqa: F821 **kwargs, @@ -274,7 +274,7 @@ def load_textual_inversion( Automatic1111 formats are supported). Parameters: - pretrained_model_name_or_path (`str` or `os.PathLike` or `List[str or os.PathLike]` or `Dict` or `List[Dict]`): + pretrained_model_name_or_path (`str` or `os.PathLike` or `list[str or os.PathLike]` or `Dict` or `list[Dict]`): Can be either one of the following or a list of them: - A string, the *model id* (for example `sd-concepts-library/low-poly-hd-logos-icons`) of a @@ -285,7 +285,7 @@ def load_textual_inversion( - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). - token (`str` or `List[str]`, *optional*): + token (`str` or `list[str]`, *optional*): Override the token to use for the textual inversion weights. If `pretrained_model_name_or_path` is a list, then `token` must also be a list of equal length. text_encoder ([`~transformers.CLIPTextModel`], *optional*): @@ -306,7 +306,7 @@ def load_textual_inversion( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -458,7 +458,7 @@ def load_textual_inversion( def unload_textual_inversion( self, - tokens: Optional[Union[str, List[str]]] = None, + tokens: Optional[str | list[str]] = None, tokenizer: Optional["PreTrainedTokenizer"] = None, text_encoder: Optional["PreTrainedModel"] = None, ): diff --git a/src/diffusers/loaders/unet.py b/src/diffusers/loaders/unet.py index c5e56af156fc..603cc5defc76 100644 --- a/src/diffusers/loaders/unet.py +++ b/src/diffusers/loaders/unet.py @@ -15,7 +15,7 @@ from collections import defaultdict from contextlib import nullcontext from pathlib import Path -from typing import Callable, Dict, Union +from typing import Callable import safetensors import torch @@ -66,7 +66,7 @@ class UNet2DConditionLoadersMixin: unet_name = UNET_NAME @validate_hf_hub_args - def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): + def load_attn_procs(self, pretrained_model_name_or_path_or_dict: str | dict[str, torch.Tensor], **kwargs): r""" Load pretrained attention processor layers into [`UNet2DConditionModel`]. Attention processor layers have to be defined in @@ -92,7 +92,7 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): @@ -106,7 +106,7 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict allowed by Git. subfolder (`str`, *optional*, defaults to `""`): The subfolder location of a model file within a larger model repository on the Hub or locally. - network_alphas (`Dict[str, float]`): + network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). @@ -412,7 +412,7 @@ def _optionally_disable_offloading(cls, _pipeline): def save_attn_procs( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, is_main_process: bool = True, weight_name: str = None, save_function: Callable = None, diff --git a/src/diffusers/loaders/unet_loader_utils.py b/src/diffusers/loaders/unet_loader_utils.py index d5b0e83cbd9e..4b265b9bec5f 100644 --- a/src/diffusers/loaders/unet_loader_utils.py +++ b/src/diffusers/loaders/unet_loader_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -from typing import TYPE_CHECKING, Dict, List, Union +from typing import TYPE_CHECKING, Dict from torch import nn @@ -40,9 +40,7 @@ def _translate_into_actual_layer_name(name): return ".".join((updown, block, attn)) -def _maybe_expand_lora_scales( - unet: "UNet2DConditionModel", weight_scales: List[Union[float, Dict]], default_scale=1.0 -): +def _maybe_expand_lora_scales(unet: "UNet2DConditionModel", weight_scales: list[float | Dict], default_scale=1.0): blocks_with_transformer = { "down": [i for i, block in enumerate(unet.down_blocks) if hasattr(block, "attentions")], "up": [i for i, block in enumerate(unet.up_blocks) if hasattr(block, "attentions")], @@ -64,9 +62,9 @@ def _maybe_expand_lora_scales( def _maybe_expand_lora_scales_for_one_adapter( - scales: Union[float, Dict], - blocks_with_transformer: Dict[str, int], - transformer_per_block: Dict[str, int], + scales: float | Dict, + blocks_with_transformer: dict[str, int], + transformer_per_block: dict[str, int], model: nn.Module, default_scale: float = 1.0, ): @@ -76,9 +74,9 @@ def _maybe_expand_lora_scales_for_one_adapter( Parameters: scales (`Union[float, Dict]`): Scales dict to expand. - blocks_with_transformer (`Dict[str, int]`): + blocks_with_transformer (`dict[str, int]`): Dict with keys 'up' and 'down', showing which blocks have transformer layers - transformer_per_block (`Dict[str, int]`): + transformer_per_block (`dict[str, int]`): Dict with keys 'up' and 'down', showing how many transformer layers each block has E.g. turns diff --git a/src/diffusers/loaders/utils.py b/src/diffusers/loaders/utils.py index 2d39e7bfb7d2..72d6453a8bbf 100644 --- a/src/diffusers/loaders/utils.py +++ b/src/diffusers/loaders/utils.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict import torch class AttnProcsLayers(torch.nn.Module): - def __init__(self, state_dict: Dict[str, torch.Tensor]): + def __init__(self, state_dict: dict[str, torch.Tensor]): super().__init__() self.layers = torch.nn.ModuleList(state_dict.values()) self.mapping = dict(enumerate(state_dict.keys())) diff --git a/src/diffusers/models/_modeling_parallel.py b/src/diffusers/models/_modeling_parallel.py index 2a1d2cc6ceea..3361f15fe847 100644 --- a/src/diffusers/models/_modeling_parallel.py +++ b/src/diffusers/models/_modeling_parallel.py @@ -16,7 +16,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Literal, Optional import torch @@ -187,19 +187,17 @@ def __repr__(self): # If the key is a string, it denotes the name of the parameter in the forward function. # If the key is an integer, split_output must be set to True, and it denotes the index of the output # to be split across context parallel region. -ContextParallelInputType = Dict[ - Union[str, int], Union[ContextParallelInput, List[ContextParallelInput], Tuple[ContextParallelInput, ...]] +ContextParallelInputType = dict[ + str | int, ContextParallelInput | list[ContextParallelInput] | tuple[ContextParallelInput, ...] ] # A dictionary where keys denote the output to be gathered across context parallel region, and the # value denotes the gathering configuration. -ContextParallelOutputType = Union[ - ContextParallelOutput, List[ContextParallelOutput], Tuple[ContextParallelOutput, ...] -] +ContextParallelOutputType = ContextParallelOutput | list[ContextParallelOutput] | tuple[ContextParallelOutput, ...] # A dictionary where keys denote the module id, and the value denotes how the inputs/outputs of # the module should be split/gathered across context parallel region. -ContextParallelModelPlan = Dict[str, Union[ContextParallelInputType, ContextParallelOutputType]] +ContextParallelModelPlan = dict[str, ContextParallelInputType | ContextParallelOutputType] # Example of a ContextParallelModelPlan (QwenImageTransformer2DModel): diff --git a/src/diffusers/models/adapter.py b/src/diffusers/models/adapter.py index e475fe6bee88..a432965256f3 100644 --- a/src/diffusers/models/adapter.py +++ b/src/diffusers/models/adapter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch import torch.nn as nn @@ -34,11 +34,11 @@ class MultiAdapter(ModelMixin): or saving. Args: - adapters (`List[T2IAdapter]`, *optional*, defaults to None): + adapters (`list[T2IAdapter]`, *optional*, defaults to None): A list of `T2IAdapter` model instances. """ - def __init__(self, adapters: List["T2IAdapter"]): + def __init__(self, adapters: list["T2IAdapter"]): super(MultiAdapter, self).__init__() self.num_adapter = len(adapters) @@ -73,7 +73,7 @@ def __init__(self, adapters: List["T2IAdapter"]): self.total_downscale_factor = first_adapter_total_downscale_factor self.downscale_factor = first_adapter_downscale_factor - def forward(self, xs: torch.Tensor, adapter_weights: Optional[List[float]] = None) -> List[torch.Tensor]: + def forward(self, xs: torch.Tensor, adapter_weights: Optional[list[float]] = None) -> list[torch.Tensor]: r""" Args: xs (`torch.Tensor`): @@ -81,7 +81,7 @@ def forward(self, xs: torch.Tensor, adapter_weights: Optional[List[float]] = Non models, concatenated along dimension 1(channel dimension). The `channel` dimension should be equal to `num_adapter` * number of channel per image. - adapter_weights (`List[float]`, *optional*, defaults to None): + adapter_weights (`list[float]`, *optional*, defaults to None): A list of floats representing the weights which will be multiplied by each adapter's output before summing them together. If `None`, equal weights will be used for all adapters. """ @@ -104,7 +104,7 @@ def forward(self, xs: torch.Tensor, adapter_weights: Optional[List[float]] = Non def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, is_main_process: bool = True, save_function: Callable = None, safe_serialization: bool = True, @@ -145,7 +145,7 @@ def save_pretrained( model_path_to_save = model_path_to_save + f"_{idx}" @classmethod - def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike]], **kwargs): + def from_pretrained(cls, pretrained_model_path: Optional[str | os.PathLike], **kwargs): r""" Instantiate a pretrained `MultiAdapter` model from multiple pre-trained adapter models. @@ -165,7 +165,7 @@ def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike] Override the default `torch.dtype` and load the model under this dtype. output_loading_info(`bool`, *optional*, defaults to `False`): Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. @@ -229,7 +229,7 @@ class T2IAdapter(ModelMixin, ConfigMixin): in_channels (`int`, *optional*, defaults to `3`): The number of channels in the adapter's input (*control image*). Set it to 1 if you're using a gray scale image. - channels (`List[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + channels (`list[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The number of channels in each downsample block's output hidden state. The `len(block_out_channels)` determines the number of downsample blocks in the adapter. num_res_blocks (`int`, *optional*, defaults to `2`): @@ -244,7 +244,7 @@ class T2IAdapter(ModelMixin, ConfigMixin): def __init__( self, in_channels: int = 3, - channels: List[int] = [320, 640, 1280, 1280], + channels: list[int] = [320, 640, 1280, 1280], num_res_blocks: int = 2, downscale_factor: int = 8, adapter_type: str = "full_adapter", @@ -263,7 +263,7 @@ def __init__( "'full_adapter_xl' or 'light_adapter'." ) - def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: r""" This function processes the input tensor `x` through the adapter model and returns a list of feature tensors, each representing information extracted at a different scale from the input. The length of the list is @@ -295,7 +295,7 @@ class FullAdapter(nn.Module): def __init__( self, in_channels: int = 3, - channels: List[int] = [320, 640, 1280, 1280], + channels: list[int] = [320, 640, 1280, 1280], num_res_blocks: int = 2, downscale_factor: int = 8, ): @@ -318,7 +318,7 @@ def __init__( self.total_downscale_factor = downscale_factor * 2 ** (len(channels) - 1) - def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: r""" This method processes the input tensor `x` through the FullAdapter model and performs operations including pixel unshuffling, convolution, and a stack of AdapterBlocks. It returns a list of feature tensors, each @@ -345,7 +345,7 @@ class FullAdapterXL(nn.Module): def __init__( self, in_channels: int = 3, - channels: List[int] = [320, 640, 1280, 1280], + channels: list[int] = [320, 640, 1280, 1280], num_res_blocks: int = 2, downscale_factor: int = 16, ): @@ -370,7 +370,7 @@ def __init__( # XL has only one downsampling AdapterBlock. self.total_downscale_factor = downscale_factor * 2 - def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: r""" This method takes the tensor x as input and processes it through FullAdapterXL model. It consists of operations including unshuffling pixels, applying convolution layer and appending each block into list of feature tensors. @@ -473,7 +473,7 @@ class LightAdapter(nn.Module): def __init__( self, in_channels: int = 3, - channels: List[int] = [320, 640, 1280], + channels: list[int] = [320, 640, 1280], num_res_blocks: int = 4, downscale_factor: int = 8, ): @@ -496,7 +496,7 @@ def __init__( self.total_downscale_factor = downscale_factor * (2 ** len(channels)) - def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: r""" This method takes the input tensor x and performs downscaling and appends it in list of feature tensors. Each feature tensor corresponds to a different level of processing within the LightAdapter. diff --git a/src/diffusers/models/attention.py b/src/diffusers/models/attention.py index 5164cf311d3c..53f5f026383c 100644 --- a/src/diffusers/models/attention.py +++ b/src/diffusers/models/attention.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch import torch.nn as nn @@ -38,7 +38,7 @@ class AttentionMixin: @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -47,7 +47,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -61,7 +61,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -184,7 +184,7 @@ def set_use_npu_flash_attention(self, use_npu_flash_attention: bool) -> None: def set_use_xla_flash_attention( self, use_xla_flash_attention: bool, - partition_spec: Optional[Tuple[Optional[str], ...]] = None, + partition_spec: Optional[tuple[Optional[str], ...]] = None, is_flux=False, ) -> None: """ @@ -193,7 +193,7 @@ def set_use_xla_flash_attention( Args: use_xla_flash_attention (`bool`): Whether to use pallas flash attention kernel from `torch_xla` or not. - partition_spec (`Tuple[]`, *optional*): + partition_spec (`tuple[]`, *optional*): Specify the partition specification if using SPMD. Otherwise None. is_flux (`bool`, *optional*, defaults to `False`): Whether the model is a Flux model. @@ -669,8 +669,8 @@ def forward( hidden_states: torch.FloatTensor, encoder_hidden_states: torch.FloatTensor, temb: torch.FloatTensor, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: joint_attention_kwargs = joint_attention_kwargs or {} if self.use_dual_attention: norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp, norm_hidden_states2, gate_msa2 = self.norm1( @@ -950,9 +950,9 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, timestep: Optional[torch.LongTensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, class_labels: Optional[torch.LongTensor] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: if cross_attention_kwargs is not None: if cross_attention_kwargs.get("scale", None) is not None: @@ -1487,7 +1487,7 @@ def __init__( self._chunk_size = None self._chunk_dim = 0 - def _get_frame_indices(self, num_frames: int) -> List[Tuple[int, int]]: + def _get_frame_indices(self, num_frames: int) -> list[tuple[int, int]]: frame_indices = [] for i in range(0, num_frames - self.context_length + 1, self.context_stride): window_start = i @@ -1495,7 +1495,7 @@ def _get_frame_indices(self, num_frames: int) -> List[Tuple[int, int]]: frame_indices.append((window_start, window_end)) return frame_indices - def _get_frame_weights(self, num_frames: int, weighting_scheme: str = "pyramid") -> List[float]: + def _get_frame_weights(self, num_frames: int, weighting_scheme: str = "pyramid") -> list[float]: if weighting_scheme == "flat": weights = [1.0] * num_frames @@ -1545,7 +1545,7 @@ def forward( attention_mask: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, *args, **kwargs, ) -> torch.Tensor: diff --git a/src/diffusers/models/attention_dispatch.py b/src/diffusers/models/attention_dispatch.py index e1694910997a..d8ed60122555 100644 --- a/src/diffusers/models/attention_dispatch.py +++ b/src/diffusers/models/attention_dispatch.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import contextlib import functools import inspect import math from enum import Enum -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional import torch @@ -215,7 +217,7 @@ class _AttentionBackendRegistry: def register( cls, backend: AttentionBackendName, - constraints: Optional[List[Callable]] = None, + constraints: Optional[list[Callable]] = None, supports_context_parallel: bool = False, ): logger.debug(f"Registering attention backend: {backend} with constraints: {constraints}") @@ -250,7 +252,7 @@ def _is_context_parallel_enabled( @contextlib.contextmanager -def attention_backend(backend: Union[str, AttentionBackendName] = AttentionBackendName.NATIVE): +def attention_backend(backend: str | AttentionBackendName = AttentionBackendName.NATIVE): """ Context manager to set the active attention backend. """ @@ -278,7 +280,7 @@ def dispatch_attention_fn( is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, *, backend: Optional[AttentionBackendName] = None, parallel_config: Optional["ParallelConfig"] = None, @@ -576,7 +578,7 @@ def _wrapped_flash_attn_3( pack_gqa: Optional[bool] = None, deterministic: bool = False, sm_margin: int = 0, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> tuple[torch.Tensor, torch.Tensor]: # Hardcoded for now because pytorch does not support tuple/int type hints window_size = (-1, -1) out, lse, *_ = flash_attn_3_func( @@ -618,7 +620,7 @@ def _( pack_gqa: Optional[bool] = None, deterministic: bool = False, sm_margin: int = 0, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> tuple[torch.Tensor, torch.Tensor]: window_size = (-1, -1) # noqa: F841 # A lot of the parameters here are not yet used in any way within diffusers. # We can safely ignore for now and keep the fake op shape propagation simple. @@ -1316,7 +1318,7 @@ def _flash_attention_3_hub( value: torch.Tensor, scale: Optional[float] = None, is_causal: bool = False, - window_size: Tuple[int, int] = (-1, -1), + window_size: tuple[int, int] = (-1, -1), softcap: float = 0.0, deterministic: bool = False, return_attn_probs: bool = False, @@ -1405,7 +1407,7 @@ def _native_flex_attention( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - attn_mask: Optional[Union[torch.Tensor, "flex_attention.BlockMask"]] = None, + attn_mask: Optional[torch.Tensor | "flex_attention.BlockMask"] = None, is_causal: bool = False, scale: Optional[float] = None, enable_gqa: bool = False, diff --git a/src/diffusers/models/attention_processor.py b/src/diffusers/models/attention_processor.py index 66455d733aee..6484eeb5a7fa 100755 --- a/src/diffusers/models/attention_processor.py +++ b/src/diffusers/models/attention_processor.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect import math -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import torch import torch.nn.functional as F @@ -309,7 +309,7 @@ def __init__( def set_use_xla_flash_attention( self, use_xla_flash_attention: bool, - partition_spec: Optional[Tuple[Optional[str], ...]] = None, + partition_spec: Optional[tuple[Optional[str], ...]] = None, is_flux=False, ) -> None: r""" @@ -318,7 +318,7 @@ def set_use_xla_flash_attention( Args: use_xla_flash_attention (`bool`): Whether to use pallas flash attention kernel from `torch_xla` or not. - partition_spec (`Tuple[]`, *optional*): + partition_spec (`tuple[]`, *optional*): Specify the partition specification if using SPMD. Otherwise None. """ if use_xla_flash_attention: @@ -872,7 +872,7 @@ def __init__( attention_head_dim: int = 8, mult: float = 1.0, norm_type: str = "batch_norm", - kernel_sizes: Tuple[int, ...] = (5,), + kernel_sizes: tuple[int, ...] = (5,), eps: float = 1e-15, residual_connection: bool = False, ): @@ -2790,7 +2790,7 @@ class XLAFlashAttnProcessor2_0: Processor for implementing scaled dot-product attention with pallas flash attention kernel if using `torch_xla`. """ - def __init__(self, partition_spec: Optional[Tuple[Optional[str], ...]] = None): + def __init__(self, partition_spec: Optional[tuple[Optional[str], ...]] = None): if not hasattr(F, "scaled_dot_product_attention"): raise ImportError( "XLAFlashAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." @@ -3001,7 +3001,7 @@ def __init__(self): def apply_partial_rotary_emb( self, x: torch.Tensor, - freqs_cis: Tuple[torch.Tensor], + freqs_cis: tuple[torch.Tensor], ) -> torch.Tensor: from .embeddings import apply_rotary_emb @@ -4212,9 +4212,9 @@ class IPAdapterAttnProcessor(nn.Module): The hidden size of the attention layer. cross_attention_dim (`int`): The number of channels in the `encoder_hidden_states`. - num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`): + num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`): The context length of the image features. - scale (`float` or List[`float`], defaults to 1.0): + scale (`float` or list[`float`], defaults to 1.0): the weight scale of image prompt. """ @@ -4305,7 +4305,7 @@ def __call__( hidden_states = attn.batch_to_head_dim(hidden_states) if ip_adapter_masks is not None: - if not isinstance(ip_adapter_masks, List): + if not isinstance(ip_adapter_masks, list): # for backward compatibility, we accept `ip_adapter_mask` as a tensor of shape [num_ip_adapter, 1, height, width] ip_adapter_masks = list(ip_adapter_masks.unsqueeze(1)) if not (len(ip_adapter_masks) == len(self.scale) == len(ip_hidden_states)): @@ -4412,9 +4412,9 @@ class IPAdapterAttnProcessor2_0(torch.nn.Module): The hidden size of the attention layer. cross_attention_dim (`int`): The number of channels in the `encoder_hidden_states`. - num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`): + num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`): The context length of the image features. - scale (`float` or `List[float]`, defaults to 1.0): + scale (`float` or `list[float]`, defaults to 1.0): the weight scale of image prompt. """ @@ -4524,7 +4524,7 @@ def __call__( hidden_states = hidden_states.to(query.dtype) if ip_adapter_masks is not None: - if not isinstance(ip_adapter_masks, List): + if not isinstance(ip_adapter_masks, list): # for backward compatibility, we accept `ip_adapter_mask` as a tensor of shape [num_ip_adapter, 1, height, width] ip_adapter_masks = list(ip_adapter_masks.unsqueeze(1)) if not (len(ip_adapter_masks) == len(self.scale) == len(ip_hidden_states)): @@ -4644,9 +4644,9 @@ class IPAdapterXFormersAttnProcessor(torch.nn.Module): The hidden size of the attention layer. cross_attention_dim (`int`): The number of channels in the `encoder_hidden_states`. - num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`): + num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`): The context length of the image features. - scale (`float` or `List[float]`, defaults to 1.0): + scale (`float` or `list[float]`, defaults to 1.0): the weight scale of image prompt. attention_op (`Callable`, *optional*, defaults to `None`): The base @@ -4763,7 +4763,7 @@ def __call__( if ip_hidden_states: if ip_adapter_masks is not None: - if not isinstance(ip_adapter_masks, List): + if not isinstance(ip_adapter_masks, list): # for backward compatibility, we accept `ip_adapter_mask` as a tensor of shape [num_ip_adapter, 1, height, width] ip_adapter_masks = list(ip_adapter_masks.unsqueeze(1)) if not (len(ip_adapter_masks) == len(self.scale) == len(ip_hidden_states)): @@ -5622,56 +5622,56 @@ def __new__(cls, *args, **kwargs): FluxIPAdapterJointAttnProcessor2_0, ) -AttentionProcessor = Union[ - AttnProcessor, - CustomDiffusionAttnProcessor, - AttnAddedKVProcessor, - AttnAddedKVProcessor2_0, - JointAttnProcessor2_0, - PAGJointAttnProcessor2_0, - PAGCFGJointAttnProcessor2_0, - FusedJointAttnProcessor2_0, - AllegroAttnProcessor2_0, - AuraFlowAttnProcessor2_0, - FusedAuraFlowAttnProcessor2_0, - FluxAttnProcessor2_0, - FluxAttnProcessor2_0_NPU, - FusedFluxAttnProcessor2_0, - FusedFluxAttnProcessor2_0_NPU, - CogVideoXAttnProcessor2_0, - FusedCogVideoXAttnProcessor2_0, - XFormersAttnAddedKVProcessor, - XFormersAttnProcessor, - XLAFlashAttnProcessor2_0, - AttnProcessorNPU, - AttnProcessor2_0, - MochiVaeAttnProcessor2_0, - MochiAttnProcessor2_0, - StableAudioAttnProcessor2_0, - HunyuanAttnProcessor2_0, - FusedHunyuanAttnProcessor2_0, - PAGHunyuanAttnProcessor2_0, - PAGCFGHunyuanAttnProcessor2_0, - LuminaAttnProcessor2_0, - FusedAttnProcessor2_0, - CustomDiffusionXFormersAttnProcessor, - CustomDiffusionAttnProcessor2_0, - SlicedAttnProcessor, - SlicedAttnAddedKVProcessor, - SanaLinearAttnProcessor2_0, - PAGCFGSanaLinearAttnProcessor2_0, - PAGIdentitySanaLinearAttnProcessor2_0, - SanaMultiscaleLinearAttention, - SanaMultiscaleAttnProcessor2_0, - SanaMultiscaleAttentionProjection, - IPAdapterAttnProcessor, - IPAdapterAttnProcessor2_0, - IPAdapterXFormersAttnProcessor, - SD3IPAdapterJointAttnProcessor2_0, - PAGIdentitySelfAttnProcessor2_0, - PAGCFGIdentitySelfAttnProcessor2_0, - LoRAAttnProcessor, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, - LoRAAttnAddedKVProcessor, -] +AttentionProcessor = ( + AttnProcessor + | CustomDiffusionAttnProcessor + | AttnAddedKVProcessor + | AttnAddedKVProcessor2_0 + | JointAttnProcessor2_0 + | PAGJointAttnProcessor2_0 + | PAGCFGJointAttnProcessor2_0 + | FusedJointAttnProcessor2_0 + | AllegroAttnProcessor2_0 + | AuraFlowAttnProcessor2_0 + | FusedAuraFlowAttnProcessor2_0 + | FluxAttnProcessor2_0 + | FluxAttnProcessor2_0_NPU + | FusedFluxAttnProcessor2_0 + | FusedFluxAttnProcessor2_0_NPU + | CogVideoXAttnProcessor2_0 + | FusedCogVideoXAttnProcessor2_0 + | XFormersAttnAddedKVProcessor + | XFormersAttnProcessor + | XLAFlashAttnProcessor2_0 + | AttnProcessorNPU + | AttnProcessor2_0 + | MochiVaeAttnProcessor2_0 + | MochiAttnProcessor2_0 + | StableAudioAttnProcessor2_0 + | HunyuanAttnProcessor2_0 + | FusedHunyuanAttnProcessor2_0 + | PAGHunyuanAttnProcessor2_0 + | PAGCFGHunyuanAttnProcessor2_0 + | LuminaAttnProcessor2_0 + | FusedAttnProcessor2_0 + | CustomDiffusionXFormersAttnProcessor + | CustomDiffusionAttnProcessor2_0 + | SlicedAttnProcessor + | SlicedAttnAddedKVProcessor + | SanaLinearAttnProcessor2_0 + | PAGCFGSanaLinearAttnProcessor2_0 + | PAGIdentitySanaLinearAttnProcessor2_0 + | SanaMultiscaleLinearAttention + | SanaMultiscaleAttnProcessor2_0 + | SanaMultiscaleAttentionProjection + | IPAdapterAttnProcessor + | IPAdapterAttnProcessor2_0 + | IPAdapterXFormersAttnProcessor + | SD3IPAdapterJointAttnProcessor2_0 + | PAGIdentitySelfAttnProcessor2_0 + | PAGCFGIdentitySelfAttnProcessor2_0 + | LoRAAttnProcessor + | LoRAAttnProcessor2_0 + | LoRAXFormersAttnProcessor + | LoRAAttnAddedKVProcessor +) diff --git a/src/diffusers/models/auto_model.py b/src/diffusers/models/auto_model.py index 947b610ea61a..ca7ea3985412 100644 --- a/src/diffusers/models/auto_model.py +++ b/src/diffusers/models/auto_model.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from typing import Optional, Union +from typing import Optional from huggingface_hub.utils import validate_hf_hub_args @@ -37,7 +37,7 @@ def __init__(self, *args, **kwargs): @classmethod @validate_hf_hub_args - def from_pretrained(cls, pretrained_model_or_path: Optional[Union[str, os.PathLike]] = None, **kwargs): + def from_pretrained(cls, pretrained_model_or_path: Optional[str | os.PathLike] = None, **kwargs): r""" Instantiate a pretrained PyTorch model from a pretrained model configuration. @@ -61,7 +61,7 @@ def from_pretrained(cls, pretrained_model_or_path: Optional[Union[str, os.PathLi force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info (`bool`, *optional*, defaults to `False`): @@ -83,7 +83,7 @@ def from_pretrained(cls, pretrained_model_or_path: Optional[Union[str, os.PathLi Mirror source to resolve accessibility issues if you're downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be defined for each parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the same device. Defaults to `None`, meaning that the model will be loaded on CPU. diff --git a/src/diffusers/models/autoencoders/autoencoder_asym_kl.py b/src/diffusers/models/autoencoders/autoencoder_asym_kl.py index 54b1fc677188..c9f1152deba7 100644 --- a/src/diffusers/models/autoencoders/autoencoder_asym_kl.py +++ b/src/diffusers/models/autoencoders/autoencoder_asym_kl.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -34,16 +34,16 @@ class AsymmetricAutoencoderKL(ModelMixin, ConfigMixin): Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): - Tuple of downsample block types. - down_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of down block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + tuple of downsample block types. + down_block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of down block output channels. layers_per_down_block (`int`, *optional*, defaults to `1`): Number layers for down block. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): - Tuple of upsample block types. - up_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of up block output channels. + up_block_types (`tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + tuple of upsample block types. + up_block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of up block output channels. layers_per_up_block (`int`, *optional*, defaults to `1`): Number layers for up block. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. @@ -67,11 +67,11 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), - down_block_out_channels: Tuple[int, ...] = (64,), + down_block_types: tuple[str, ...] = ("DownEncoderBlock2D",), + down_block_out_channels: tuple[int, ...] = (64,), layers_per_down_block: int = 1, - up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), - up_block_out_channels: Tuple[int, ...] = (64,), + up_block_types: tuple[str, ...] = ("UpDecoderBlock2D",), + up_block_out_channels: tuple[int, ...] = (64,), layers_per_up_block: int = 1, act_fn: str = "silu", latent_channels: int = 4, @@ -114,7 +114,7 @@ def __init__( self.register_to_config(force_upcast=False) @apply_forward_hook - def encode(self, x: torch.Tensor, return_dict: bool = True) -> Union[AutoencoderKLOutput, Tuple[torch.Tensor]]: + def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput | tuple[torch.Tensor]: h = self.encoder(x) moments = self.quant_conv(h) posterior = DiagonalGaussianDistribution(moments) @@ -130,7 +130,7 @@ def _decode( image: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: z = self.post_quant_conv(z) dec = self.decoder(z, image, mask) @@ -147,7 +147,7 @@ def decode( image: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: decoded = self._decode(z, image, mask).sample if not return_dict: @@ -162,7 +162,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_dc.py b/src/diffusers/models/autoencoders/autoencoder_dc.py index 783f22e97daf..13b50e66a5ea 100644 --- a/src/diffusers/models/autoencoders/autoencoder_dc.py +++ b/src/diffusers/models/autoencoders/autoencoder_dc.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -68,7 +68,7 @@ def __init__( in_channels: int, mult: float = 1.0, attention_head_dim: int = 32, - qkv_multiscales: Tuple[int, ...] = (5,), + qkv_multiscales: tuple[int, ...] = (5,), norm_type: str = "batch_norm", ) -> None: super().__init__() @@ -102,7 +102,7 @@ def get_block( attention_head_dim: int, norm_type: str, act_fn: str, - qkv_mutliscales: Tuple[int] = (), + qkv_mutliscales: tuple[int] = (), ): if block_type == "ResBlock": block = ResBlock(in_channels, out_channels, norm_type, act_fn) @@ -205,10 +205,10 @@ def __init__( in_channels: int, latent_channels: int, attention_head_dim: int = 32, - block_type: Union[str, Tuple[str]] = "ResBlock", - block_out_channels: Tuple[int] = (128, 256, 512, 512, 1024, 1024), - layers_per_block: Tuple[int] = (2, 2, 2, 2, 2, 2), - qkv_multiscales: Tuple[Tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), + block_type: str | tuple[str] = "ResBlock", + block_out_channels: tuple[int] = (128, 256, 512, 512, 1024, 1024), + layers_per_block: tuple[int] = (2, 2, 2, 2, 2, 2), + qkv_multiscales: tuple[tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), downsample_block_type: str = "pixel_unshuffle", out_shortcut: bool = True, ): @@ -291,12 +291,12 @@ def __init__( in_channels: int, latent_channels: int, attention_head_dim: int = 32, - block_type: Union[str, Tuple[str]] = "ResBlock", - block_out_channels: Tuple[int] = (128, 256, 512, 512, 1024, 1024), - layers_per_block: Tuple[int] = (2, 2, 2, 2, 2, 2), - qkv_multiscales: Tuple[Tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), - norm_type: Union[str, Tuple[str]] = "rms_norm", - act_fn: Union[str, Tuple[str]] = "silu", + block_type: str | tuple[str] = "ResBlock", + block_out_channels: tuple[int] = (128, 256, 512, 512, 1024, 1024), + layers_per_block: tuple[int] = (2, 2, 2, 2, 2, 2), + qkv_multiscales: tuple[tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), + norm_type: str | tuple[str] = "rms_norm", + act_fn: str | tuple[str] = "silu", upsample_block_type: str = "pixel_shuffle", in_shortcut: bool = True, conv_act_fn: str = "relu", @@ -391,29 +391,29 @@ class AutoencoderDC(ModelMixin, ConfigMixin, FromOriginalModelMixin): The number of input channels in samples. latent_channels (`int`, defaults to `32`): The number of channels in the latent space representation. - encoder_block_types (`Union[str, Tuple[str]]`, defaults to `"ResBlock"`): + encoder_block_types (`Union[str, tuple[str]]`, defaults to `"ResBlock"`): The type(s) of block to use in the encoder. - decoder_block_types (`Union[str, Tuple[str]]`, defaults to `"ResBlock"`): + decoder_block_types (`Union[str, tuple[str]]`, defaults to `"ResBlock"`): The type(s) of block to use in the decoder. - encoder_block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512, 1024, 1024)`): + encoder_block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512, 1024, 1024)`): The number of output channels for each block in the encoder. - decoder_block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512, 1024, 1024)`): + decoder_block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512, 1024, 1024)`): The number of output channels for each block in the decoder. - encoder_layers_per_block (`Tuple[int]`, defaults to `(2, 2, 2, 3, 3, 3)`): + encoder_layers_per_block (`tuple[int]`, defaults to `(2, 2, 2, 3, 3, 3)`): The number of layers per block in the encoder. - decoder_layers_per_block (`Tuple[int]`, defaults to `(3, 3, 3, 3, 3, 3)`): + decoder_layers_per_block (`tuple[int]`, defaults to `(3, 3, 3, 3, 3, 3)`): The number of layers per block in the decoder. - encoder_qkv_multiscales (`Tuple[Tuple[int, ...], ...]`, defaults to `((), (), (), (5,), (5,), (5,))`): + encoder_qkv_multiscales (`tuple[tuple[int, ...], ...]`, defaults to `((), (), (), (5,), (5,), (5,))`): Multi-scale configurations for the encoder's QKV (query-key-value) transformations. - decoder_qkv_multiscales (`Tuple[Tuple[int, ...], ...]`, defaults to `((), (), (), (5,), (5,), (5,))`): + decoder_qkv_multiscales (`tuple[tuple[int, ...], ...]`, defaults to `((), (), (), (5,), (5,), (5,))`): Multi-scale configurations for the decoder's QKV (query-key-value) transformations. upsample_block_type (`str`, defaults to `"pixel_shuffle"`): The type of block to use for upsampling in the decoder. downsample_block_type (`str`, defaults to `"pixel_unshuffle"`): The type of block to use for downsampling in the encoder. - decoder_norm_types (`Union[str, Tuple[str]]`, defaults to `"rms_norm"`): + decoder_norm_types (`Union[str, tuple[str]]`, defaults to `"rms_norm"`): The normalization type(s) to use in the decoder. - decoder_act_fns (`Union[str, Tuple[str]]`, defaults to `"silu"`): + decoder_act_fns (`Union[str, tuple[str]]`, defaults to `"silu"`): The activation function(s) to use in the decoder. encoder_out_shortcut (`bool`, defaults to `True`): Whether to use shortcut at the end of the encoder. @@ -436,18 +436,18 @@ def __init__( in_channels: int = 3, latent_channels: int = 32, attention_head_dim: int = 32, - encoder_block_types: Union[str, Tuple[str]] = "ResBlock", - decoder_block_types: Union[str, Tuple[str]] = "ResBlock", - encoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512, 1024, 1024), - decoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512, 1024, 1024), - encoder_layers_per_block: Tuple[int] = (2, 2, 2, 3, 3, 3), - decoder_layers_per_block: Tuple[int] = (3, 3, 3, 3, 3, 3), - encoder_qkv_multiscales: Tuple[Tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), - decoder_qkv_multiscales: Tuple[Tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), + encoder_block_types: str | tuple[str] = "ResBlock", + decoder_block_types: str | tuple[str] = "ResBlock", + encoder_block_out_channels: tuple[int, ...] = (128, 256, 512, 512, 1024, 1024), + decoder_block_out_channels: tuple[int, ...] = (128, 256, 512, 512, 1024, 1024), + encoder_layers_per_block: tuple[int] = (2, 2, 2, 3, 3, 3), + decoder_layers_per_block: tuple[int] = (3, 3, 3, 3, 3, 3), + encoder_qkv_multiscales: tuple[tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), + decoder_qkv_multiscales: tuple[tuple[int, ...], ...] = ((), (), (), (5,), (5,), (5,)), upsample_block_type: str = "pixel_shuffle", downsample_block_type: str = "pixel_unshuffle", - decoder_norm_types: Union[str, Tuple[str]] = "rms_norm", - decoder_act_fns: Union[str, Tuple[str]] = "silu", + decoder_norm_types: str | tuple[str] = "rms_norm", + decoder_act_fns: str | tuple[str] = "silu", encoder_out_shortcut: bool = True, decoder_in_shortcut: bool = True, decoder_conv_act_fn: str = "relu", @@ -568,7 +568,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: return encoded @apply_forward_hook - def encode(self, x: torch.Tensor, return_dict: bool = True) -> Union[EncoderOutput, Tuple[torch.Tensor]]: + def encode(self, x: torch.Tensor, return_dict: bool = True) -> EncoderOutput | tuple[torch.Tensor]: r""" Encode a batch of images into latents. @@ -602,7 +602,7 @@ def _decode(self, z: torch.Tensor) -> torch.Tensor: return decoded @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: r""" Decode a batch of images. @@ -686,7 +686,7 @@ def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> torch.Tenso return (encoded,) return EncoderOutput(latent=encoded) - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, height, width = z.shape tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio diff --git a/src/diffusers/models/autoencoders/autoencoder_kl.py b/src/diffusers/models/autoencoders/autoencoder_kl.py index d823c2fb8b04..2c8e04106aa2 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -45,12 +45,12 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin, PeftAdapter Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): - Tuple of downsample block types. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): - Tuple of upsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + tuple of downsample block types. + up_block_types (`tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + tuple of upsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of block output channels. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. sample_size (`int`, *optional*, defaults to `32`): Sample input size. @@ -78,9 +78,9 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str] = ("DownEncoderBlock2D",), - up_block_types: Tuple[str] = ("UpDecoderBlock2D",), - block_out_channels: Tuple[int] = (64,), + down_block_types: tuple[str] = ("DownEncoderBlock2D",), + up_block_types: tuple[str] = ("UpDecoderBlock2D",), + block_out_channels: tuple[int] = (64,), layers_per_block: int = 1, act_fn: str = "silu", latent_channels: int = 4, @@ -88,8 +88,8 @@ def __init__( sample_size: int = 32, scaling_factor: float = 0.18215, shift_factor: Optional[float] = None, - latents_mean: Optional[Tuple[float]] = None, - latents_std: Optional[Tuple[float]] = None, + latents_mean: Optional[tuple[float]] = None, + latents_std: Optional[tuple[float]] = None, force_upcast: bool = True, use_quant_conv: bool = True, use_post_quant_conv: bool = True, @@ -169,7 +169,7 @@ def disable_slicing(self): @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -178,7 +178,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -193,7 +193,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -258,7 +258,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -284,7 +284,7 @@ def encode( return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: if self.use_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size): return self.tiled_decode(z, return_dict=return_dict) @@ -301,7 +301,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut @apply_forward_hook def decode( self, z: torch.FloatTensor, return_dict: bool = True, generator=None - ) -> Union[DecoderOutput, torch.FloatTensor]: + ) -> DecoderOutput | torch.FloatTensor: """ Decode a batch of images. @@ -449,7 +449,7 @@ def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> Autoencoder return AutoencoderKLOutput(latent_dist=posterior) - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -504,7 +504,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_allegro.py b/src/diffusers/models/autoencoders/autoencoder_kl_allegro.py index c24b8f42aca4..f6b136bda732 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_allegro.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_allegro.py @@ -14,7 +14,7 @@ # limitations under the License. import math -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -416,14 +416,14 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - temporal_downsample_blocks: Tuple[bool, ...] = [True, True, False, False], + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + temporal_downsample_blocks: tuple[bool, ...] = [True, True, False, False], layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -543,14 +543,14 @@ def __init__( self, in_channels: int = 4, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", ), - temporal_upsample_blocks: Tuple[bool, ...] = [False, True, True, False], - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + temporal_upsample_blocks: tuple[bool, ...] = [False, True, True, False], + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -686,14 +686,14 @@ class AutoencoderKLAllegro(ModelMixin, ConfigMixin): Number of channels in the input image. out_channels (int, defaults to `3`): Number of channels in the output. - down_block_types (`Tuple[str, ...]`, defaults to `("AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D")`): - Tuple of strings denoting which types of down blocks to use. - up_block_types (`Tuple[str, ...]`, defaults to `("AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D")`): - Tuple of strings denoting which types of up blocks to use. - block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): - Tuple of integers denoting number of output channels in each block. - temporal_downsample_blocks (`Tuple[bool, ...]`, defaults to `(True, True, False, False)`): - Tuple of booleans denoting which blocks to enable temporal downsampling in. + down_block_types (`tuple[str, ...]`, defaults to `("AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D")`): + tuple of strings denoting which types of down blocks to use. + up_block_types (`tuple[str, ...]`, defaults to `("AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D")`): + tuple of strings denoting which types of up blocks to use. + block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): + tuple of integers denoting number of output channels in each block. + temporal_downsample_blocks (`tuple[bool, ...]`, defaults to `(True, True, False, False)`): + tuple of booleans denoting which blocks to enable temporal downsampling in. latent_channels (`int`, defaults to `4`): Number of channels in latents. layers_per_block (`int`, defaults to `2`): @@ -726,21 +726,21 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", "AllegroDownBlock3D", ), - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", "AllegroUpBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - temporal_downsample_blocks: Tuple[bool, ...] = (True, True, False, False), - temporal_upsample_blocks: Tuple[bool, ...] = (False, True, True, False), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + temporal_downsample_blocks: tuple[bool, ...] = (True, True, False, False), + temporal_upsample_blocks: tuple[bool, ...] = (False, True, True, False), latent_channels: int = 4, layers_per_block: int = 2, act_fn: str = "silu", @@ -835,7 +835,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: r""" Encode a batch of videos into latents. @@ -870,7 +870,7 @@ def _decode(self, z: torch.Tensor) -> torch.Tensor: raise NotImplementedError("Decoding without tiling has not been implemented yet.") @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: """ Decode a batch of videos. @@ -1073,7 +1073,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py b/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py index e0e9436e8984..d035fb413b3a 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -72,7 +72,7 @@ class CogVideoXCausalConv3d(nn.Module): Args: in_channels (`int`): Number of channels in the input tensor. out_channels (`int`): Number of output channels produced by the convolution. - kernel_size (`int` or `Tuple[int, int, int]`): Kernel size of the convolutional kernel. + kernel_size (`int` or `tuple[int, int, int]`): Kernel size of the convolutional kernel. stride (`int`, defaults to `1`): Stride of the convolution. dilation (`int`, defaults to `1`): Dilation rate of the convolution. pad_mode (`str`, defaults to `"constant"`): Padding mode. @@ -82,7 +82,7 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int, int]], + kernel_size: int | tuple[int, int, int], stride: int = 1, dilation: int = 1, pad_mode: str = "constant", @@ -174,7 +174,7 @@ def __init__( self.conv_b = CogVideoXCausalConv3d(zq_channels, f_channels, kernel_size=1, stride=1) def forward( - self, f: torch.Tensor, zq: torch.Tensor, conv_cache: Optional[Dict[str, torch.Tensor]] = None + self, f: torch.Tensor, zq: torch.Tensor, conv_cache: Optional[dict[str, torch.Tensor]] = None ) -> torch.Tensor: new_conv_cache = {} conv_cache = conv_cache or {} @@ -289,7 +289,7 @@ def forward( inputs: torch.Tensor, temb: Optional[torch.Tensor] = None, zq: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: new_conv_cache = {} conv_cache = conv_cache or {} @@ -411,7 +411,7 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, zq: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""Forward method of the `CogVideoXDownBlock3D` class.""" @@ -506,7 +506,7 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, zq: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""Forward method of the `CogVideoXMidBlock3D` class.""" @@ -613,7 +613,7 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, zq: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""Forward method of the `CogVideoXUpBlock3D` class.""" @@ -652,10 +652,10 @@ class CogVideoXEncoder3D(nn.Module): The number of input channels. out_channels (`int`, *optional*, defaults to 3): The number of output channels. - down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + down_block_types (`tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): The types of down blocks to use. See `~diffusers.models.unet_2d_blocks.get_down_block` for available options. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(64,)`): The number of output channels for each block. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. See `~diffusers.models.activations.get_activation` for available options. @@ -671,13 +671,13 @@ def __init__( self, in_channels: int = 3, out_channels: int = 16, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 256, 512), + block_out_channels: tuple[int, ...] = (128, 256, 256, 512), layers_per_block: int = 3, act_fn: str = "silu", norm_eps: float = 1e-6, @@ -744,7 +744,7 @@ def forward( self, sample: torch.Tensor, temb: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""The forward method of the `CogVideoXEncoder3D` class.""" @@ -805,9 +805,9 @@ class CogVideoXDecoder3D(nn.Module): The number of input channels. out_channels (`int`, *optional*, defaults to 3): The number of output channels. - up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + up_block_types (`tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(64,)`): The number of output channels for each block. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. See `~diffusers.models.activations.get_activation` for available options. @@ -823,13 +823,13 @@ def __init__( self, in_channels: int = 16, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 256, 512), + block_out_channels: tuple[int, ...] = (128, 256, 256, 512), layers_per_block: int = 3, act_fn: str = "silu", norm_eps: float = 1e-6, @@ -903,7 +903,7 @@ def forward( self, sample: torch.Tensor, temb: Optional[torch.Tensor] = None, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""The forward method of the `CogVideoXDecoder3D` class.""" @@ -966,12 +966,12 @@ class AutoencoderKLCogVideoX(ModelMixin, ConfigMixin, FromOriginalModelMixin): Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): - Tuple of downsample block types. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): - Tuple of upsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + tuple of downsample block types. + up_block_types (`tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + tuple of upsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of block output channels. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. sample_size (`int`, *optional*, defaults to `32`): Sample input size. scaling_factor (`float`, *optional*, defaults to `1.15258426`): @@ -995,19 +995,19 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", ), - up_block_types: Tuple[str] = ( + up_block_types: tuple[str] = ( "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", ), - block_out_channels: Tuple[int] = (128, 256, 256, 512), + block_out_channels: tuple[int] = (128, 256, 256, 512), latent_channels: int = 16, layers_per_block: int = 3, act_fn: str = "silu", @@ -1018,8 +1018,8 @@ def __init__( sample_width: int = 720, scaling_factor: float = 1.15258426, shift_factor: Optional[float] = None, - latents_mean: Optional[Tuple[float]] = None, - latents_std: Optional[Tuple[float]] = None, + latents_mean: Optional[tuple[float]] = None, + latents_std: Optional[tuple[float]] = None, force_upcast: float = True, use_quant_conv: bool = False, use_post_quant_conv: bool = False, @@ -1174,7 +1174,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -1199,7 +1199,7 @@ def encode( return (posterior,) return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape if self.use_tiling and (width > self.tile_latent_min_width or height > self.tile_latent_min_height): @@ -1228,7 +1228,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut return DecoderOutput(sample=dec) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: """ Decode a batch of images. @@ -1342,7 +1342,7 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: enc = torch.cat(result_rows, dim=3) return enc - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1431,7 +1431,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor | torch.Tensor: x = sample posterior = self.encode(x).latent_dist if sample_posterior: diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_cosmos.py b/src/diffusers/models/autoencoders/autoencoder_kl_cosmos.py index 500e316ebcf0..c8502fa10cdb 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_cosmos.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_cosmos.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import math -from typing import List, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -47,9 +49,9 @@ def __init__( self, in_channels: int = 1, out_channels: int = 1, - kernel_size: Union[int, Tuple[int, int, int]] = (3, 3, 3), - dilation: Union[int, Tuple[int, int, int]] = (1, 1, 1), - stride: Union[int, Tuple[int, int, int]] = (1, 1, 1), + kernel_size: int | tuple[int, int, int] = (3, 3, 3), + dilation: int | tuple[int, int, int] = (1, 1, 1), + stride: int | tuple[int, int, int] = (1, 1, 1), padding: int = 1, pad_mode: str = "constant", ) -> None: @@ -419,7 +421,7 @@ def __init__( attention_head_dim: int, num_groups: int = 1, dropout: float = 0.0, - processor: Union["CosmosSpatialAttentionProcessor2_0", "CosmosTemporalAttentionProcessor2_0"] = None, + processor: "CosmosSpatialAttentionProcessor2_0" | "CosmosTemporalAttentionProcessor2_0" = None, ) -> None: super().__init__() self.num_attention_heads = num_attention_heads @@ -711,9 +713,9 @@ def __init__( self, in_channels: int = 3, out_channels: int = 16, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), num_resnet_blocks: int = 2, - attention_resolutions: Tuple[int, ...] = (32,), + attention_resolutions: tuple[int, ...] = (32,), resolution: int = 1024, patch_size: int = 4, patch_type: str = "haar", @@ -795,9 +797,9 @@ def __init__( self, in_channels: int = 16, out_channels: int = 3, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), num_resnet_blocks: int = 2, - attention_resolutions: Tuple[int, ...] = (32,), + attention_resolutions: tuple[int, ...] = (32,), resolution: int = 1024, patch_size: int = 4, patch_type: str = "haar", @@ -886,12 +888,12 @@ class AutoencoderKLCosmos(ModelMixin, ConfigMixin): Number of output channels. latent_channels (`int`, defaults to `16`): Number of latent channels. - encoder_block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): + encoder_block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): Number of output channels for each encoder down block. - decode_block_out_channels (`Tuple[int, ...]`, defaults to `(256, 512, 512, 512)`): + decode_block_out_channels (`tuple[int, ...]`, defaults to `(256, 512, 512, 512)`): Number of output channels for each decoder up block. - attention_resolutions (`Tuple[int, ...]`, defaults to `(32,)`): - List of image/video resolutions at which to apply attention. + attention_resolutions (`tuple[int, ...]`, defaults to `(32,)`): + list of image/video resolutions at which to apply attention. resolution (`int`, defaults to `1024`): Base image/video resolution used for computing whether a block should have attention layers. num_layers (`int`, defaults to `2`): @@ -924,9 +926,9 @@ def __init__( in_channels: int = 3, out_channels: int = 3, latent_channels: int = 16, - encoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - decode_block_out_channels: Tuple[int, ...] = (256, 512, 512, 512), - attention_resolutions: Tuple[int, ...] = (32,), + encoder_block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + decode_block_out_channels: tuple[int, ...] = (256, 512, 512, 512), + attention_resolutions: tuple[int, ...] = (32,), resolution: int = 1024, num_layers: int = 2, patch_size: int = 4, @@ -934,8 +936,8 @@ def __init__( scaling_factor: float = 1.0, spatial_compression_ratio: int = 8, temporal_compression_ratio: int = 8, - latents_mean: Optional[List[float]] = LATENTS_MEAN, - latents_std: Optional[List[float]] = LATENTS_STD, + latents_mean: Optional[list[float]] = LATENTS_MEAN, + latents_std: Optional[list[float]] = LATENTS_STD, ) -> None: super().__init__() @@ -1071,7 +1073,7 @@ def encode(self, x: torch.Tensor, return_dict: bool = True) -> torch.Tensor: return (posterior,) return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: z = self.post_quant_conv(z) dec = self.decoder(z) @@ -1080,7 +1082,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut return DecoderOutput(sample=dec) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: if self.use_slicing and z.shape[0] > 1: decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] decoded = torch.cat(decoded_slices) @@ -1097,7 +1099,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[Tuple[torch.Tensor], DecoderOutput]: + ) -> tuple[torch.Tensor] | DecoderOutput: x = sample posterior = self.encode(x).latent_dist if sample_posterior: diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py b/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py index dc5e775f6769..90ae2dfc3118 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -50,10 +50,10 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int, int]] = 3, - stride: Union[int, Tuple[int, int, int]] = 1, - padding: Union[int, Tuple[int, int, int]] = 0, - dilation: Union[int, Tuple[int, int, int]] = 1, + kernel_size: int | tuple[int, int, int] = 3, + stride: int | tuple[int, int, int] = 1, + padding: int | tuple[int, int, int] = 0, + dilation: int | tuple[int, int, int] = 1, bias: bool = True, pad_mode: str = "replicate", ) -> None: @@ -86,7 +86,7 @@ def __init__( kernel_size: int = 3, stride: int = 1, bias: bool = True, - upsample_factor: Tuple[float, float, float] = (2, 2, 2), + upsample_factor: tuple[float, float, float] = (2, 2, 2), ) -> None: super().__init__() @@ -357,7 +357,7 @@ def __init__( resnet_act_fn: str = "swish", resnet_groups: int = 32, add_upsample: bool = True, - upsample_scale_factor: Tuple[int, int, int] = (2, 2, 2), + upsample_scale_factor: tuple[int, int, int] = (2, 2, 2), ) -> None: super().__init__() resnets = [] @@ -418,13 +418,13 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -526,13 +526,13 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", ), - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -641,19 +641,19 @@ def __init__( in_channels: int = 3, out_channels: int = 3, latent_channels: int = 16, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", "HunyuanVideoDownBlock3D", ), - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", "HunyuanVideoUpBlock3D", ), - block_out_channels: Tuple[int] = (128, 256, 512, 512), + block_out_channels: tuple[int] = (128, 256, 512, 512), layers_per_block: int = 2, act_fn: str = "silu", norm_num_groups: int = 32, @@ -800,7 +800,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: r""" Encode a batch of images into latents. @@ -825,7 +825,7 @@ def encode( return (posterior,) return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio @@ -846,7 +846,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut return DecoderOutput(sample=dec) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images. @@ -945,7 +945,7 @@ def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] return enc - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1034,7 +1034,7 @@ def _temporal_tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: enc = torch.cat(result_row, dim=2)[:, :, :latent_num_frames] return enc - def _temporal_tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _temporal_tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 @@ -1076,7 +1076,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_ltx.py b/src/diffusers/models/autoencoders/autoencoder_kl_ltx.py index 51c600a4e915..b099b906417f 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_ltx.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_ltx.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -34,9 +34,9 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int, int]] = 3, - stride: Union[int, Tuple[int, int, int]] = 1, - dilation: Union[int, Tuple[int, int, int]] = 1, + kernel_size: int | tuple[int, int, int] = 3, + stride: int | tuple[int, int, int] = 1, + dilation: int | tuple[int, int, int] = 1, groups: int = 1, padding_mode: str = "zeros", is_causal: bool = True, @@ -201,7 +201,7 @@ def __init__( self, in_channels: int, out_channels: int, - stride: Union[int, Tuple[int, int, int]] = 1, + stride: int | tuple[int, int, int] = 1, is_causal: bool = True, padding_mode: str = "zeros", ) -> None: @@ -249,7 +249,7 @@ class LTXVideoUpsampler3d(nn.Module): def __init__( self, in_channels: int, - stride: Union[int, Tuple[int, int, int]] = 1, + stride: int | tuple[int, int, int] = 1, is_causal: bool = True, residual: bool = False, upscale_factor: int = 1, @@ -735,11 +735,11 @@ class LTXVideoEncoder3d(nn.Module): Number of input channels. out_channels (`int`, defaults to 128): Number of latent channels. - block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): + block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): The number of output channels for each block. - spatio_temporal_scaling (`Tuple[bool, ...], defaults to `(True, True, True, False)`: + spatio_temporal_scaling (`tuple[bool, ...], defaults to `(True, True, True, False)`: Whether a block should contain spatio-temporal downscaling layers or not. - layers_per_block (`Tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): + layers_per_block (`tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): The number of layers per block. patch_size (`int`, defaults to `4`): The size of spatial patches. @@ -755,16 +755,16 @@ def __init__( self, in_channels: int = 3, out_channels: int = 128, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - down_block_types: Tuple[str, ...] = ( + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + down_block_types: tuple[str, ...] = ( "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", ), - spatio_temporal_scaling: Tuple[bool, ...] = (True, True, True, False), - layers_per_block: Tuple[int, ...] = (4, 3, 3, 3, 4), - downsample_type: Tuple[str, ...] = ("conv", "conv", "conv", "conv"), + spatio_temporal_scaling: tuple[bool, ...] = (True, True, True, False), + layers_per_block: tuple[int, ...] = (4, 3, 3, 3, 4), + downsample_type: tuple[str, ...] = ("conv", "conv", "conv", "conv"), patch_size: int = 4, patch_size_t: int = 1, resnet_norm_eps: float = 1e-6, @@ -888,11 +888,11 @@ class LTXVideoDecoder3d(nn.Module): Number of latent channels. out_channels (`int`, defaults to 3): Number of output channels. - block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): + block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): The number of output channels for each block. - spatio_temporal_scaling (`Tuple[bool, ...], defaults to `(True, True, True, False)`: + spatio_temporal_scaling (`tuple[bool, ...], defaults to `(True, True, True, False)`: Whether a block should contain spatio-temporal upscaling layers or not. - layers_per_block (`Tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): + layers_per_block (`tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): The number of layers per block. patch_size (`int`, defaults to `4`): The size of spatial patches. @@ -910,17 +910,17 @@ def __init__( self, in_channels: int = 128, out_channels: int = 3, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - spatio_temporal_scaling: Tuple[bool, ...] = (True, True, True, False), - layers_per_block: Tuple[int, ...] = (4, 3, 3, 3, 4), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + spatio_temporal_scaling: tuple[bool, ...] = (True, True, True, False), + layers_per_block: tuple[int, ...] = (4, 3, 3, 3, 4), patch_size: int = 4, patch_size_t: int = 1, resnet_norm_eps: float = 1e-6, is_causal: bool = False, - inject_noise: Tuple[bool, ...] = (False, False, False, False), + inject_noise: tuple[bool, ...] = (False, False, False, False), timestep_conditioning: bool = False, - upsample_residual: Tuple[bool, ...] = (False, False, False, False), - upsample_factor: Tuple[bool, ...] = (1, 1, 1, 1), + upsample_residual: tuple[bool, ...] = (False, False, False, False), + upsample_factor: tuple[bool, ...] = (1, 1, 1, 1), ) -> None: super().__init__() @@ -1049,11 +1049,11 @@ class AutoencoderKLLTXVideo(ModelMixin, ConfigMixin, FromOriginalModelMixin): Number of output channels. latent_channels (`int`, defaults to `128`): Number of latent channels. - block_out_channels (`Tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): + block_out_channels (`tuple[int, ...]`, defaults to `(128, 256, 512, 512)`): The number of output channels for each block. - spatio_temporal_scaling (`Tuple[bool, ...], defaults to `(True, True, True, False)`: + spatio_temporal_scaling (`tuple[bool, ...], defaults to `(True, True, True, False)`: Whether a block should contain spatio-temporal downscaling or not. - layers_per_block (`Tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): + layers_per_block (`tuple[int, ...]`, defaults to `(4, 3, 3, 3, 4)`): The number of layers per block. patch_size (`int`, defaults to `4`): The size of spatial patches. @@ -1082,22 +1082,22 @@ def __init__( in_channels: int = 3, out_channels: int = 3, latent_channels: int = 128, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - down_block_types: Tuple[str, ...] = ( + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + down_block_types: tuple[str, ...] = ( "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", "LTXVideoDownBlock3D", ), - decoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), - layers_per_block: Tuple[int, ...] = (4, 3, 3, 3, 4), - decoder_layers_per_block: Tuple[int, ...] = (4, 3, 3, 3, 4), - spatio_temporal_scaling: Tuple[bool, ...] = (True, True, True, False), - decoder_spatio_temporal_scaling: Tuple[bool, ...] = (True, True, True, False), - decoder_inject_noise: Tuple[bool, ...] = (False, False, False, False, False), - downsample_type: Tuple[str, ...] = ("conv", "conv", "conv", "conv"), - upsample_residual: Tuple[bool, ...] = (False, False, False, False), - upsample_factor: Tuple[int, ...] = (1, 1, 1, 1), + decoder_block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: tuple[int, ...] = (4, 3, 3, 3, 4), + decoder_layers_per_block: tuple[int, ...] = (4, 3, 3, 3, 4), + spatio_temporal_scaling: tuple[bool, ...] = (True, True, True, False), + decoder_spatio_temporal_scaling: tuple[bool, ...] = (True, True, True, False), + decoder_inject_noise: tuple[bool, ...] = (False, False, False, False, False), + downsample_type: tuple[str, ...] = ("conv", "conv", "conv", "conv"), + upsample_residual: tuple[bool, ...] = (False, False, False, False), + upsample_factor: tuple[int, ...] = (1, 1, 1, 1), timestep_conditioning: bool = False, patch_size: int = 4, patch_size_t: int = 1, @@ -1256,7 +1256,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -1282,7 +1282,7 @@ def encode( def _decode( self, z: torch.Tensor, temb: Optional[torch.Tensor] = None, return_dict: bool = True - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio @@ -1304,7 +1304,7 @@ def _decode( @apply_forward_hook def decode( self, z: torch.Tensor, temb: Optional[torch.Tensor] = None, return_dict: bool = True - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: """ Decode a batch of images. @@ -1411,7 +1411,7 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: def tiled_decode( self, z: torch.Tensor, temb: Optional[torch.Tensor], return_dict: bool = True - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1501,7 +1501,7 @@ def _temporal_tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: def _temporal_tiled_decode( self, z: torch.Tensor, temb: Optional[torch.Tensor], return_dict: bool = True - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 @@ -1544,7 +1544,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor | torch.Tensor: x = sample posterior = self.encode(x).latent_dist if sample_posterior: diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_magvit.py b/src/diffusers/models/autoencoders/autoencoder_kl_magvit.py index 43294a901f02..09ee1bf4be6e 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_magvit.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_magvit.py @@ -14,7 +14,7 @@ # limitations under the License. import math -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -37,10 +37,10 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, ...]] = 3, - stride: Union[int, Tuple[int, ...]] = 1, - padding: Union[int, Tuple[int, ...]] = 1, - dilation: Union[int, Tuple[int, ...]] = 1, + kernel_size: int | tuple[int, ...] = 3, + stride: int | tuple[int, ...] = 1, + padding: int | tuple[int, ...] = 1, + dilation: int | tuple[int, ...] = 1, groups: int = 1, bias: bool = True, padding_mode: str = "zeros", @@ -437,13 +437,13 @@ def __init__( self, in_channels: int = 3, out_channels: int = 8, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "SpatialDownBlock3D", "SpatialTemporalDownBlock3D", "SpatialTemporalDownBlock3D", "SpatialTemporalDownBlock3D", ), - block_out_channels: Tuple[int, ...] = [128, 256, 512, 512], + block_out_channels: tuple[int, ...] = [128, 256, 512, 512], layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -553,13 +553,13 @@ def __init__( self, in_channels: int = 8, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "SpatialUpBlock3D", "SpatialTemporalUpBlock3D", "SpatialTemporalUpBlock3D", "SpatialTemporalUpBlock3D", ), - block_out_channels: Tuple[int, ...] = [128, 256, 512, 512], + block_out_channels: tuple[int, ...] = [128, 256, 512, 512], layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -680,14 +680,14 @@ def __init__( in_channels: int = 3, latent_channels: int = 16, out_channels: int = 3, - block_out_channels: Tuple[int, ...] = [128, 256, 512, 512], - down_block_types: Tuple[str, ...] = [ + block_out_channels: tuple[int, ...] = [128, 256, 512, 512], + down_block_types: tuple[str, ...] = [ "SpatialDownBlock3D", "SpatialTemporalDownBlock3D", "SpatialTemporalDownBlock3D", "SpatialTemporalDownBlock3D", ], - up_block_types: Tuple[str, ...] = [ + up_block_types: tuple[str, ...] = [ "SpatialUpBlock3D", "SpatialTemporalUpBlock3D", "SpatialTemporalUpBlock3D", @@ -829,7 +829,7 @@ def disable_slicing(self) -> None: @apply_forward_hook def _encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -859,7 +859,7 @@ def _encode( @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -884,7 +884,7 @@ def encode( return (posterior,) return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio @@ -911,7 +911,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut return DecoderOutput(sample=dec) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: """ Decode a batch of images. @@ -1004,7 +1004,7 @@ def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> Autoencoder moments = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] return moments - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape sample_height = height * self.spatial_compression_ratio sample_width = width * self.spatial_compression_ratio @@ -1071,7 +1071,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_mochi.py b/src/diffusers/models/autoencoders/autoencoder_kl_mochi.py index 404d2f6d860a..c901446a8249 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_mochi.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_mochi.py @@ -14,7 +14,7 @@ # limitations under the License. import functools -from typing import Dict, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -106,7 +106,7 @@ def __init__( def forward( self, inputs: torch.Tensor, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: new_conv_cache = {} conv_cache = conv_cache or {} @@ -193,7 +193,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, chunk_size: int = 2**15, ) -> torch.Tensor: r"""Forward method of the `MochiUpBlock3D` class.""" @@ -294,7 +294,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""Forward method of the `MochiMidBlock3D` class.""" @@ -368,7 +368,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - conv_cache: Optional[Dict[str, torch.Tensor]] = None, + conv_cache: Optional[dict[str, torch.Tensor]] = None, ) -> torch.Tensor: r"""Forward method of the `MochiUpBlock3D` class.""" @@ -445,13 +445,13 @@ class MochiEncoder3D(nn.Module): The number of input channels. out_channels (`int`, *optional*): The number of output channels. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(128, 256, 512, 768)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(128, 256, 512, 768)`): The number of output channels for each block. - layers_per_block (`Tuple[int, ...]`, *optional*, defaults to `(3, 3, 4, 6, 3)`): + layers_per_block (`tuple[int, ...]`, *optional*, defaults to `(3, 3, 4, 6, 3)`): The number of resnet blocks for each block. - temporal_expansions (`Tuple[int, ...]`, *optional*, defaults to `(1, 2, 3)`): + temporal_expansions (`tuple[int, ...]`, *optional*, defaults to `(1, 2, 3)`): The temporal expansion factor for each of the up blocks. - spatial_expansions (`Tuple[int, ...]`, *optional*, defaults to `(2, 2, 2)`): + spatial_expansions (`tuple[int, ...]`, *optional*, defaults to `(2, 2, 2)`): The spatial expansion factor for each of the up blocks. non_linearity (`str`, *optional*, defaults to `"swish"`): The non-linearity to use in the decoder. @@ -461,11 +461,11 @@ def __init__( self, in_channels: int, out_channels: int, - block_out_channels: Tuple[int, ...] = (128, 256, 512, 768), - layers_per_block: Tuple[int, ...] = (3, 3, 4, 6, 3), - temporal_expansions: Tuple[int, ...] = (1, 2, 3), - spatial_expansions: Tuple[int, ...] = (2, 2, 2), - add_attention_block: Tuple[bool, ...] = (False, True, True, True, True), + block_out_channels: tuple[int, ...] = (128, 256, 512, 768), + layers_per_block: tuple[int, ...] = (3, 3, 4, 6, 3), + temporal_expansions: tuple[int, ...] = (1, 2, 3), + spatial_expansions: tuple[int, ...] = (2, 2, 2), + add_attention_block: tuple[bool, ...] = (False, True, True, True, True), act_fn: str = "swish", ): super().__init__() @@ -500,7 +500,7 @@ def __init__( self.gradient_checkpointing = False def forward( - self, hidden_states: torch.Tensor, conv_cache: Optional[Dict[str, torch.Tensor]] = None + self, hidden_states: torch.Tensor, conv_cache: Optional[dict[str, torch.Tensor]] = None ) -> torch.Tensor: r"""Forward method of the `MochiEncoder3D` class.""" @@ -558,13 +558,13 @@ class MochiDecoder3D(nn.Module): The number of input channels. out_channels (`int`, *optional*): The number of output channels. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(128, 256, 512, 768)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(128, 256, 512, 768)`): The number of output channels for each block. - layers_per_block (`Tuple[int, ...]`, *optional*, defaults to `(3, 3, 4, 6, 3)`): + layers_per_block (`tuple[int, ...]`, *optional*, defaults to `(3, 3, 4, 6, 3)`): The number of resnet blocks for each block. - temporal_expansions (`Tuple[int, ...]`, *optional*, defaults to `(1, 2, 3)`): + temporal_expansions (`tuple[int, ...]`, *optional*, defaults to `(1, 2, 3)`): The temporal expansion factor for each of the up blocks. - spatial_expansions (`Tuple[int, ...]`, *optional*, defaults to `(2, 2, 2)`): + spatial_expansions (`tuple[int, ...]`, *optional*, defaults to `(2, 2, 2)`): The spatial expansion factor for each of the up blocks. non_linearity (`str`, *optional*, defaults to `"swish"`): The non-linearity to use in the decoder. @@ -574,10 +574,10 @@ def __init__( self, in_channels: int, # 12 out_channels: int, # 3 - block_out_channels: Tuple[int, ...] = (128, 256, 512, 768), - layers_per_block: Tuple[int, ...] = (3, 3, 4, 6, 3), - temporal_expansions: Tuple[int, ...] = (1, 2, 3), - spatial_expansions: Tuple[int, ...] = (2, 2, 2), + block_out_channels: tuple[int, ...] = (128, 256, 512, 768), + layers_per_block: tuple[int, ...] = (3, 3, 4, 6, 3), + temporal_expansions: tuple[int, ...] = (1, 2, 3), + spatial_expansions: tuple[int, ...] = (2, 2, 2), act_fn: str = "swish", ): super().__init__() @@ -613,7 +613,7 @@ def __init__( self.gradient_checkpointing = False def forward( - self, hidden_states: torch.Tensor, conv_cache: Optional[Dict[str, torch.Tensor]] = None + self, hidden_states: torch.Tensor, conv_cache: Optional[dict[str, torch.Tensor]] = None ) -> torch.Tensor: r"""Forward method of the `MochiDecoder3D` class.""" @@ -668,8 +668,8 @@ class AutoencoderKLMochi(ModelMixin, ConfigMixin): Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of block output channels. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. scaling_factor (`float`, *optional*, defaults to `1.15258426`): The component-wise standard deviation of the trained latent space computed using the first batch of the @@ -688,15 +688,15 @@ def __init__( self, in_channels: int = 15, out_channels: int = 3, - encoder_block_out_channels: Tuple[int] = (64, 128, 256, 384), - decoder_block_out_channels: Tuple[int] = (128, 256, 512, 768), + encoder_block_out_channels: tuple[int] = (64, 128, 256, 384), + decoder_block_out_channels: tuple[int] = (128, 256, 512, 768), latent_channels: int = 12, - layers_per_block: Tuple[int, ...] = (3, 3, 4, 6, 3), + layers_per_block: tuple[int, ...] = (3, 3, 4, 6, 3), act_fn: str = "silu", - temporal_expansions: Tuple[int, ...] = (1, 2, 3), - spatial_expansions: Tuple[int, ...] = (2, 2, 2), - add_attention_block: Tuple[bool, ...] = (False, True, True, True, True), - latents_mean: Tuple[float, ...] = ( + temporal_expansions: tuple[int, ...] = (1, 2, 3), + spatial_expansions: tuple[int, ...] = (2, 2, 2), + add_attention_block: tuple[bool, ...] = (False, True, True, True, True), + latents_mean: tuple[float, ...] = ( -0.06730895953510081, -0.038011381506090416, -0.07477820912866141, @@ -710,7 +710,7 @@ def __init__( -0.011931556316503654, -0.0321993391887285, ), - latents_std: Tuple[float, ...] = ( + latents_std: tuple[float, ...] = ( 0.9263795028493863, 0.9248894543193766, 0.9393059390890617, @@ -881,7 +881,7 @@ def _encode(self, x: torch.Tensor) -> torch.Tensor: @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -906,7 +906,7 @@ def encode( return (posterior,) return AutoencoderKLOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: batch_size, num_channels, num_frames, height, width = z.shape tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio @@ -936,7 +936,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOut return DecoderOutput(sample=dec) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: """ Decode a batch of images. @@ -1034,7 +1034,7 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] return enc - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1118,7 +1118,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor | torch.Tensor: x = sample posterior = self.encode(x).latent_dist if sample_posterior: diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py b/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py index 9872cf096875..d155431aa9de 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py @@ -18,7 +18,7 @@ # - GitHub: https://github.com/Wan-Video/Wan2.1 # - arXiv: https://arxiv.org/abs/2503.20314 -from typing import List, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -58,9 +58,9 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int, int]], - stride: Union[int, Tuple[int, int, int]] = 1, - padding: Union[int, Tuple[int, int, int]] = 0, + kernel_size: int | tuple[int, int, int], + stride: int | tuple[int, int, int] = 1, + padding: int | tuple[int, int, int] = 0, ) -> None: super().__init__( in_channels=in_channels, @@ -679,13 +679,13 @@ def __init__( self, base_dim: int = 96, z_dim: int = 16, - dim_mult: Tuple[int] = [1, 2, 4, 4], + dim_mult: tuple[int] = [1, 2, 4, 4], num_res_blocks: int = 2, - attn_scales: List[float] = [], - temperal_downsample: List[bool] = [False, True, True], + attn_scales: list[float] = [], + temperal_downsample: list[bool] = [False, True, True], dropout: float = 0.0, - latents_mean: List[float] = [-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921], - latents_std: List[float] = [2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160], + latents_mean: list[float] = [-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921], + latents_std: list[float] = [2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160], ) -> None: # fmt: on super().__init__() @@ -827,7 +827,7 @@ def _encode(self, x: torch.Tensor): @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: r""" Encode a batch of images into latents. @@ -877,7 +877,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True): return DecoderOutput(sample=out) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images. @@ -983,7 +983,7 @@ def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] return enc - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1052,7 +1052,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: """ Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py b/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py index cf46e52564bf..7886cd019d3c 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools -from typing import Dict, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -31,7 +31,7 @@ def __init__( self, in_channels: int = 4, out_channels: int = 3, - block_out_channels: Tuple[int] = (128, 256, 512, 512), + block_out_channels: tuple[int] = (128, 256, 512, 512), layers_per_block: int = 2, ): super().__init__() @@ -145,10 +145,10 @@ class AutoencoderKLTemporalDecoder(ModelMixin, ConfigMixin): Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): - Tuple of downsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + tuple of downsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of block output channels. layers_per_block: (`int`, *optional*, defaults to 1): Number of layers per block. latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. sample_size (`int`, *optional*, defaults to `32`): Sample input size. @@ -172,8 +172,8 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str] = ("DownEncoderBlock2D",), - block_out_channels: Tuple[int] = (64,), + down_block_types: tuple[str] = ("DownEncoderBlock2D",), + block_out_channels: tuple[int] = (64,), layers_per_block: int = 1, latent_channels: int = 4, sample_size: int = 32, @@ -204,7 +204,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -213,7 +213,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -228,7 +228,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -278,7 +278,7 @@ def set_default_attn_processor(self): @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -308,7 +308,7 @@ def decode( z: torch.Tensor, num_frames: int, return_dict: bool = True, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: """ Decode a batch of images. @@ -339,7 +339,7 @@ def forward( return_dict: bool = True, generator: Optional[torch.Generator] = None, num_frames: int = 1, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_wan.py b/src/diffusers/models/autoencoders/autoencoder_kl_wan.py index f95c4cf37475..6b12492e18bf 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_wan.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_wan.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -149,9 +149,9 @@ def __init__( self, in_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int, int]], - stride: Union[int, Tuple[int, int, int]] = 1, - padding: Union[int, Tuple[int, int, int]] = 0, + kernel_size: int | tuple[int, int, int], + stride: int | tuple[int, int, int] = 1, + padding: int | tuple[int, int, int] = 0, ) -> None: super().__init__( in_channels=in_channels, @@ -968,12 +968,12 @@ def __init__( base_dim: int = 96, decoder_base_dim: Optional[int] = None, z_dim: int = 16, - dim_mult: Tuple[int] = [1, 2, 4, 4], + dim_mult: tuple[int] = [1, 2, 4, 4], num_res_blocks: int = 2, - attn_scales: List[float] = [], - temperal_downsample: List[bool] = [False, True, True], + attn_scales: list[float] = [], + temperal_downsample: list[bool] = [False, True, True], dropout: float = 0.0, - latents_mean: List[float] = [ + latents_mean: list[float] = [ -0.7571, -0.7089, -0.9113, @@ -991,7 +991,7 @@ def __init__( 0.2503, -0.2921, ], - latents_std: List[float] = [ + latents_std: list[float] = [ 2.8184, 1.4541, 2.3275, @@ -1171,7 +1171,7 @@ def _encode(self, x: torch.Tensor): @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> AutoencoderKLOutput | tuple[DiagonalGaussianDistribution]: r""" Encode a batch of images into latents. @@ -1227,7 +1227,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True): return DecoderOutput(sample=out) @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images. @@ -1333,7 +1333,7 @@ def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] return enc - def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Decode a batch of images using a tiled decoder. @@ -1402,7 +1402,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: """ Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_oobleck.py b/src/diffusers/models/autoencoders/autoencoder_oobleck.py index a10b616b4e25..ee2d0cd78705 100644 --- a/src/diffusers/models/autoencoders/autoencoder_oobleck.py +++ b/src/diffusers/models/autoencoders/autoencoder_oobleck.py @@ -13,7 +13,7 @@ # limitations under the License. import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -302,9 +302,9 @@ class AutoencoderOobleck(ModelMixin, ConfigMixin): Parameters: encoder_hidden_size (`int`, *optional*, defaults to 128): Intermediate representation dimension for the encoder. - downsampling_ratios (`List[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`): + downsampling_ratios (`list[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`): Ratios for downsampling in the encoder. These are used in reverse order for upsampling in the decoder. - channel_multiples (`List[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`): + channel_multiples (`list[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`): Multiples used to determine the hidden sizes of the hidden layers. decoder_channels (`int`, *optional*, defaults to 128): Intermediate representation dimension for the decoder. @@ -373,7 +373,7 @@ def disable_slicing(self): @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[AutoencoderOobleckOutput, Tuple[OobleckDiagonalGaussianDistribution]]: + ) -> AutoencoderOobleckOutput | tuple[OobleckDiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -399,7 +399,7 @@ def encode( return AutoencoderOobleckOutput(latent_dist=posterior) - def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[OobleckDecoderOutput, torch.Tensor]: + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> OobleckDecoderOutput | torch.Tensor: dec = self.decoder(z) if not return_dict: @@ -410,7 +410,7 @@ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[OobleckDec @apply_forward_hook def decode( self, z: torch.FloatTensor, return_dict: bool = True, generator=None - ) -> Union[OobleckDecoderOutput, torch.FloatTensor]: + ) -> OobleckDecoderOutput | torch.FloatTensor: """ Decode a batch of images. @@ -442,7 +442,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[OobleckDecoderOutput, torch.Tensor]: + ) -> OobleckDecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/autoencoder_tiny.py b/src/diffusers/models/autoencoders/autoencoder_tiny.py index 3e2b28606e29..ec6705a704a3 100644 --- a/src/diffusers/models/autoencoders/autoencoder_tiny.py +++ b/src/diffusers/models/autoencoders/autoencoder_tiny.py @@ -14,7 +14,7 @@ from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch @@ -50,11 +50,11 @@ class AutoencoderTiny(ModelMixin, ConfigMixin): Parameters: in_channels (`int`, *optional*, defaults to 3): Number of channels in the input image. out_channels (`int`, *optional*, defaults to 3): Number of channels in the output. - encoder_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): - Tuple of integers representing the number of output channels for each encoder block. The length of the + encoder_block_out_channels (`tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): + tuple of integers representing the number of output channels for each encoder block. The length of the tuple should be equal to the number of encoder blocks. - decoder_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): - Tuple of integers representing the number of output channels for each decoder block. The length of the + decoder_block_out_channels (`tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): + tuple of integers representing the number of output channels for each decoder block. The length of the tuple should be equal to the number of decoder blocks. act_fn (`str`, *optional*, defaults to `"relu"`): Activation function to be used throughout the model. @@ -64,12 +64,12 @@ class AutoencoderTiny(ModelMixin, ConfigMixin): upsampling_scaling_factor (`int`, *optional*, defaults to 2): Scaling factor for upsampling in the decoder. It determines the size of the output image during the upsampling process. - num_encoder_blocks (`Tuple[int]`, *optional*, defaults to `(1, 3, 3, 3)`): - Tuple of integers representing the number of encoder blocks at each stage of the encoding process. The + num_encoder_blocks (`tuple[int]`, *optional*, defaults to `(1, 3, 3, 3)`): + tuple of integers representing the number of encoder blocks at each stage of the encoding process. The length of the tuple should be equal to the number of stages in the encoder. Each stage has a different number of encoder blocks. - num_decoder_blocks (`Tuple[int]`, *optional*, defaults to `(3, 3, 3, 1)`): - Tuple of integers representing the number of decoder blocks at each stage of the decoding process. The + num_decoder_blocks (`tuple[int]`, *optional*, defaults to `(3, 3, 3, 1)`): + tuple of integers representing the number of decoder blocks at each stage of the decoding process. The length of the tuple should be equal to the number of stages in the decoder. Each stage has a different number of decoder blocks. latent_magnitude (`float`, *optional*, defaults to 3.0): @@ -99,14 +99,14 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - encoder_block_out_channels: Tuple[int, ...] = (64, 64, 64, 64), - decoder_block_out_channels: Tuple[int, ...] = (64, 64, 64, 64), + encoder_block_out_channels: tuple[int, ...] = (64, 64, 64, 64), + decoder_block_out_channels: tuple[int, ...] = (64, 64, 64, 64), act_fn: str = "relu", upsample_fn: str = "nearest", latent_channels: int = 4, upsampling_scaling_factor: int = 2, - num_encoder_blocks: Tuple[int, ...] = (1, 3, 3, 3), - num_decoder_blocks: Tuple[int, ...] = (3, 3, 3, 1), + num_encoder_blocks: tuple[int, ...] = (1, 3, 3, 3), + num_decoder_blocks: tuple[int, ...] = (3, 3, 3, 1), latent_magnitude: int = 3, latent_shift: float = 0.5, force_upcast: bool = False, @@ -287,7 +287,7 @@ def _tiled_decode(self, x: torch.Tensor) -> torch.Tensor: return out @apply_forward_hook - def encode(self, x: torch.Tensor, return_dict: bool = True) -> Union[AutoencoderTinyOutput, Tuple[torch.Tensor]]: + def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderTinyOutput | tuple[torch.Tensor]: if self.use_slicing and x.shape[0] > 1: output = [ self._tiled_encode(x_slice) if self.use_tiling else self.encoder(x_slice) for x_slice in x.split(1) @@ -304,7 +304,7 @@ def encode(self, x: torch.Tensor, return_dict: bool = True) -> Union[Autoencoder @apply_forward_hook def decode( self, x: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: if self.use_slicing and x.shape[0] > 1: output = [ self._tiled_decode(x_slice) if self.use_tiling else self.decoder(x_slice) for x_slice in x.split(1) @@ -322,7 +322,7 @@ def forward( self, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/consistency_decoder_vae.py b/src/diffusers/models/autoencoders/consistency_decoder_vae.py index b3017a878092..a626f020021d 100644 --- a/src/diffusers/models/autoencoders/consistency_decoder_vae.py +++ b/src/diffusers/models/autoencoders/consistency_decoder_vae.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Dict, Optional, Tuple, Union +from typing import Optional import torch import torch.nn.functional as F @@ -77,9 +77,9 @@ def __init__( latent_channels: int = 4, sample_size: int = 32, encoder_act_fn: str = "silu", - encoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + encoder_block_out_channels: tuple[int, ...] = (128, 256, 512, 512), encoder_double_z: bool = True, - encoder_down_block_types: Tuple[str, ...] = ( + encoder_down_block_types: tuple[str, ...] = ( "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", @@ -90,8 +90,8 @@ def __init__( encoder_norm_num_groups: int = 32, encoder_out_channels: int = 4, decoder_add_attention: bool = False, - decoder_block_out_channels: Tuple[int, ...] = (320, 640, 1024, 1024), - decoder_down_block_types: Tuple[str, ...] = ( + decoder_block_out_channels: tuple[int, ...] = (320, 640, 1024, 1024), + decoder_down_block_types: tuple[str, ...] = ( "ResnetDownsampleBlock2D", "ResnetDownsampleBlock2D", "ResnetDownsampleBlock2D", @@ -106,7 +106,7 @@ def __init__( decoder_out_channels: int = 6, decoder_resnet_time_scale_shift: str = "scale_shift", decoder_time_embedding_type: str = "learned", - decoder_up_block_types: Tuple[str, ...] = ( + decoder_up_block_types: tuple[str, ...] = ( "ResnetUpsampleBlock2D", "ResnetUpsampleBlock2D", "ResnetUpsampleBlock2D", @@ -202,7 +202,7 @@ def disable_slicing(self): @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -211,7 +211,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -226,7 +226,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -279,7 +279,7 @@ def set_default_attn_processor(self): @apply_forward_hook def encode( self, x: torch.Tensor, return_dict: bool = True - ) -> Union[ConsistencyDecoderVAEOutput, Tuple[DiagonalGaussianDistribution]]: + ) -> ConsistencyDecoderVAEOutput | tuple[DiagonalGaussianDistribution]: """ Encode a batch of images into latents. @@ -318,7 +318,7 @@ def decode( generator: Optional[torch.Generator] = None, return_dict: bool = True, num_inference_steps: int = 2, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: """ Decodes the input latent vector `z` using the consistency decoder VAE model. @@ -329,7 +329,7 @@ def decode( num_inference_steps (int): The number of inference steps. Default is 2. Returns: - Union[DecoderOutput, Tuple[torch.Tensor]]: The decoded output. + Union[DecoderOutput, tuple[torch.Tensor]]: The decoded output. """ z = (z * self.config.scaling_factor - self.means) / self.stds @@ -372,7 +372,7 @@ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch. b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent) return b - def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> Union[ConsistencyDecoderVAEOutput, Tuple]: + def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> ConsistencyDecoderVAEOutput | tuple: r"""Encode a batch of images using a tiled encoder. When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several @@ -433,7 +433,7 @@ def forward( sample_posterior: bool = False, return_dict: bool = True, generator: Optional[torch.Generator] = None, - ) -> Union[DecoderOutput, Tuple[torch.Tensor]]: + ) -> DecoderOutput | tuple[torch.Tensor]: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/models/autoencoders/vae.py b/src/diffusers/models/autoencoders/vae.py index 1d74d4f472d7..c8e40a2216b3 100644 --- a/src/diffusers/models/autoencoders/vae.py +++ b/src/diffusers/models/autoencoders/vae.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional import numpy as np import torch @@ -66,10 +66,10 @@ class Encoder(nn.Module): The number of input channels. out_channels (`int`, *optional*, defaults to 3): The number of output channels. - down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + down_block_types (`tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): The types of down blocks to use. See `~diffusers.models.unet_2d_blocks.get_down_block` for available options. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(64,)`): The number of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. @@ -85,8 +85,8 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), - block_out_channels: Tuple[int, ...] = (64,), + down_block_types: tuple[str, ...] = ("DownEncoderBlock2D",), + block_out_channels: tuple[int, ...] = (64,), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -187,9 +187,9 @@ class Decoder(nn.Module): The number of input channels. out_channels (`int`, *optional*, defaults to 3): The number of output channels. - up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + up_block_types (`tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(64,)`): The number of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. @@ -205,8 +205,8 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), - block_out_channels: Tuple[int, ...] = (64,), + up_block_types: tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: tuple[int, ...] = (64,), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -405,9 +405,9 @@ class MaskConditionDecoder(nn.Module): The number of input channels. out_channels (`int`, *optional*, defaults to 3): The number of output channels. - up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + up_block_types (`tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. - block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + block_out_channels (`tuple[int, ...]`, *optional*, defaults to `(64,)`): The number of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. @@ -423,8 +423,8 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), - block_out_channels: Tuple[int, ...] = (64,), + up_block_types: tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: tuple[int, ...] = (64,), layers_per_block: int = 2, norm_num_groups: int = 32, act_fn: str = "silu", @@ -636,7 +636,7 @@ def unmap_to_all(self, inds: torch.LongTensor) -> torch.LongTensor: back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds) return back.reshape(ishape) - def forward(self, z: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, Tuple]: + def forward(self, z: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, tuple]: # reshape z -> (batch, height, width, channel) and flatten z = z.permute(0, 2, 3, 1).contiguous() z_flattened = z.view(-1, self.vq_embed_dim) @@ -670,7 +670,7 @@ def forward(self, z: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, Tuple]: return z_q, loss, (perplexity, min_encodings, min_encoding_indices) - def get_codebook_entry(self, indices: torch.LongTensor, shape: Tuple[int, ...]) -> torch.Tensor: + def get_codebook_entry(self, indices: torch.LongTensor, shape: tuple[int, ...]) -> torch.Tensor: # shape specifying (batch, height, width, channel) if self.remap is not None: indices = indices.reshape(shape[0], -1) # add batch axis @@ -731,7 +731,7 @@ def kl(self, other: "DiagonalGaussianDistribution" = None) -> torch.Tensor: dim=[1, 2, 3], ) - def nll(self, sample: torch.Tensor, dims: Tuple[int, ...] = [1, 2, 3]) -> torch.Tensor: + def nll(self, sample: torch.Tensor, dims: tuple[int, ...] = [1, 2, 3]) -> torch.Tensor: if self.deterministic: return torch.Tensor([0.0]) logtwopi = np.log(2.0 * np.pi) @@ -764,10 +764,10 @@ class EncoderTiny(nn.Module): The number of input channels. out_channels (`int`): The number of output channels. - num_blocks (`Tuple[int, ...]`): + num_blocks (`tuple[int, ...]`): Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to use. - block_out_channels (`Tuple[int, ...]`): + block_out_channels (`tuple[int, ...]`): The number of output channels for each block. act_fn (`str`): The activation function to use. See `~diffusers.models.activations.get_activation` for available options. @@ -777,8 +777,8 @@ def __init__( self, in_channels: int, out_channels: int, - num_blocks: Tuple[int, ...], - block_out_channels: Tuple[int, ...], + num_blocks: tuple[int, ...], + block_out_channels: tuple[int, ...], act_fn: str, ): super().__init__() @@ -830,10 +830,10 @@ class DecoderTiny(nn.Module): The number of input channels. out_channels (`int`): The number of output channels. - num_blocks (`Tuple[int, ...]`): + num_blocks (`tuple[int, ...]`): Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to use. - block_out_channels (`Tuple[int, ...]`): + block_out_channels (`tuple[int, ...]`): The number of output channels for each block. upsampling_scaling_factor (`int`): The scaling factor to use for upsampling. @@ -845,8 +845,8 @@ def __init__( self, in_channels: int, out_channels: int, - num_blocks: Tuple[int, ...], - block_out_channels: Tuple[int, ...], + num_blocks: tuple[int, ...], + block_out_channels: tuple[int, ...], upsampling_scaling_factor: int, act_fn: str, upsample_fn: str, diff --git a/src/diffusers/models/autoencoders/vq_model.py b/src/diffusers/models/autoencoders/vq_model.py index c1094e62f7ec..e9db33835ef3 100644 --- a/src/diffusers/models/autoencoders/vq_model.py +++ b/src/diffusers/models/autoencoders/vq_model.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -47,12 +47,12 @@ class VQModel(ModelMixin, ConfigMixin): Parameters: in_channels (int, *optional*, defaults to 3): Number of channels in the input image. out_channels (int, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): - Tuple of downsample block types. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): - Tuple of upsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + tuple of downsample block types. + up_block_types (`tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + tuple of upsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(64,)`): + tuple of block output channels. layers_per_block (`int`, *optional*, defaults to `1`): Number of layers per block. act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. latent_channels (`int`, *optional*, defaults to `3`): Number of channels in the latent space. @@ -79,9 +79,9 @@ def __init__( self, in_channels: int = 3, out_channels: int = 3, - down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), - up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), - block_out_channels: Tuple[int, ...] = (64,), + down_block_types: tuple[str, ...] = ("DownEncoderBlock2D",), + up_block_types: tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: tuple[int, ...] = (64,), layers_per_block: int = 1, act_fn: str = "silu", latent_channels: int = 3, @@ -142,7 +142,7 @@ def encode(self, x: torch.Tensor, return_dict: bool = True) -> VQEncoderOutput: @apply_forward_hook def decode( self, h: torch.Tensor, force_not_quantize: bool = False, return_dict: bool = True, shape=None - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: # also go through quantization layer if not force_not_quantize: quant, commit_loss, _ = self.quantize(h) @@ -160,9 +160,7 @@ def decode( return DecoderOutput(sample=dec, commit_loss=commit_loss) - def forward( - self, sample: torch.Tensor, return_dict: bool = True - ) -> Union[DecoderOutput, Tuple[torch.Tensor, ...]]: + def forward(self, sample: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor, ...]: r""" The [`VQModel`] forward method. diff --git a/src/diffusers/models/controlnet.py b/src/diffusers/models/controlnet.py index c18bd8751dcb..dd8897b2ca71 100644 --- a/src/diffusers/models/controlnet.py +++ b/src/diffusers/models/controlnet.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional from ..utils import deprecate from .controlnets.controlnet import ( # noqa @@ -36,15 +36,15 @@ def __init__( conditioning_channels: int = 3, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -52,11 +52,11 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 1280, - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, + transformer_layers_per_block: int | tuple[int, ...] = 1, encoder_hid_dim: Optional[int] = None, encoder_hid_dim_type: Optional[str] = None, - attention_head_dim: Union[int, Tuple[int, ...]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + attention_head_dim: int | tuple[int, ...] = 8, + num_attention_heads: Optional[int | tuple[int, ...]] = None, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, addition_embed_type: Optional[str] = None, @@ -66,7 +66,7 @@ def __init__( resnet_time_scale_shift: str = "default", projection_class_embeddings_input_dim: Optional[int] = None, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), global_pool_conditions: bool = False, addition_embed_type_num_heads: int = 64, ): diff --git a/src/diffusers/models/controlnet_flux.py b/src/diffusers/models/controlnet_flux.py index e82748436d86..8c5f733a27d2 100644 --- a/src/diffusers/models/controlnet_flux.py +++ b/src/diffusers/models/controlnet_flux.py @@ -13,8 +13,6 @@ # limitations under the License. -from typing import List - from ..utils import deprecate, logging from .controlnets.controlnet_flux import FluxControlNetModel, FluxControlNetOutput, FluxMultiControlNetModel @@ -41,7 +39,7 @@ def __init__( joint_attention_dim: int = 4096, pooled_projection_dim: int = 768, guidance_embeds: bool = False, - axes_dims_rope: List[int] = [16, 56, 56], + axes_dims_rope: list[int] = [16, 56, 56], num_mode: int = None, conditioning_embedding_channels: int = None, ): diff --git a/src/diffusers/models/controlnet_sparsectrl.py b/src/diffusers/models/controlnet_sparsectrl.py index 5c67af4fe9c1..3d0e56f9c432 100644 --- a/src/diffusers/models/controlnet_sparsectrl.py +++ b/src/diffusers/models/controlnet_sparsectrl.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional from ..utils import deprecate, logging from .controlnets.controlnet_sparsectrl import ( # noqa @@ -50,14 +50,14 @@ def __init__( conditioning_channels: int = 4, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "DownBlockMotion", ), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -65,15 +65,15 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 768, - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, - transformer_layers_per_mid_block: Optional[Union[int, Tuple[int]]] = None, - temporal_transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, - attention_head_dim: Union[int, Tuple[int, ...]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + transformer_layers_per_block: int | tuple[int, ...] = 1, + transformer_layers_per_mid_block: Optional[int | tuple[int]] = None, + temporal_transformer_layers_per_block: int | tuple[int, ...] = 1, + attention_head_dim: int | tuple[int, ...] = 8, + num_attention_heads: Optional[int | tuple[int, ...]] = None, use_linear_projection: bool = False, upcast_attention: bool = False, resnet_time_scale_shift: str = "default", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), global_pool_conditions: bool = False, controlnet_conditioning_channel_order: str = "rgb", motion_max_seq_length: int = 32, diff --git a/src/diffusers/models/controlnets/controlnet.py b/src/diffusers/models/controlnets/controlnet.py index 3f330999d4c8..8d0435142e98 100644 --- a/src/diffusers/models/controlnets/controlnet.py +++ b/src/diffusers/models/controlnets/controlnet.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -57,7 +57,7 @@ class ControlNetOutput(BaseOutput): Output can be used to condition the original UNet's middle block activation. """ - down_block_res_samples: Tuple[torch.Tensor] + down_block_res_samples: tuple[torch.Tensor] mid_block_res_sample: torch.Tensor @@ -75,7 +75,7 @@ def __init__( self, conditioning_embedding_channels: int, conditioning_channels: int = 3, - block_out_channels: Tuple[int, ...] = (16, 32, 96, 256), + block_out_channels: tuple[int, ...] = (16, 32, 96, 256), ): super().__init__() @@ -119,7 +119,7 @@ class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The frequency shift to apply to the time embedding. down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. - only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`): + only_cross_attention (`Union[bool, tuple[bool]]`, defaults to `False`): block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, defaults to 2): @@ -137,7 +137,7 @@ class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The epsilon to use for the normalization. cross_attention_dim (`int`, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + transformer_layers_per_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. @@ -147,7 +147,7 @@ class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): encoder_hid_dim_type (`str`, *optional*, defaults to `None`): If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. - attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8): + attention_head_dim (`Union[int, tuple[int]]`, defaults to 8): The dimension of the attention heads. use_linear_projection (`bool`, defaults to `False`): class_embed_type (`str`, *optional*, defaults to `None`): @@ -184,15 +184,15 @@ def __init__( conditioning_channels: int = 3, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -200,11 +200,11 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 1280, - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, + transformer_layers_per_block: int | tuple[int, ...] = 1, encoder_hid_dim: Optional[int] = None, encoder_hid_dim_type: Optional[str] = None, - attention_head_dim: Union[int, Tuple[int, ...]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + attention_head_dim: int | tuple[int, ...] = 8, + num_attention_heads: Optional[int | tuple[int, ...]] = None, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, addition_embed_type: Optional[str] = None, @@ -214,7 +214,7 @@ def __init__( resnet_time_scale_shift: str = "default", projection_class_embeddings_input_dim: Optional[int] = None, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), global_pool_conditions: bool = False, addition_embed_type_num_heads: int = 64, ): @@ -444,7 +444,7 @@ def from_unet( cls, unet: UNet2DConditionModel, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), load_weights_from_unet: bool = True, conditioning_channels: int = 3, ): @@ -517,7 +517,7 @@ def from_unet( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -526,7 +526,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -541,7 +541,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -592,7 +592,7 @@ def set_default_attn_processor(self): self.set_attn_processor(processor) # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice - def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + def set_attention_slice(self, slice_size: str | int | list[int]) -> None: r""" Enable sliced attention computation. @@ -646,7 +646,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -660,18 +660,18 @@ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[i def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, controlnet_cond: torch.Tensor, conditioning_scale: float = 1.0, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guess_mode: bool = False, return_dict: bool = True, - ) -> Union[ControlNetOutput, Tuple[Tuple[torch.Tensor, ...], torch.Tensor]]: + ) -> ControlNetOutput | tuple[tuple[torch.Tensor, ...], torch.Tensor]: """ The [`ControlNetModel`] forward method. diff --git a/src/diffusers/models/controlnets/controlnet_flax.py b/src/diffusers/models/controlnets/controlnet_flax.py index f7a8b98fa2f0..1b98e220cb79 100644 --- a/src/diffusers/models/controlnets/controlnet_flax.py +++ b/src/diffusers/models/controlnets/controlnet_flax.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import flax import flax.linen as nn @@ -49,7 +49,7 @@ class FlaxControlNetOutput(BaseOutput): class FlaxControlNetConditioningEmbedding(nn.Module): conditioning_embedding_channels: int - block_out_channels: Tuple[int, ...] = (16, 32, 96, 256) + block_out_channels: tuple[int, ...] = (16, 32, 96, 256) dtype: jnp.dtype = jnp.float32 def setup(self) -> None: @@ -132,15 +132,15 @@ class FlaxControlNetModel(nn.Module, FlaxModelMixin, ConfigMixin): The size of the input sample. in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. - down_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): The tuple of downsample blocks to use. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. - attention_head_dim (`int` or `Tuple[int]`, *optional*, defaults to 8): + attention_head_dim (`int` or `tuple[int]`, *optional*, defaults to 8): The dimension of the attention heads. - num_attention_heads (`int` or `Tuple[int]`, *optional*): + num_attention_heads (`int` or `tuple[int]`, *optional*): The number of attention heads. cross_attention_dim (`int`, *optional*, defaults to 768): The dimension of the cross attention features. @@ -157,17 +157,17 @@ class FlaxControlNetModel(nn.Module, FlaxModelMixin, ConfigMixin): sample_size: int = 32 in_channels: int = 4 - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ) - only_cross_attention: Union[bool, Tuple[bool, ...]] = False - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280) + only_cross_attention: bool | tuple[bool, ...] = False + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280) layers_per_block: int = 2 - attention_head_dim: Union[int, Tuple[int, ...]] = 8 - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None + attention_head_dim: int | tuple[int, ...] = 8 + num_attention_heads: Optional[int | tuple[int, ...]] = None cross_attention_dim: int = 1280 dropout: float = 0.0 use_linear_projection: bool = False @@ -175,7 +175,7 @@ class FlaxControlNetModel(nn.Module, FlaxModelMixin, ConfigMixin): flip_sin_to_cos: bool = True freq_shift: int = 0 controlnet_conditioning_channel_order: str = "rgb" - conditioning_embedding_out_channels: Tuple[int, ...] = (16, 32, 96, 256) + conditioning_embedding_out_channels: tuple[int, ...] = (16, 32, 96, 256) def init_weights(self, rng: jax.Array) -> FrozenDict: # init input tensors @@ -327,13 +327,13 @@ def setup(self) -> None: def __call__( self, sample: jnp.ndarray, - timesteps: Union[jnp.ndarray, float, int], + timesteps: jnp.ndarray | float | int, encoder_hidden_states: jnp.ndarray, controlnet_cond: jnp.ndarray, conditioning_scale: float = 1.0, return_dict: bool = True, train: bool = False, - ) -> Union[FlaxControlNetOutput, Tuple[Tuple[jnp.ndarray, ...], jnp.ndarray]]: + ) -> FlaxControlNetOutput | tuple[tuple[jnp.ndarray, ...], jnp.ndarray]: r""" Args: sample (`jnp.ndarray`): (batch, channel, height, width) noisy inputs tensor diff --git a/src/diffusers/models/controlnets/controlnet_flux.py b/src/diffusers/models/controlnets/controlnet_flux.py index 063ff5bd8e2d..d2a37327db8a 100644 --- a/src/diffusers/models/controlnets/controlnet_flux.py +++ b/src/diffusers/models/controlnets/controlnet_flux.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -34,8 +34,8 @@ @dataclass class FluxControlNetOutput(BaseOutput): - controlnet_block_samples: Tuple[torch.Tensor] - controlnet_single_block_samples: Tuple[torch.Tensor] + controlnet_block_samples: tuple[torch.Tensor] + controlnet_single_block_samples: tuple[torch.Tensor] class FluxControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin): @@ -53,7 +53,7 @@ def __init__( joint_attention_dim: int = 4096, pooled_projection_dim: int = 768, guidance_embeds: bool = False, - axes_dims_rope: List[int] = [16, 56, 56], + axes_dims_rope: list[int] = [16, 56, 56], num_mode: int = None, conditioning_embedding_channels: int = None, ): @@ -129,7 +129,7 @@ def attn_processors(self): # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -222,9 +222,9 @@ def forward( img_ids: torch.Tensor = None, txt_ids: torch.Tensor = None, guidance: torch.Tensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[torch.FloatTensor, Transformer2DModelOutput]: + ) -> torch.FloatTensor | Transformer2DModelOutput: """ The [`FluxTransformer2DModel`] forward method. @@ -404,7 +404,7 @@ class FluxMultiControlNetModel(ModelMixin): compatible with `FluxControlNetModel`. Args: - controlnets (`List[FluxControlNetModel]`): + controlnets (`list[FluxControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `FluxControlNetModel` as a list. """ @@ -416,18 +416,18 @@ def __init__(self, controlnets): def forward( self, hidden_states: torch.FloatTensor, - controlnet_cond: List[torch.tensor], - controlnet_mode: List[torch.tensor], - conditioning_scale: List[float], + controlnet_cond: list[torch.tensor], + controlnet_mode: list[torch.tensor], + conditioning_scale: list[float], encoder_hidden_states: torch.Tensor = None, pooled_projections: torch.Tensor = None, timestep: torch.LongTensor = None, img_ids: torch.Tensor = None, txt_ids: torch.Tensor = None, guidance: torch.Tensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[FluxControlNetOutput, Tuple]: + ) -> FluxControlNetOutput | tuple: # ControlNet-Union with multiple conditions # only load one ControlNet for saving memories if len(self.nets) == 1: diff --git a/src/diffusers/models/controlnets/controlnet_hunyuan.py b/src/diffusers/models/controlnets/controlnet_hunyuan.py index d17d5692aa40..da1db28c96ec 100644 --- a/src/diffusers/models/controlnets/controlnet_hunyuan.py +++ b/src/diffusers/models/controlnets/controlnet_hunyuan.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Dict, Optional, Union +from typing import Optional import torch from torch import nn @@ -27,7 +27,7 @@ ) from ..modeling_utils import ModelMixin from ..transformers.hunyuan_transformer_2d import HunyuanDiTBlock -from .controlnet import Tuple, zero_module +from .controlnet import zero_module logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -35,7 +35,7 @@ @dataclass class HunyuanControlNetOutput(BaseOutput): - controlnet_block_samples: Tuple[torch.Tensor] + controlnet_block_samples: tuple[torch.Tensor] class HunyuanDiT2DControlNetModel(ModelMixin, ConfigMixin): @@ -116,7 +116,7 @@ def __init__( self.controlnet_blocks.append(controlnet_block) @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -125,7 +125,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) @@ -139,7 +139,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -317,7 +317,7 @@ class HunyuanDiT2DMultiControlNetModel(ModelMixin): designed to be compatible with `HunyuanDiT2DControlNetModel`. Args: - controlnets (`List[HunyuanDiT2DControlNetModel]`): + controlnets (`list[HunyuanDiT2DControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `HunyuanDiT2DControlNetModel` as a list. """ diff --git a/src/diffusers/models/controlnets/controlnet_qwenimage.py b/src/diffusers/models/controlnets/controlnet_qwenimage.py index 7c4955eb5828..e7750555c8db 100644 --- a/src/diffusers/models/controlnets/controlnet_qwenimage.py +++ b/src/diffusers/models/controlnets/controlnet_qwenimage.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -39,7 +39,7 @@ @dataclass class QwenImageControlNetOutput(BaseOutput): - controlnet_block_samples: Tuple[torch.Tensor] + controlnet_block_samples: tuple[torch.Tensor] class QwenImageControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, CacheMixin): @@ -55,7 +55,7 @@ def __init__( attention_head_dim: int = 128, num_attention_heads: int = 24, joint_attention_dim: int = 3584, - axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + axes_dims_rope: tuple[int, int, int] = (16, 56, 56), extra_condition_channels: int = 0, # for controlnet-inpainting ): super().__init__() @@ -103,7 +103,7 @@ def attn_processors(self): # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -188,11 +188,11 @@ def forward( encoder_hidden_states: torch.Tensor = None, encoder_hidden_states_mask: torch.Tensor = None, timestep: torch.LongTensor = None, - img_shapes: Optional[List[Tuple[int, int, int]]] = None, - txt_seq_lens: Optional[List[int]] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + img_shapes: Optional[list[tuple[int, int, int]]] = None, + txt_seq_lens: Optional[list[int]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[torch.FloatTensor, Transformer2DModelOutput]: + ) -> torch.FloatTensor | Transformer2DModelOutput: """ The [`FluxTransformer2DModel`] forward method. @@ -303,7 +303,7 @@ class QwenImageMultiControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin, F to be compatible with `QwenImageControlNetModel`. Args: - controlnets (`List[QwenImageControlNetModel]`): + controlnets (`list[QwenImageControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `QwenImageControlNetModel` as a list. """ @@ -315,16 +315,16 @@ def __init__(self, controlnets): def forward( self, hidden_states: torch.FloatTensor, - controlnet_cond: List[torch.tensor], - conditioning_scale: List[float], + controlnet_cond: list[torch.tensor], + conditioning_scale: list[float], encoder_hidden_states: torch.Tensor = None, encoder_hidden_states_mask: torch.Tensor = None, timestep: torch.LongTensor = None, - img_shapes: Optional[List[Tuple[int, int, int]]] = None, - txt_seq_lens: Optional[List[int]] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + img_shapes: Optional[list[tuple[int, int, int]]] = None, + txt_seq_lens: Optional[list[int]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[QwenImageControlNetOutput, Tuple]: + ) -> QwenImageControlNetOutput | tuple: # ControlNet-Union with multiple conditions # only load one ControlNet for saving memories if len(self.nets) == 1: diff --git a/src/diffusers/models/controlnets/controlnet_sana.py b/src/diffusers/models/controlnets/controlnet_sana.py index ed521adbedda..9ffa27bf759b 100644 --- a/src/diffusers/models/controlnets/controlnet_sana.py +++ b/src/diffusers/models/controlnets/controlnet_sana.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -35,7 +35,7 @@ @dataclass class SanaControlNetOutput(BaseOutput): - controlnet_block_samples: Tuple[torch.Tensor] + controlnet_block_samples: tuple[torch.Tensor] class SanaControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin): @@ -119,7 +119,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -128,7 +128,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -143,7 +143,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -186,9 +186,9 @@ def forward( conditioning_scale: float = 1.0, encoder_attention_mask: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor, ...], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor, ...] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/controlnets/controlnet_sd3.py b/src/diffusers/models/controlnets/controlnet_sd3.py index 0641c8bc0114..81c4ff826524 100644 --- a/src/diffusers/models/controlnets/controlnet_sd3.py +++ b/src/diffusers/models/controlnets/controlnet_sd3.py @@ -14,7 +14,7 @@ from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -36,7 +36,7 @@ @dataclass class SD3ControlNetOutput(BaseOutput): - controlnet_block_samples: Tuple[torch.Tensor] + controlnet_block_samples: tuple[torch.Tensor] class SD3ControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin): @@ -69,7 +69,7 @@ class SD3ControlNetModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginal The maximum latent height/width of positional embeddings. extra_conditioning_channels (`int`, defaults to `0`): The number of extra channels to use for conditioning for patch embedding. - dual_attention_layers (`Tuple[int, ...]`, defaults to `()`): + dual_attention_layers (`tuple[int, ...]`, defaults to `()`): The number of dual-stream transformer blocks to use. qk_norm (`str`, *optional*, defaults to `None`): The normalization to use for query and key in the attention layer. If `None`, no normalization is used. @@ -99,7 +99,7 @@ def __init__( out_channels: int = 16, pos_embed_max_size: int = 96, extra_conditioning_channels: int = 0, - dual_attention_layers: Tuple[int, ...] = (), + dual_attention_layers: tuple[int, ...] = (), qk_norm: Optional[str] = None, pos_embed_type: Optional[str] = "sincos", use_pos_embed: bool = True, @@ -206,7 +206,7 @@ def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -215,7 +215,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -230,7 +230,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -337,9 +337,9 @@ def forward( encoder_hidden_states: torch.Tensor = None, pooled_projections: torch.Tensor = None, timestep: torch.LongTensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + ) -> torch.Tensor | Transformer2DModelOutput: """ The [`SD3Transformer2DModel`] forward method. @@ -460,7 +460,7 @@ class SD3MultiControlNetModel(ModelMixin): compatible with `SD3ControlNetModel`. Args: - controlnets (`List[SD3ControlNetModel]`): + controlnets (`list[SD3ControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `SD3ControlNetModel` as a list. """ @@ -472,14 +472,14 @@ def __init__(self, controlnets): def forward( self, hidden_states: torch.Tensor, - controlnet_cond: List[torch.tensor], - conditioning_scale: List[float], + controlnet_cond: list[torch.tensor], + conditioning_scale: list[float], pooled_projections: torch.Tensor, encoder_hidden_states: torch.Tensor = None, timestep: torch.LongTensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[SD3ControlNetOutput, Tuple]: + ) -> SD3ControlNetOutput | tuple: for i, (image, scale, controlnet) in enumerate(zip(controlnet_cond, conditioning_scale, self.nets)): block_samples = controlnet( hidden_states=hidden_states, diff --git a/src/diffusers/models/controlnets/controlnet_sparsectrl.py b/src/diffusers/models/controlnets/controlnet_sparsectrl.py index b04be5726234..a63939c775e4 100644 --- a/src/diffusers/models/controlnets/controlnet_sparsectrl.py +++ b/src/diffusers/models/controlnets/controlnet_sparsectrl.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -55,7 +55,7 @@ class SparseControlNetOutput(BaseOutput): Output can be used to condition the original UNet's middle block activation. """ - down_block_res_samples: Tuple[torch.Tensor] + down_block_res_samples: tuple[torch.Tensor] mid_block_res_sample: torch.Tensor @@ -64,7 +64,7 @@ def __init__( self, conditioning_embedding_channels: int, conditioning_channels: int = 3, - block_out_channels: Tuple[int, ...] = (16, 32, 96, 256), + block_out_channels: tuple[int, ...] = (16, 32, 96, 256), ): super().__init__() @@ -110,7 +110,7 @@ class SparseControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The frequency shift to apply to the time embedding. down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. - only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`): + only_cross_attention (`Union[bool, tuple[bool]]`, defaults to `False`): block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, defaults to 2): @@ -128,28 +128,28 @@ class SparseControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The epsilon to use for the normalization. cross_attention_dim (`int`, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + transformer_layers_per_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. - transformer_layers_per_mid_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + transformer_layers_per_mid_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer layers to use in each layer in the middle block. - attention_head_dim (`int` or `Tuple[int]`, defaults to 8): + attention_head_dim (`int` or `tuple[int]`, defaults to 8): The dimension of the attention heads. - num_attention_heads (`int` or `Tuple[int]`, *optional*): + num_attention_heads (`int` or `tuple[int]`, *optional*): The number of heads to use for multi-head attention. use_linear_projection (`bool`, defaults to `False`): upcast_attention (`bool`, defaults to `False`): resnet_time_scale_shift (`str`, defaults to `"default"`): Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`. - conditioning_embedding_out_channels (`Tuple[int]`, defaults to `(16, 32, 96, 256)`): + conditioning_embedding_out_channels (`tuple[int]`, defaults to `(16, 32, 96, 256)`): The tuple of output channel for each block in the `conditioning_embedding` layer. global_pool_conditions (`bool`, defaults to `False`): TODO(Patrick) - unused parameter controlnet_conditioning_channel_order (`str`, defaults to `rgb`): motion_max_seq_length (`int`, defaults to `32`): The maximum sequence length to use in the motion module. - motion_num_attention_heads (`int` or `Tuple[int]`, defaults to `8`): + motion_num_attention_heads (`int` or `tuple[int]`, defaults to `8`): The number of heads to use in each attention layer of the motion module. concat_conditioning_mask (`bool`, defaults to `True`): use_simplified_condition_embedding (`bool`, defaults to `True`): @@ -164,14 +164,14 @@ def __init__( conditioning_channels: int = 4, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "DownBlockMotion", ), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -179,15 +179,15 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 768, - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, - transformer_layers_per_mid_block: Optional[Union[int, Tuple[int]]] = None, - temporal_transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, - attention_head_dim: Union[int, Tuple[int, ...]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + transformer_layers_per_block: int | tuple[int, ...] = 1, + transformer_layers_per_mid_block: Optional[int | tuple[int]] = None, + temporal_transformer_layers_per_block: int | tuple[int, ...] = 1, + attention_head_dim: int | tuple[int, ...] = 8, + num_attention_heads: Optional[int | tuple[int, ...]] = None, use_linear_projection: bool = False, upcast_attention: bool = False, resnet_time_scale_shift: str = "default", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), global_pool_conditions: bool = False, controlnet_conditioning_channel_order: str = "rgb", motion_max_seq_length: int = 32, @@ -389,7 +389,7 @@ def from_unet( cls, unet: UNet2DConditionModel, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), load_weights_from_unet: bool = True, conditioning_channels: int = 3, ) -> "SparseControlNetModel": @@ -450,7 +450,7 @@ def from_unet( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -459,7 +459,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -474,7 +474,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -525,7 +525,7 @@ def set_default_attn_processor(self): self.set_attn_processor(processor) # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice - def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + def set_attention_slice(self, slice_size: str | int | list[int]) -> None: r""" Enable sliced attention computation. @@ -579,7 +579,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -593,17 +593,17 @@ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[i def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, controlnet_cond: torch.Tensor, conditioning_scale: float = 1.0, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, conditioning_mask: Optional[torch.Tensor] = None, guess_mode: bool = False, return_dict: bool = True, - ) -> Union[SparseControlNetOutput, Tuple[Tuple[torch.Tensor, ...], torch.Tensor]]: + ) -> SparseControlNetOutput | tuple[tuple[torch.Tensor, ...], torch.Tensor]: """ The [`SparseControlNetModel`] forward method. diff --git a/src/diffusers/models/controlnets/controlnet_union.py b/src/diffusers/models/controlnets/controlnet_union.py index 3df3bbe312e9..61f1f4a79b1a 100644 --- a/src/diffusers/models/controlnets/controlnet_union.py +++ b/src/diffusers/models/controlnets/controlnet_union.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -94,7 +94,7 @@ class ControlNetUnionModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The frequency shift to apply to the time embedding. down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. - only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`): + only_cross_attention (`Union[bool, tuple[bool]]`, defaults to `False`): block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, defaults to 2): @@ -112,7 +112,7 @@ class ControlNetUnionModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): The epsilon to use for the normalization. cross_attention_dim (`int`, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + transformer_layers_per_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. @@ -122,7 +122,7 @@ class ControlNetUnionModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): encoder_hid_dim_type (`str`, *optional*, defaults to `None`): If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. - attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8): + attention_head_dim (`Union[int, tuple[int]]`, defaults to 8): The dimension of the attention heads. use_linear_projection (`bool`, defaults to `False`): class_embed_type (`str`, *optional*, defaults to `None`): @@ -156,14 +156,14 @@ def __init__( conditioning_channels: int = 3, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -171,11 +171,11 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 1280, - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, + transformer_layers_per_block: int | tuple[int, ...] = 1, encoder_hid_dim: Optional[int] = None, encoder_hid_dim_type: Optional[str] = None, - attention_head_dim: Union[int, Tuple[int, ...]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + attention_head_dim: int | tuple[int, ...] = 8, + num_attention_heads: Optional[int | tuple[int, ...]] = None, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, addition_embed_type: Optional[str] = None, @@ -185,7 +185,7 @@ def __init__( resnet_time_scale_shift: str = "default", projection_class_embeddings_input_dim: Optional[int] = None, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (48, 96, 192, 384), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (48, 96, 192, 384), global_pool_conditions: bool = False, addition_embed_type_num_heads: int = 64, num_control_type: int = 6, @@ -390,7 +390,7 @@ def from_unet( cls, unet: UNet2DConditionModel, controlnet_conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + conditioning_embedding_out_channels: Optional[tuple[int, ...]] = (16, 32, 96, 256), load_weights_from_unet: bool = True, ): r""" @@ -457,7 +457,7 @@ def from_unet( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -466,7 +466,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -481,7 +481,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -532,7 +532,7 @@ def set_default_attn_processor(self): self.set_attn_processor(processor) # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice - def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + def set_attention_slice(self, slice_size: str | int | list[int]) -> None: r""" Enable sliced attention computation. @@ -586,7 +586,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -600,21 +600,21 @@ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[i def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, - controlnet_cond: List[torch.Tensor], + controlnet_cond: list[torch.Tensor], control_type: torch.Tensor, - control_type_idx: List[int], - conditioning_scale: Union[float, List[float]] = 1.0, + control_type_idx: list[int], + conditioning_scale: float | list[float] = 1.0, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, from_multi: bool = False, guess_mode: bool = False, return_dict: bool = True, - ) -> Union[ControlNetOutput, Tuple[Tuple[torch.Tensor, ...], torch.Tensor]]: + ) -> ControlNetOutput | tuple[tuple[torch.Tensor, ...], torch.Tensor]: """ The [`ControlNetUnionModel`] forward method. @@ -625,12 +625,12 @@ def forward( The number of timesteps to denoise an input. encoder_hidden_states (`torch.Tensor`): The encoder hidden states. - controlnet_cond (`List[torch.Tensor]`): + controlnet_cond (`list[torch.Tensor]`): The conditional input tensors. control_type (`torch.Tensor`): A tensor of shape `(batch, num_control_type)` with values `0` or `1` depending on whether the control type is used. - control_type_idx (`List[int]`): + control_type_idx (`list[int]`): The indices of `control_type`. conditioning_scale (`float`, defaults to `1.0`): The scale factor for ControlNet outputs. diff --git a/src/diffusers/models/controlnets/controlnet_xs.py b/src/diffusers/models/controlnets/controlnet_xs.py index f5c69b9a46ad..f1854e5edb43 100644 --- a/src/diffusers/models/controlnets/controlnet_xs.py +++ b/src/diffusers/models/controlnets/controlnet_xs.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass from math import gcd -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Optional import torch from torch import Tensor, nn @@ -109,7 +109,7 @@ def get_down_block_adapter( temb_channels: int, max_norm_num_groups: Optional[int] = 32, has_crossattn=True, - transformer_layers_per_block: Optional[Union[int, Tuple[int]]] = 1, + transformer_layers_per_block: Optional[int | tuple[int]] = 1, num_attention_heads: Optional[int] = 1, cross_attention_dim: Optional[int] = 1024, add_downsample: bool = True, @@ -230,7 +230,7 @@ def get_mid_block_adapter( def get_up_block_adapter( out_channels: int, prev_output_channel: int, - ctrl_skip_channels: List[int], + ctrl_skip_channels: list[int], ): ctrl_to_base = [] num_layers = 3 # only support sd + sdxl @@ -278,7 +278,7 @@ class ControlNetXSAdapter(ModelMixin, ConfigMixin): The tuple of downsample blocks to use. sample_size (`int`, defaults to 96): Height and width of input/output sample. - transformer_layers_per_block (`Union[int, Tuple[int]]`, defaults to 1): + transformer_layers_per_block (`Union[int, tuple[int]]`, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. upcast_attention (`bool`, defaults to `True`): @@ -293,21 +293,21 @@ def __init__( self, conditioning_channels: int = 3, conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Tuple[int] = (16, 32, 96, 256), + conditioning_embedding_out_channels: tuple[int] = (16, 32, 96, 256), time_embedding_mix: float = 1.0, learn_time_embedding: bool = False, - num_attention_heads: Union[int, Tuple[int]] = 4, - block_out_channels: Tuple[int] = (4, 8, 16, 16), - base_block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + num_attention_heads: int | tuple[int] = 4, + block_out_channels: tuple[int] = (4, 8, 16, 16), + base_block_out_channels: tuple[int] = (320, 640, 1280, 1280), cross_attention_dim: int = 1024, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), sample_size: Optional[int] = 96, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, upcast_attention: bool = True, max_norm_num_groups: int = 32, use_linear_projection: bool = True, @@ -430,13 +430,13 @@ def from_unet( cls, unet: UNet2DConditionModel, size_ratio: Optional[float] = None, - block_out_channels: Optional[List[int]] = None, - num_attention_heads: Optional[List[int]] = None, + block_out_channels: Optional[list[int]] = None, + num_attention_heads: Optional[list[int]] = None, learn_time_embedding: bool = False, time_embedding_mix: int = 1.0, conditioning_channels: int = 3, conditioning_channel_order: str = "rgb", - conditioning_embedding_out_channels: Tuple[int] = (16, 32, 96, 256), + conditioning_embedding_out_channels: tuple[int] = (16, 32, 96, 256), ): r""" Instantiate a [`ControlNetXSAdapter`] from a [`UNet2DConditionModel`]. @@ -447,9 +447,9 @@ def from_unet( size_ratio (float, *optional*, defaults to `None`): When given, block_out_channels is set to a fraction of the base model's block_out_channels. Either this or `block_out_channels` must be given. - block_out_channels (`List[int]`, *optional*, defaults to `None`): + block_out_channels (`list[int]`, *optional*, defaults to `None`): Down blocks output channels in control model. Either this or `size_ratio` must be given. - num_attention_heads (`List[int]`, *optional*, defaults to `None`): + num_attention_heads (`list[int]`, *optional*, defaults to `None`): The dimension of the attention heads. The naming seems a bit confusing and it is, see https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 for why. learn_time_embedding (`bool`, defaults to `False`): @@ -461,7 +461,7 @@ def from_unet( Number of channels of conditioning input (e.g. an image) conditioning_channel_order (`str`, defaults to `"rgb"`): The channel order of conditional image. Will convert to `rgb` if it's `bgr`. - conditioning_embedding_out_channels (`Tuple[int]`, defaults to `(16, 32, 96, 256)`): + conditioning_embedding_out_channels (`tuple[int]`, defaults to `(16, 32, 96, 256)`): The tuple of output channel for each block in the `controlnet_cond_embedding` layer. """ @@ -529,18 +529,18 @@ def __init__( self, # unet configs sample_size: Optional[int] = 96, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), - up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), - block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + up_block_types: tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels: tuple[int] = (320, 640, 1280, 1280), norm_num_groups: Optional[int] = 32, - cross_attention_dim: Union[int, Tuple[int]] = 1024, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, - num_attention_heads: Union[int, Tuple[int]] = 8, + cross_attention_dim: int | tuple[int] = 1024, + transformer_layers_per_block: int | tuple[int] = 1, + num_attention_heads: int | tuple[int] = 8, addition_embed_type: Optional[str] = None, addition_time_embed_dim: Optional[int] = None, upcast_attention: bool = True, @@ -550,11 +550,11 @@ def __init__( # additional controlnet configs time_embedding_mix: float = 1.0, ctrl_conditioning_channels: int = 3, - ctrl_conditioning_embedding_out_channels: Tuple[int] = (16, 32, 96, 256), + ctrl_conditioning_embedding_out_channels: tuple[int] = (16, 32, 96, 256), ctrl_conditioning_channel_order: str = "rgb", ctrl_learn_time_embedding: bool = False, - ctrl_block_out_channels: Tuple[int] = (4, 8, 16, 16), - ctrl_num_attention_heads: Union[int, Tuple[int]] = 4, + ctrl_block_out_channels: tuple[int] = (4, 8, 16, 16), + ctrl_num_attention_heads: int | tuple[int] = 4, ctrl_max_norm_num_groups: int = 32, ): super().__init__() @@ -721,7 +721,7 @@ def from_unet( unet: UNet2DConditionModel, controlnet: Optional[ControlNetXSAdapter] = None, size_ratio: Optional[float] = None, - ctrl_block_out_channels: Optional[List[float]] = None, + ctrl_block_out_channels: Optional[list[float]] = None, time_embedding_mix: Optional[float] = None, ctrl_optional_kwargs: Optional[Dict] = None, ): @@ -737,7 +737,7 @@ def from_unet( adapter will be created. size_ratio (float, *optional*, defaults to `None`): Used to construct the controlnet if none is given. See [`ControlNetXSAdapter.from_unet`] for details. - ctrl_block_out_channels (`List[int]`, *optional*, defaults to `None`): + ctrl_block_out_channels (`list[int]`, *optional*, defaults to `None`): Used to construct the controlnet if none is given. See [`ControlNetXSAdapter.from_unet`] for details, where this parameter is called `block_out_channels`. time_embedding_mix (`float`, *optional*, defaults to None): @@ -865,7 +865,7 @@ def freeze_unet_params(self) -> None: @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -874,7 +874,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -889,7 +889,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -1008,18 +1008,18 @@ def unfuse_qkv_projections(self): def forward( self, sample: Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, controlnet_cond: Optional[torch.Tensor] = None, conditioning_scale: Optional[float] = 1.0, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, return_dict: bool = True, apply_control: bool = True, - ) -> Union[ControlNetXSOutput, Tuple]: + ) -> ControlNetXSOutput | tuple: """ The [`ControlNetXSModel`] forward method. @@ -1221,7 +1221,7 @@ def __init__( norm_num_groups: int = 32, ctrl_max_norm_num_groups: int = 32, has_crossattn=True, - transformer_layers_per_block: Optional[Union[int, Tuple[int]]] = 1, + transformer_layers_per_block: Optional[int | tuple[int]] = 1, base_num_attention_heads: Optional[int] = 1, ctrl_num_attention_heads: Optional[int] = 1, cross_attention_dim: Optional[int] = 1024, @@ -1420,10 +1420,10 @@ def forward( hidden_states_ctrl: Optional[Tensor] = None, conditioning_scale: Optional[float] = 1.0, attention_mask: Optional[Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[Tensor] = None, apply_control: bool = True, - ) -> Tuple[Tensor, Tensor, Tuple[Tensor, ...], Tuple[Tensor, ...]]: + ) -> tuple[Tensor, Tensor, tuple[Tensor, ...], tuple[Tensor, ...]]: if cross_attention_kwargs is not None: if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -1625,11 +1625,11 @@ def forward( encoder_hidden_states: Tensor, hidden_states_ctrl: Optional[Tensor] = None, conditioning_scale: Optional[float] = 1.0, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, attention_mask: Optional[Tensor] = None, encoder_attention_mask: Optional[Tensor] = None, apply_control: bool = True, - ) -> Tuple[Tensor, Tensor]: + ) -> tuple[Tensor, Tensor]: if cross_attention_kwargs is not None: if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -1661,7 +1661,7 @@ def __init__( in_channels: int, out_channels: int, prev_output_channel: int, - ctrl_skip_channels: List[int], + ctrl_skip_channels: list[int], temb_channels: int, norm_num_groups: int = 32, resolution_idx: Optional[int] = None, @@ -1806,12 +1806,12 @@ def freeze_base_params(self) -> None: def forward( self, hidden_states: Tensor, - res_hidden_states_tuple_base: Tuple[Tensor, ...], - res_hidden_states_tuple_ctrl: Tuple[Tensor, ...], + res_hidden_states_tuple_base: tuple[Tensor, ...], + res_hidden_states_tuple_ctrl: tuple[Tensor, ...], temb: Tensor, encoder_hidden_states: Optional[Tensor] = None, conditioning_scale: Optional[float] = 1.0, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, attention_mask: Optional[Tensor] = None, upsample_size: Optional[int] = None, encoder_attention_mask: Optional[Tensor] = None, diff --git a/src/diffusers/models/controlnets/multicontrolnet.py b/src/diffusers/models/controlnets/multicontrolnet.py index 87a952294997..1d4d4971d1b0 100644 --- a/src/diffusers/models/controlnets/multicontrolnet.py +++ b/src/diffusers/models/controlnets/multicontrolnet.py @@ -1,5 +1,5 @@ import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from torch import nn @@ -20,30 +20,30 @@ class MultiControlNetModel(ModelMixin): compatible with `ControlNetModel`. Args: - controlnets (`List[ControlNetModel]`): + controlnets (`list[ControlNetModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `ControlNetModel` as a list. """ - def __init__(self, controlnets: Union[List[ControlNetModel], Tuple[ControlNetModel]]): + def __init__(self, controlnets: list[ControlNetModel] | tuple[ControlNetModel]): super().__init__() self.nets = nn.ModuleList(controlnets) def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, - controlnet_cond: List[torch.tensor], - conditioning_scale: List[float], + controlnet_cond: list[torch.tensor], + conditioning_scale: list[float], class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guess_mode: bool = False, return_dict: bool = True, - ) -> Union[ControlNetOutput, Tuple]: + ) -> ControlNetOutput | tuple: for i, (image, scale, controlnet) in enumerate(zip(controlnet_cond, conditioning_scale, self.nets)): down_samples, mid_sample = controlnet( sample=sample, @@ -74,7 +74,7 @@ def forward( def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, is_main_process: bool = True, save_function: Callable = None, safe_serialization: bool = True, @@ -111,7 +111,7 @@ def save_pretrained( ) @classmethod - def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike]], **kwargs): + def from_pretrained(cls, pretrained_model_path: Optional[str | os.PathLike], **kwargs): r""" Instantiate a pretrained MultiControlNet model from multiple pre-trained controlnet models. @@ -134,7 +134,7 @@ def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike] Override the default `torch.dtype` and load the model under this dtype. output_loading_info(`bool`, *optional*, defaults to `False`): Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. diff --git a/src/diffusers/models/controlnets/multicontrolnet_union.py b/src/diffusers/models/controlnets/multicontrolnet_union.py index d5506dc186e3..4fbacfb77c88 100644 --- a/src/diffusers/models/controlnets/multicontrolnet_union.py +++ b/src/diffusers/models/controlnets/multicontrolnet_union.py @@ -1,5 +1,5 @@ import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from torch import nn @@ -21,32 +21,32 @@ class MultiControlNetUnionModel(ModelMixin): be compatible with `ControlNetUnionModel`. Args: - controlnets (`List[ControlNetUnionModel]`): + controlnets (`list[ControlNetUnionModel]`): Provides additional conditioning to the unet during the denoising process. You must set multiple `ControlNetUnionModel` as a list. """ - def __init__(self, controlnets: Union[List[ControlNetUnionModel], Tuple[ControlNetUnionModel]]): + def __init__(self, controlnets: list[ControlNetUnionModel] | tuple[ControlNetUnionModel]): super().__init__() self.nets = nn.ModuleList(controlnets) def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, - controlnet_cond: List[torch.tensor], - control_type: List[torch.Tensor], - control_type_idx: List[List[int]], - conditioning_scale: List[float], + controlnet_cond: list[torch.tensor], + control_type: list[torch.Tensor], + control_type_idx: list[list[int]], + conditioning_scale: list[float], class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guess_mode: bool = False, return_dict: bool = True, - ) -> Union[ControlNetOutput, Tuple]: + ) -> ControlNetOutput | tuple: down_block_res_samples, mid_block_res_sample = None, None for i, (image, ctype, ctype_idx, scale, controlnet) in enumerate( zip(controlnet_cond, control_type, control_type_idx, conditioning_scale, self.nets) @@ -86,7 +86,7 @@ def forward( # Copied from diffusers.models.controlnets.multicontrolnet.MultiControlNetModel.save_pretrained with ControlNet->ControlNetUnion def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, is_main_process: bool = True, save_function: Callable = None, safe_serialization: bool = True, @@ -124,7 +124,7 @@ def save_pretrained( @classmethod # Copied from diffusers.models.controlnets.multicontrolnet.MultiControlNetModel.from_pretrained with ControlNet->ControlNetUnion - def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike]], **kwargs): + def from_pretrained(cls, pretrained_model_path: Optional[str | os.PathLike], **kwargs): r""" Instantiate a pretrained MultiControlNetUnion model from multiple pre-trained controlnet models. @@ -147,7 +147,7 @@ def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike] Override the default `torch.dtype` and load the model under this dtype. output_loading_info(`bool`, *optional*, defaults to `False`): Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. diff --git a/src/diffusers/models/downsampling.py b/src/diffusers/models/downsampling.py index 505816422b2a..24a731c29634 100644 --- a/src/diffusers/models/downsampling.py +++ b/src/diffusers/models/downsampling.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Optional import torch import torch.nn as nn @@ -168,7 +168,7 @@ def __init__( channels: Optional[int] = None, out_channels: Optional[int] = None, use_conv: bool = False, - fir_kernel: Tuple[int, int, int, int] = (1, 3, 3, 1), + fir_kernel: tuple[int, int, int, int] = (1, 3, 3, 1), ): super().__init__() out_channels = out_channels if out_channels else channels diff --git a/src/diffusers/models/embeddings.py b/src/diffusers/models/embeddings.py index b51f5d7aec25..bc95b8e2b2ab 100644 --- a/src/diffusers/models/embeddings.py +++ b/src/diffusers/models/embeddings.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -80,7 +80,7 @@ def get_timestep_embedding( def get_3d_sincos_pos_embed( embed_dim: int, - spatial_size: Union[int, Tuple[int, int]], + spatial_size: int | tuple[int, int], temporal_size: int, spatial_interpolation_scale: float = 1.0, temporal_interpolation_scale: float = 1.0, @@ -93,7 +93,7 @@ def get_3d_sincos_pos_embed( Args: embed_dim (`int`): The embedding dimension of inputs. It must be divisible by 16. - spatial_size (`int` or `Tuple[int, int]`): + spatial_size (`int` or `tuple[int, int]`): The spatial dimension of positional embeddings. If an integer is provided, the same size is applied to both spatial dimensions (height and width). temporal_size (`int`): @@ -154,7 +154,7 @@ def get_3d_sincos_pos_embed( def _get_3d_sincos_pos_embed_np( embed_dim: int, - spatial_size: Union[int, Tuple[int, int]], + spatial_size: int | tuple[int, int], temporal_size: int, spatial_interpolation_scale: float = 1.0, temporal_interpolation_scale: float = 1.0, @@ -165,7 +165,7 @@ def _get_3d_sincos_pos_embed_np( Args: embed_dim (`int`): The embedding dimension of inputs. It must be divisible by 16. - spatial_size (`int` or `Tuple[int, int]`): + spatial_size (`int` or `tuple[int, int]`): The spatial dimension of positional embeddings. If an integer is provided, the same size is applied to both spatial dimensions (height and width). temporal_size (`int`): @@ -601,10 +601,10 @@ def forward(self, x, freqs_cis): Patchifies and embeds the input tensor(s). Args: - x (List[torch.Tensor] | torch.Tensor): The input tensor(s) to be patchified and embedded. + x (list[torch.Tensor] | torch.Tensor): The input tensor(s) to be patchified and embedded. Returns: - Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], torch.Tensor]: A tuple containing the patchified + tuple[torch.Tensor, torch.Tensor, list[tuple[int, int]], torch.Tensor]: A tuple containing the patchified and embedded tensor(s), the mask indicating the valid patches, the original image size(s), and the frequency tensor(s). """ @@ -828,18 +828,18 @@ def get_3d_rotary_pos_embed( theta: int = 10000, use_real: bool = True, grid_type: str = "linspace", - max_size: Optional[Tuple[int, int]] = None, + max_size: Optional[tuple[int, int]] = None, device: Optional[torch.device] = None, -) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """ RoPE for video tokens with 3D structure. Args: embed_dim: (`int`): The embedding dimension size, corresponding to hidden_size_head. - crops_coords (`Tuple[int]`): + crops_coords (`tuple[int]`): The top-left and bottom-right coordinates of the crop. - grid_size (`Tuple[int]`): + grid_size (`tuple[int]`): The grid size of the spatial positional embedding (height, width). temporal_size (`int`): The size of the temporal dimension. @@ -926,10 +926,10 @@ def get_3d_rotary_pos_embed_allegro( crops_coords, grid_size, temporal_size, - interpolation_scale: Tuple[float, float, float] = (1.0, 1.0, 1.0), + interpolation_scale: tuple[float, float, float] = (1.0, 1.0, 1.0), theta: int = 10000, device: Optional[torch.device] = None, -) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: # TODO(aryan): docs start, stop = crops_coords grid_size_h, grid_size_w = grid_size @@ -973,9 +973,9 @@ def get_2d_rotary_pos_embed( Args: embed_dim: (`int`): The embedding dimension size - crops_coords (`Tuple[int]`) + crops_coords (`tuple[int]`) The top-left and bottom-right coordinates of the crop. - grid_size (`Tuple[int]`): + grid_size (`tuple[int]`): The grid size of the positional embedding. use_real (`bool`): If True, return real part and imaginary part separately. Otherwise, return complex numbers. @@ -1021,9 +1021,9 @@ def _get_2d_rotary_pos_embed_np(embed_dim, crops_coords, grid_size, use_real=Tru Args: embed_dim: (`int`): The embedding dimension size - crops_coords (`Tuple[int]`) + crops_coords (`tuple[int]`) The top-left and bottom-right coordinates of the crop. - grid_size (`Tuple[int]`): + grid_size (`tuple[int]`): The grid size of the positional embedding. use_real (`bool`): If True, return real part and imaginary part separately. Otherwise, return complex numbers. @@ -1111,7 +1111,7 @@ def get_2d_rotary_pos_embed_lumina(embed_dim, len_h, len_w, linear_factor=1.0, n def get_1d_rotary_pos_embed( dim: int, - pos: Union[np.ndarray, int], + pos: np.ndarray | int, theta: float = 10000.0, use_real=False, linear_factor=1.0, @@ -1178,11 +1178,11 @@ def get_1d_rotary_pos_embed( def apply_rotary_emb( x: torch.Tensor, - freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + freqs_cis: torch.Tensor | tuple[torch.Tensor], use_real: bool = True, use_real_unbind_dim: int = -1, sequence_dim: int = 2, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> tuple[torch.Tensor, torch.Tensor]: """ Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are @@ -1192,10 +1192,10 @@ def apply_rotary_emb( Args: x (`torch.Tensor`): Query or key tensor to apply rotary embeddings. [B, H, S, D] xk (torch.Tensor): Key tensor to apply - freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) Returns: - Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings. """ if use_real: cos, sin = freqs_cis # [S, D] @@ -2535,7 +2535,7 @@ def __init__( self.time_proj = Timesteps(timestep_in_dim, timestep_flip_sin_to_cos, timestep_freq_shift) self.time_embedding = TimestepEmbedding(timestep_in_dim, hidden_dim, act_fn="silu") - def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """Forward pass. Args: @@ -2544,7 +2544,7 @@ def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> Tuple[torch.Tensor timestep (`torch.Tensor`): Timestep in denoising process. Returns: - `Tuple`[`torch.Tensor`, `torch.Tensor`]: The pair (latents, timestep_emb). + `tuple`[`torch.Tensor`, `torch.Tensor`]: The pair (latents, timestep_emb). """ timestep_emb = self.time_proj(timestep).to(dtype=x.dtype) timestep_emb = self.time_embedding(timestep_emb) @@ -2564,7 +2564,7 @@ def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> Tuple[torch.Tensor class MultiIPAdapterImageProjection(nn.Module): - def __init__(self, IPAdapterImageProjectionLayers: Union[List[nn.Module], Tuple[nn.Module]]): + def __init__(self, IPAdapterImageProjectionLayers: list[nn.Module] | tuple[nn.Module]): super().__init__() self.image_projection_layers = nn.ModuleList(IPAdapterImageProjectionLayers) @@ -2573,7 +2573,7 @@ def num_ip_adapters(self) -> int: """Number of IP-Adapters loaded.""" return len(self.image_projection_layers) - def forward(self, image_embeds: List[torch.Tensor]): + def forward(self, image_embeds: list[torch.Tensor]): projected_image_embeds = [] # currently, we accept `image_embeds` as diff --git a/src/diffusers/models/lora.py b/src/diffusers/models/lora.py index 85d61d6d7cdf..ecd63688afaf 100644 --- a/src/diffusers/models/lora.py +++ b/src/diffusers/models/lora.py @@ -21,7 +21,7 @@ # ----------------------------------------------------------------# ################################################################### -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn.functional as F @@ -199,7 +199,7 @@ def __init__( out_features: int, rank: int = 4, network_alpha: Optional[float] = None, - device: Optional[Union[torch.device, str]] = None, + device: Optional[torch.device | str] = None, dtype: Optional[torch.dtype] = None, ): super().__init__() @@ -260,9 +260,9 @@ def __init__( in_features: int, out_features: int, rank: int = 4, - kernel_size: Union[int, Tuple[int, int]] = (1, 1), - stride: Union[int, Tuple[int, int]] = (1, 1), - padding: Union[int, Tuple[int, int], str] = 0, + kernel_size: int | tuple[int, int] = (1, 1), + stride: int | tuple[int, int] = (1, 1), + padding: int | tuple[int, int] | str = 0, network_alpha: Optional[float] = None, ): super().__init__() diff --git a/src/diffusers/models/model_loading_utils.py b/src/diffusers/models/model_loading_utils.py index 8b48ba6b4873..070e88d51d7b 100644 --- a/src/diffusers/models/model_loading_utils.py +++ b/src/diffusers/models/model_loading_utils.py @@ -22,7 +22,7 @@ from collections import OrderedDict, defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Optional from zipfile import is_zipfile import safetensors @@ -135,7 +135,7 @@ def _fetch_remapped_cls_from_config(config, old_class): return old_class -def _determine_param_device(param_name: str, device_map: Optional[Dict[str, Union[int, str, torch.device]]]): +def _determine_param_device(param_name: str, device_map: Optional[dict[str, int | str | torch.device]]): """ Find the device of param_name from the device_map. """ @@ -153,10 +153,10 @@ def _determine_param_device(param_name: str, device_map: Optional[Dict[str, Unio def load_state_dict( - checkpoint_file: Union[str, os.PathLike], - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + checkpoint_file: str | os.PathLike, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, disable_mmap: bool = False, - map_location: Union[str, torch.device] = "cpu", + map_location: str | torch.device = "cpu", ): """ Reads a checkpoint file, returning properly formatted errors if they arise. @@ -213,17 +213,17 @@ def load_state_dict( def load_model_dict_into_meta( model, state_dict: OrderedDict, - dtype: Optional[Union[str, torch.dtype]] = None, + dtype: Optional[str | torch.dtype] = None, model_name_or_path: Optional[str] = None, hf_quantizer: Optional[DiffusersQuantizer] = None, - keep_in_fp32_modules: Optional[List] = None, - device_map: Optional[Dict[str, Union[int, str, torch.device]]] = None, - unexpected_keys: Optional[List[str]] = None, - offload_folder: Optional[Union[str, os.PathLike]] = None, + keep_in_fp32_modules: Optional[list] = None, + device_map: Optional[dict[str, int | str | torch.device]] = None, + unexpected_keys: Optional[list[str]] = None, + offload_folder: Optional[str | os.PathLike] = None, offload_index: Optional[Dict] = None, state_dict_index: Optional[Dict] = None, - state_dict_folder: Optional[Union[str, os.PathLike]] = None, -) -> List[str]: + state_dict_folder: Optional[str | os.PathLike] = None, +) -> list[str]: """ This is somewhat similar to `_load_state_dict_into_model`, but deals with a model that has some or all of its params on a `meta` device. It replaces the model params with the data from the `state_dict` @@ -466,7 +466,7 @@ def _find_mismatched_keys( def _load_state_dict_into_model( model_to_load, state_dict: OrderedDict, assign_to_params_buffers: bool = False -) -> List[str]: +) -> list[str]: # Convert old format to new format if needed from a PyTorch state_dict # copy state_dict so _load_from_state_dict can modify it state_dict = state_dict.copy() @@ -505,7 +505,7 @@ def _fetch_index_file( revision, user_agent, commit_hash, - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, ): if is_local: index_file = Path( @@ -555,7 +555,7 @@ def _fetch_index_file_legacy( revision, user_agent, commit_hash, - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, ): if is_local: index_file = Path( @@ -714,7 +714,7 @@ def _expand_device_map(device_map, param_names): # Adapted from: https://github.com/huggingface/transformers/blob/0687d481e2c71544501ef9cb3eef795a6e79b1de/src/transformers/modeling_utils.py#L5859 def _caching_allocator_warmup( - model, expanded_device_map: Dict[str, torch.device], dtype: torch.dtype, hf_quantizer: Optional[DiffusersQuantizer] + model, expanded_device_map: dict[str, torch.device], dtype: torch.dtype, hf_quantizer: Optional[DiffusersQuantizer] ) -> None: """ This function warm-ups the caching allocator based on the size of the model tensors that will reside on each diff --git a/src/diffusers/models/modeling_flax_utils.py b/src/diffusers/models/modeling_flax_utils.py index 3f060993190f..8725b3f073f2 100644 --- a/src/diffusers/models/modeling_flax_utils.py +++ b/src/diffusers/models/modeling_flax_utils.py @@ -15,7 +15,7 @@ import os from pickle import UnpicklingError -from typing import Any, Dict, Union +from typing import Any, Dict import jax import jax.numpy as jnp @@ -68,7 +68,7 @@ def _from_config(cls, config, **kwargs): """ return cls(config, **kwargs) - def _cast_floating_to(self, params: Union[Dict, FrozenDict], dtype: jnp.dtype, mask: Any = None) -> Any: + def _cast_floating_to(self, params: Dict | FrozenDict, dtype: jnp.dtype, mask: Any = None) -> Any: """ Helper method to cast floating-point values of given parameter `PyTree` to given `dtype`. """ @@ -92,7 +92,7 @@ def conditional_cast(param): return unflatten_dict(flat_params) - def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None): + def to_bf16(self, params: Dict | FrozenDict, mask: Any = None): r""" Cast the floating-point `params` to `jax.numpy.bfloat16`. This returns a new `params` tree and does not cast the `params` in place. @@ -131,7 +131,7 @@ def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None): ```""" return self._cast_floating_to(params, jnp.bfloat16, mask) - def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None): + def to_fp32(self, params: Dict | FrozenDict, mask: Any = None): r""" Cast the floating-point `params` to `jax.numpy.float32`. This method can be used to explicitly convert the model parameters to fp32 precision. This returns a new `params` tree and does not cast the `params` in place. @@ -158,7 +158,7 @@ def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None): ```""" return self._cast_floating_to(params, jnp.float32, mask) - def to_fp16(self, params: Union[Dict, FrozenDict], mask: Any = None): + def to_fp16(self, params: Dict | FrozenDict, mask: Any = None): r""" Cast the floating-point `params` to `jax.numpy.float16`. This returns a new `params` tree and does not cast the `params` in place. @@ -204,7 +204,7 @@ def init_weights(self, rng: jax.Array) -> Dict: @validate_hf_hub_args def from_pretrained( cls, - pretrained_model_name_or_path: Union[str, os.PathLike], + pretrained_model_name_or_path: str | os.PathLike, dtype: jnp.dtype = jnp.float32, *model_args, **kwargs, @@ -240,7 +240,7 @@ def from_pretrained( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only(`bool`, *optional*, defaults to `False`): @@ -493,8 +493,8 @@ def from_pretrained( def save_pretrained( self, - save_directory: Union[str, os.PathLike], - params: Union[Dict, FrozenDict], + save_directory: str | os.PathLike, + params: Dict | FrozenDict, is_main_process: bool = True, push_to_hub: bool = False, **kwargs, @@ -516,7 +516,7 @@ def save_pretrained( Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index ce26785f63ea..1147952bd793 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -27,7 +27,7 @@ from contextlib import ExitStack, contextmanager from functools import wraps from pathlib import Path -from typing import Any, Callable, ContextManager, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Callable, ContextManager, Optional, Type import safetensors import torch @@ -84,7 +84,7 @@ class ContextManagers: in the `fastcore` library. """ - def __init__(self, context_managers: List[ContextManager]): + def __init__(self, context_managers: list[ContextManager]): self.context_managers = context_managers self.stack = ExitStack() @@ -146,7 +146,7 @@ def get_parameter_device(parameter: torch.nn.Module) -> torch.device: except StopIteration: # For torch.nn.DataParallel compatibility in PyTorch 1.5 - def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + def find_tensor_attributes(module: torch.nn.Module) -> list[tuple[str, Tensor]]: tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] return tuples @@ -194,7 +194,7 @@ def get_parameter_dtype(parameter: torch.nn.Module) -> torch.dtype: return last_dtype # For nn.DataParallel compatibility in PyTorch > 1.5 - def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]: + def find_tensor_attributes(module: nn.Module) -> list[tuple[str, Tensor]]: tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] return tuples @@ -438,8 +438,8 @@ def enable_layerwise_casting( self, storage_dtype: torch.dtype = torch.float8_e4m3fn, compute_dtype: Optional[torch.dtype] = None, - skip_modules_pattern: Optional[Tuple[str, ...]] = None, - skip_modules_classes: Optional[Tuple[Type[torch.nn.Module], ...]] = None, + skip_modules_pattern: Optional[tuple[str, ...]] = None, + skip_modules_classes: Optional[tuple[Type[torch.nn.Module], ...]] = None, non_blocking: bool = False, ) -> None: r""" @@ -475,11 +475,11 @@ def enable_layerwise_casting( The dtype to which the model should be cast for storage. compute_dtype (`torch.dtype`): The dtype to which the model weights should be cast during the forward pass. - skip_modules_pattern (`Tuple[str, ...]`, *optional*): + skip_modules_pattern (`tuple[str, ...]`, *optional*): A list of patterns to match the names of the modules to skip during the layerwise casting process. If set to `None`, default skip patterns are used to ignore certain internal layers of modules and PEFT layers. - skip_modules_classes (`Tuple[Type[torch.nn.Module], ...]`, *optional*): + skip_modules_classes (`tuple[Type[torch.nn.Module], ...]`, *optional*): A list of module classes to skip during the layerwise casting process. non_blocking (`bool`, *optional*, defaults to `False`): If `True`, the weight casting operations are non-blocking. @@ -638,12 +638,12 @@ def reset_attention_backend(self) -> None: def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, is_main_process: bool = True, save_function: Optional[Callable] = None, safe_serialization: bool = True, variant: Optional[str] = None, - max_shard_size: Union[int, str] = "10GB", + max_shard_size: int | str = "10GB", push_to_hub: bool = False, **kwargs, ): @@ -677,7 +677,7 @@ def save_pretrained( Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -805,7 +805,7 @@ def dequantize(self): @classmethod @validate_hf_hub_args - def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs) -> Self: + def from_pretrained(cls, pretrained_model_name_or_path: Optional[str | os.PathLike], **kwargs) -> Self: r""" Instantiate a pretrained PyTorch model from a pretrained model configuration. @@ -829,7 +829,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info (`bool`, *optional*, defaults to `False`): @@ -851,7 +851,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P Mirror source to resolve accessibility issues if you're downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. - device_map (`Union[int, str, torch.device]` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`Union[int, str, torch.device]` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be defined for each parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the same device. Defaults to `None`, meaning that the model will be loaded on CPU. @@ -953,9 +953,9 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P variant = kwargs.pop("variant", None) use_safetensors = kwargs.pop("use_safetensors", None) quantization_config = kwargs.pop("quantization_config", None) - dduf_entries: Optional[Dict[str, DDUFEntry]] = kwargs.pop("dduf_entries", None) + dduf_entries: Optional[dict[str, DDUFEntry]] = kwargs.pop("dduf_entries", None) disable_mmap = kwargs.pop("disable_mmap", False) - parallel_config: Optional[Union[ParallelConfig, ContextParallelConfig]] = kwargs.pop("parallel_config", None) + parallel_config: Optional[ParallelConfig | ContextParallelConfig] = kwargs.pop("parallel_config", None) is_parallel_loading_enabled = HF_ENABLE_PARALLEL_LOADING if is_parallel_loading_enabled and not low_cpu_mem_usage: @@ -1480,8 +1480,8 @@ def compile_repeated_blocks(self, *args, **kwargs): def enable_parallelism( self, *, - config: Union[ParallelConfig, ContextParallelConfig], - cp_plan: Optional[Dict[str, ContextParallelModelPlan]] = None, + config: ParallelConfig | ContextParallelConfig, + cp_plan: Optional[dict[str, ContextParallelModelPlan]] = None, ): from ..hooks.context_parallel import apply_context_parallel from .attention import AttentionModuleMixin @@ -1549,19 +1549,19 @@ def _load_pretrained_model( cls, model, state_dict: OrderedDict, - resolved_model_file: List[str], - pretrained_model_name_or_path: Union[str, os.PathLike], - loaded_keys: List[str], + resolved_model_file: list[str], + pretrained_model_name_or_path: str | os.PathLike, + loaded_keys: list[str], ignore_mismatched_sizes: bool = False, assign_to_params_buffers: bool = False, hf_quantizer: Optional[DiffusersQuantizer] = None, low_cpu_mem_usage: bool = True, - dtype: Optional[Union[str, torch.dtype]] = None, - keep_in_fp32_modules: Optional[List[str]] = None, - device_map: Union[str, int, torch.device, Dict[str, Union[int, str, torch.device]]] = None, + dtype: Optional[str | torch.dtype] = None, + keep_in_fp32_modules: Optional[list[str]] = None, + device_map: str | int | torch.device | dict[str, int | str | torch.device] = None, offload_state_dict: Optional[bool] = None, - offload_folder: Optional[Union[str, os.PathLike]] = None, - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + offload_folder: Optional[str | os.PathLike] = None, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, is_parallel_loading_enabled: Optional[bool] = False, ): model_state_dict = model.state_dict() @@ -1721,7 +1721,7 @@ def _get_no_split_modules(self, device_map: str): The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"] Returns: - `List[str]`: List of modules that should not be split + `list[str]`: list of modules that should not be split """ _no_split_modules = set() modules_to_check = [self] @@ -1942,7 +1942,7 @@ class LegacyModelMixin(ModelMixin): @classmethod @validate_hf_hub_args - def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + def from_pretrained(cls, pretrained_model_name_or_path: Optional[str | os.PathLike], **kwargs): # To prevent dependency import problem. from .model_loading_utils import _fetch_remapped_cls_from_config diff --git a/src/diffusers/models/normalization.py b/src/diffusers/models/normalization.py index ae2a6298f5f7..6502d5e4ef93 100644 --- a/src/diffusers/models/normalization.py +++ b/src/diffusers/models/normalization.py @@ -14,7 +14,7 @@ # limitations under the License. import numbers -from typing import Dict, Optional, Tuple +from typing import Optional import torch import torch.nn as nn @@ -117,7 +117,7 @@ def forward( self, hidden_states: torch.Tensor, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, ...]: + ) -> tuple[torch.Tensor, ...]: emb = self.linear(self.silu(emb)) shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp, shift_msa2, scale_msa2, gate_msa2 = emb.chunk( 9, dim=1 @@ -162,7 +162,7 @@ def forward( class_labels: Optional[torch.LongTensor] = None, hidden_dtype: Optional[torch.dtype] = None, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: if self.emb is not None: emb = self.emb(timestep, class_labels, hidden_dtype=hidden_dtype) emb = self.linear(self.silu(emb)) @@ -196,7 +196,7 @@ def forward( self, x: torch.Tensor, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: emb = self.linear(self.silu(emb)) shift_msa, scale_msa, gate_msa = emb.chunk(3, dim=1) x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] @@ -225,7 +225,7 @@ def forward( self, x: torch.Tensor, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: emb = self.linear(self.silu(emb)) scale_msa, gate_msa, scale_mlp, gate_mlp = emb.chunk(4, dim=1) x = self.norm(x) * (1 + scale_msa[:, None]) @@ -257,10 +257,10 @@ def __init__(self, embedding_dim: int, use_additional_conditions: bool = False): def forward( self, timestep: torch.Tensor, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, batch_size: Optional[int] = None, hidden_dtype: Optional[torch.dtype] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: # No modulation happening here. added_cond_kwargs = added_cond_kwargs or {"resolution": None, "aspect_ratio": None} embedded_timestep = self.emb(timestep, **added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_dtype) @@ -423,7 +423,7 @@ def forward( x: torch.Tensor, context: torch.Tensor, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: emb = self.linear(self.silu(emb)) ( shift_msa, @@ -463,7 +463,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: shift, scale, gate, enc_shift, enc_scale, enc_gate = self.linear(self.silu(temb)).chunk(6, dim=1) hidden_states = self.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :] encoder_hidden_states = self.norm(encoder_hidden_states) * (1 + enc_scale)[:, None, :] + enc_shift[:, None, :] diff --git a/src/diffusers/models/resnet.py b/src/diffusers/models/resnet.py index c0b4ad40055a..e33353815fda 100644 --- a/src/diffusers/models/resnet.py +++ b/src/diffusers/models/resnet.py @@ -14,7 +14,7 @@ # limitations under the License. from functools import partial -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -401,7 +401,7 @@ def __init__( self, inp_channels: int, out_channels: int, - kernel_size: Union[int, Tuple[int, int]], + kernel_size: int | tuple[int, int], n_groups: int = 8, activation: str = "mish", ): @@ -438,7 +438,7 @@ def __init__( inp_channels: int, out_channels: int, embed_dim: int, - kernel_size: Union[int, Tuple[int, int]] = 5, + kernel_size: int | tuple[int, int] = 5, activation: str = "mish", ): super().__init__() diff --git a/src/diffusers/models/transformers/auraflow_transformer_2d.py b/src/diffusers/models/transformers/auraflow_transformer_2d.py index bf6d9e1b3803..7b1268d7e56d 100644 --- a/src/diffusers/models/transformers/auraflow_transformer_2d.py +++ b/src/diffusers/models/transformers/auraflow_transformer_2d.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -172,7 +172,7 @@ def forward( self, hidden_states: torch.FloatTensor, temb: torch.FloatTensor, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, ) -> torch.Tensor: residual = hidden_states attention_kwargs = attention_kwargs or {} @@ -241,8 +241,8 @@ def forward( hidden_states: torch.FloatTensor, encoder_hidden_states: torch.FloatTensor, temb: torch.FloatTensor, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: residual = hidden_states residual_context = encoder_hidden_states attention_kwargs = attention_kwargs or {} @@ -367,7 +367,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -376,7 +376,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -391,7 +391,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -462,9 +462,9 @@ def forward( hidden_states: torch.FloatTensor, encoder_hidden_states: torch.FloatTensor = None, timestep: torch.LongTensor = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/cogvideox_transformer_3d.py b/src/diffusers/models/transformers/cogvideox_transformer_3d.py index 9e0afdee6615..351b7e81b4fb 100644 --- a/src/diffusers/models/transformers/cogvideox_transformer_3d.py +++ b/src/diffusers/models/transformers/cogvideox_transformer_3d.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -120,9 +120,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_length = encoder_hidden_states.size(1) attention_kwargs = attention_kwargs or {} @@ -333,7 +333,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -342,7 +342,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -357,7 +357,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -427,13 +427,13 @@ def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, - timestep: Union[int, float, torch.LongTensor], + timestep: int | float | torch.LongTensor, timestep_cond: Optional[torch.Tensor] = None, - ofs: Optional[Union[int, float, torch.LongTensor]] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + ofs: Optional[int | float | torch.LongTensor] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/consisid_transformer_3d.py b/src/diffusers/models/transformers/consisid_transformer_3d.py index 91fe811f0013..3f9c260c31c6 100644 --- a/src/diffusers/models/transformers/consisid_transformer_3d.py +++ b/src/diffusers/models/transformers/consisid_transformer_3d.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -152,7 +152,7 @@ def __init__( nn.Linear(vit_dim, vit_dim * num_id_token), ) - def forward(self, id_embeds: torch.Tensor, vit_hidden_states: List[torch.Tensor]) -> torch.Tensor: + def forward(self, id_embeds: torch.Tensor, vit_hidden_states: list[torch.Tensor]) -> torch.Tensor: # Repeat latent queries for the batch size latents = self.latents.repeat(id_embeds.size(0), 1, 1) @@ -314,8 +314,8 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_length = encoder_hidden_states.size(1) # norm & modulate @@ -622,7 +622,7 @@ def _init_face_inputs(self): @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -631,7 +631,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -646,7 +646,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -684,14 +684,14 @@ def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, - timestep: Union[int, float, torch.LongTensor], + timestep: int | float | torch.LongTensor, timestep_cond: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, id_cond: Optional[torch.Tensor] = None, id_vit_hidden: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/dit_transformer_2d.py b/src/diffusers/models/transformers/dit_transformer_2d.py index 68f6f769436e..1d7589e7b9b8 100644 --- a/src/diffusers/models/transformers/dit_transformer_2d.py +++ b/src/diffusers/models/transformers/dit_transformer_2d.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional +from typing import Any, Optional import torch import torch.nn.functional as F @@ -150,7 +150,7 @@ def forward( hidden_states: torch.Tensor, timestep: Optional[torch.LongTensor] = None, class_labels: Optional[torch.LongTensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, return_dict: bool = True, ): """ @@ -164,7 +164,7 @@ def forward( class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in `AdaLayerZeroNorm`. - cross_attention_kwargs ( `Dict[str, Any]`, *optional*): + cross_attention_kwargs ( `dict[str, Any]`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). diff --git a/src/diffusers/models/transformers/hunyuan_transformer_2d.py b/src/diffusers/models/transformers/hunyuan_transformer_2d.py index fbe9fe8df91c..1511d418732f 100644 --- a/src/diffusers/models/transformers/hunyuan_transformer_2d.py +++ b/src/diffusers/models/transformers/hunyuan_transformer_2d.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Union +from typing import Optional import torch from torch import nn @@ -352,7 +352,7 @@ def unfuse_qkv_projections(self): @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -361,7 +361,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -376,7 +376,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. diff --git a/src/diffusers/models/transformers/lumina_nextdit2d.py b/src/diffusers/models/transformers/lumina_nextdit2d.py index bed5e69c2d36..d176646b7bb2 100644 --- a/src/diffusers/models/transformers/lumina_nextdit2d.py +++ b/src/diffusers/models/transformers/lumina_nextdit2d.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -123,7 +123,7 @@ def forward( encoder_hidden_states: torch.Tensor, encoder_mask: torch.Tensor, temb: torch.Tensor, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ) -> torch.Tensor: """ Perform a forward pass through the LuminaNextDiTBlock. @@ -135,7 +135,7 @@ def forward( encoder_hidden_states: (`torch.Tensor`): The hidden_states of text prompt are processed by Gemma encoder. encoder_mask (`torch.Tensor`): The hidden_states of text prompt attention mask. temb (`torch.Tensor`): Timestep embedding with text prompt embedding. - cross_attention_kwargs (`Dict[str, Any]`): kwargs for cross attention. + cross_attention_kwargs (`dict[str, Any]`): kwargs for cross attention. """ residual = hidden_states @@ -295,9 +295,9 @@ def forward( encoder_hidden_states: torch.Tensor, encoder_mask: torch.Tensor, image_rotary_emb: torch.Tensor, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, return_dict=True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: """ Forward pass of LuminaNextDiT. diff --git a/src/diffusers/models/transformers/pixart_transformer_2d.py b/src/diffusers/models/transformers/pixart_transformer_2d.py index 5a22144228ae..07f177a4e42f 100644 --- a/src/diffusers/models/transformers/pixart_transformer_2d.py +++ b/src/diffusers/models/transformers/pixart_transformer_2d.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Union +from typing import Any, Optional import torch from torch import nn @@ -186,7 +186,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -195,7 +195,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -210,7 +210,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -289,8 +289,8 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, timestep: Optional[torch.LongTensor] = None, - added_cond_kwargs: Dict[str, torch.Tensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, + added_cond_kwargs: dict[str, torch.Tensor] = None, + cross_attention_kwargs: dict[str, Any] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, @@ -306,8 +306,8 @@ def forward( self-attention. timestep (`torch.LongTensor`, *optional*): Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`. - added_cond_kwargs: (`Dict[str, Any]`, *optional*): Additional conditions to be used as inputs. - cross_attention_kwargs ( `Dict[str, Any]`, *optional*): + added_cond_kwargs: (`dict[str, Any]`, *optional*): Additional conditions to be used as inputs. + cross_attention_kwargs ( `dict[str, Any]`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). diff --git a/src/diffusers/models/transformers/prior_transformer.py b/src/diffusers/models/transformers/prior_transformer.py index 565da0da8b6e..890271acabc0 100644 --- a/src/diffusers/models/transformers/prior_transformer.py +++ b/src/diffusers/models/transformers/prior_transformer.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, Optional, Union +from typing import Optional import torch import torch.nn.functional as F @@ -168,7 +168,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -177,7 +177,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -192,7 +192,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -245,7 +245,7 @@ def set_default_attn_processor(self): def forward( self, hidden_states, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, proj_embedding: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.BoolTensor] = None, diff --git a/src/diffusers/models/transformers/sana_transformer.py b/src/diffusers/models/transformers/sana_transformer.py index 1e02ac32e86c..7872366019b4 100644 --- a/src/diffusers/models/transformers/sana_transformer.py +++ b/src/diffusers/models/transformers/sana_transformer.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn.functional as F @@ -416,7 +416,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -425,7 +425,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -440,7 +440,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -482,10 +482,10 @@ def forward( guidance: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_block_samples: Optional[Tuple[torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_block_samples: Optional[tuple[torch.Tensor]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor, ...], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor, ...] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/stable_audio_transformer.py b/src/diffusers/models/transformers/stable_audio_transformer.py index ac9b3fca419c..509f159abbeb 100644 --- a/src/diffusers/models/transformers/stable_audio_transformer.py +++ b/src/diffusers/models/transformers/stable_audio_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Dict, Optional, Union +from typing import Optional import numpy as np import torch @@ -276,7 +276,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -285,7 +285,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -300,7 +300,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -351,7 +351,7 @@ def forward( return_dict: bool = True, attention_mask: Optional[torch.LongTensor] = None, encoder_attention_mask: Optional[torch.LongTensor] = None, - ) -> Union[torch.FloatTensor, Transformer2DModelOutput]: + ) -> torch.FloatTensor | Transformer2DModelOutput: """ The [`StableAudioDiTModel`] forward method. diff --git a/src/diffusers/models/transformers/t5_film_transformer.py b/src/diffusers/models/transformers/t5_film_transformer.py index 7a9608735e32..e6eeaf14957c 100644 --- a/src/diffusers/models/transformers/t5_film_transformer.py +++ b/src/diffusers/models/transformers/t5_film_transformer.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import math -from typing import Optional, Tuple +from typing import Optional import torch from torch import nn @@ -201,7 +201,7 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, encoder_decoder_position_bias=None, - ) -> Tuple[torch.Tensor]: + ) -> tuple[torch.Tensor]: hidden_states = self.layer[0]( hidden_states, conditioning_emb=conditioning_emb, diff --git a/src/diffusers/models/transformers/transformer_2d.py b/src/diffusers/models/transformers/transformer_2d.py index 67fe9a33109b..1fbdb9a5135f 100644 --- a/src/diffusers/models/transformers/transformer_2d.py +++ b/src/diffusers/models/transformers/transformer_2d.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional +from typing import Any, Optional import torch import torch.nn.functional as F @@ -326,9 +326,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, timestep: Optional[torch.LongTensor] = None, - added_cond_kwargs: Dict[str, torch.Tensor] = None, + added_cond_kwargs: dict[str, torch.Tensor] = None, class_labels: Optional[torch.LongTensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, @@ -347,7 +347,7 @@ def forward( class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in `AdaLayerZeroNorm`. - cross_attention_kwargs ( `Dict[str, Any]`, *optional*): + cross_attention_kwargs ( `dict[str, Any]`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). diff --git a/src/diffusers/models/transformers/transformer_allegro.py b/src/diffusers/models/transformers/transformer_allegro.py index 5fa59a71d977..dab8ea95fbe9 100644 --- a/src/diffusers/models/transformers/transformer_allegro.py +++ b/src/diffusers/models/transformers/transformer_allegro.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Optional import torch import torch.nn as nn @@ -311,7 +311,7 @@ def forward( timestep: torch.LongTensor, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, return_dict: bool = True, ): batch_size, num_channels, num_frames, height, width = hidden_states.shape diff --git a/src/diffusers/models/transformers/transformer_bria.py b/src/diffusers/models/transformers/transformer_bria.py index d54679306e64..2757d0b5c4ab 100644 --- a/src/diffusers/models/transformers/transformer_bria.py +++ b/src/diffusers/models/transformers/transformer_bria.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -54,7 +54,7 @@ def _get_qkv_projections(attn: "BriaAttention", hidden_states, encoder_hidden_st def get_1d_rotary_pos_embed( dim: int, - pos: Union[np.ndarray, int], + pos: np.ndarray | int, theta: float = 10000.0, use_real=False, linear_factor=1.0, @@ -266,7 +266,7 @@ def forward( class BriaEmbedND(torch.nn.Module): # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 - def __init__(self, theta: int, axes_dim: List[int]): + def __init__(self, theta: int, axes_dim: list[int]): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -334,7 +334,7 @@ def forward(self, timestep, dtype): class BriaPosEmbed(torch.nn.Module): # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 - def __init__(self, theta: int, axes_dim: List[int]): + def __init__(self, theta: int, axes_dim: list[int]): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -395,9 +395,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( @@ -476,9 +476,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_len = encoder_hidden_states.shape[1] hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) @@ -537,7 +537,7 @@ def __init__( joint_attention_dim: int = 4096, pooled_projection_dim: int = None, guidance_embeds: bool = False, - axes_dims_rope: List[int] = [16, 56, 56], + axes_dims_rope: list[int] = [16, 56, 56], rope_theta=10000, time_theta=10000, ): @@ -590,11 +590,11 @@ def forward( img_ids: torch.Tensor = None, txt_ids: torch.Tensor = None, guidance: torch.Tensor = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, controlnet_block_samples=None, controlnet_single_block_samples=None, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: """ The [`BriaTransformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_chroma.py b/src/diffusers/models/transformers/transformer_chroma.py index 5823ae9d3da6..b3b0e467ba7c 100644 --- a/src/diffusers/models/transformers/transformer_chroma.py +++ b/src/diffusers/models/transformers/transformer_chroma.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -68,7 +68,7 @@ def forward( class_labels: Optional[torch.LongTensor] = None, hidden_dtype: Optional[torch.dtype] = None, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: if self.emb is not None: emb = self.emb(timestep, class_labels, hidden_dtype=hidden_dtype) shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.flatten(1, 2).chunk(6, dim=1) @@ -99,7 +99,7 @@ def forward( self, x: torch.Tensor, emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: shift_msa, scale_msa, gate_msa = emb.flatten(1, 2).chunk(3, dim=1) x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] return x, gate_msa @@ -243,9 +243,9 @@ def forward( self, hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, ) -> torch.Tensor: residual = hidden_states norm_hidden_states, gate = self.norm(hidden_states, emb=temb) @@ -309,10 +309,10 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: temb_img, temb_txt = temb[:, :6], temb[:, 6:] norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb_img) @@ -399,7 +399,7 @@ class ChromaTransformer2DModel( joint_attention_dim (`int`, defaults to `4096`): The number of dimensions to use for the joint attention (embedding/channel dimension of `encoder_hidden_states`). - axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. """ @@ -419,7 +419,7 @@ def __init__( attention_head_dim: int = 128, num_attention_heads: int = 24, joint_attention_dim: int = 4096, - axes_dims_rope: Tuple[int, ...] = (16, 56, 56), + axes_dims_rope: tuple[int, ...] = (16, 56, 56), approximator_num_channels: int = 64, approximator_hidden_dim: int = 5120, approximator_layers: int = 5, @@ -481,12 +481,12 @@ def forward( img_ids: torch.Tensor = None, txt_ids: torch.Tensor = None, attention_mask: torch.Tensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, controlnet_block_samples=None, controlnet_single_block_samples=None, return_dict: bool = True, controlnet_blocks_repeat: bool = False, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + ) -> torch.Tensor | Transformer2DModelOutput: """ The [`FluxTransformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_cogview3plus.py b/src/diffusers/models/transformers/transformer_cogview3plus.py index 7356f4a606bb..15337bda64df 100644 --- a/src/diffusers/models/transformers/transformer_cogview3plus.py +++ b/src/diffusers/models/transformers/transformer_cogview3plus.py @@ -13,8 +13,6 @@ # limitations under the License. -from typing import Dict, Tuple, Union - import torch import torch.nn as nn @@ -79,7 +77,7 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, emb: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_length = encoder_hidden_states.size(1) # norm & modulate @@ -226,7 +224,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -235,7 +233,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -250,7 +248,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -293,7 +291,7 @@ def forward( target_size: torch.Tensor, crop_coords: torch.Tensor, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: """ The [`CogView3PlusTransformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_cogview4.py b/src/diffusers/models/transformers/transformer_cogview4.py index 64e9a538a7c2..62f7068453f6 100644 --- a/src/diffusers/models/transformers/transformer_cogview4.py +++ b/src/diffusers/models/transformers/transformer_cogview4.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -73,7 +73,7 @@ def __init__(self, embedding_dim: int, dim: int) -> None: def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: dtype = hidden_states.dtype norm_hidden_states = self.norm(hidden_states).to(dtype=dtype) norm_encoder_hidden_states = self.norm_context(encoder_hidden_states).to(dtype=dtype) @@ -130,8 +130,8 @@ def __call__( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: dtype = encoder_hidden_states.dtype batch_size, text_seq_length, embed_dim = encoder_hidden_states.shape @@ -214,11 +214,9 @@ def __call__( latent_attn_mask: Optional[torch.Tensor] = None, text_attn_mask: Optional[torch.Tensor] = None, batch_flag: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[ - Union[Tuple[torch.Tensor, torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]] - ] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor] | list[tuple[torch.Tensor, torch.Tensor]]] = None, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: """ Args: attn (`Attention`): @@ -238,10 +236,10 @@ def __call__( Values from 0 to n-1 indicating which samples belong to the same batch. Samples with the same batch_flag are packed together. Example: [0, 1, 1, 2, 2] means sample 0 forms batch0, samples 1-2 form batch1, and samples 3-4 form batch2. If None, no packing is used. - image_rotary_emb (`Tuple[torch.Tensor, torch.Tensor]` or `list[Tuple[torch.Tensor, torch.Tensor]]`, *optional*): + image_rotary_emb (`tuple[torch.Tensor, torch.Tensor]` or `list[tuple[torch.Tensor, torch.Tensor]]`, *optional*): The rotary embedding for the image part of the input. Returns: - `Tuple[torch.Tensor, torch.Tensor]`: The processed hidden states for both image and text streams. + `tuple[torch.Tensor, torch.Tensor]`: The processed hidden states for both image and text streams. """ # Get dimensions and device info @@ -489,12 +487,10 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[ - Union[Tuple[torch.Tensor, torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]] - ] = None, - attention_mask: Optional[Dict[str, torch.Tensor]] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor] | list[tuple[torch.Tensor, torch.Tensor]]] = None, + attention_mask: Optional[dict[str, torch.Tensor]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: # 1. Timestep conditioning ( norm_hidden_states, @@ -537,7 +533,7 @@ def forward( class CogView4RotaryPosEmbed(nn.Module): - def __init__(self, dim: int, patch_size: int, rope_axes_dim: Tuple[int, int], theta: float = 10000.0) -> None: + def __init__(self, dim: int, patch_size: int, rope_axes_dim: tuple[int, int], theta: float = 10000.0) -> None: super().__init__() self.dim = dim @@ -545,7 +541,7 @@ def __init__(self, dim: int, patch_size: int, rope_axes_dim: Tuple[int, int], th self.rope_axes_dim = rope_axes_dim self.theta = theta - def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: batch_size, num_channels, height, width = hidden_states.shape height, width = height // self.patch_size, width // self.patch_size @@ -666,7 +662,7 @@ def __init__( condition_dim: int = 256, pos_embed_max_size: int = 128, sample_size: int = 128, - rope_axes_dim: Tuple[int, int] = (256, 256), + rope_axes_dim: tuple[int, int] = (256, 256), ): super().__init__() @@ -711,13 +707,11 @@ def forward( original_size: torch.Tensor, target_size: torch.Tensor, crop_coords: torch.Tensor, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, attention_mask: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[ - Union[Tuple[torch.Tensor, torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]] - ] = None, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor] | list[tuple[torch.Tensor, torch.Tensor]]] = None, + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/transformer_cosmos.py b/src/diffusers/models/transformers/transformer_cosmos.py index 373b470ae37b..4ce58856d651 100644 --- a/src/diffusers/models/transformers/transformer_cosmos.py +++ b/src/diffusers/models/transformers/transformer_cosmos.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Optional import numpy as np import torch @@ -36,7 +36,7 @@ class CosmosPatchEmbed(nn.Module): def __init__( - self, in_channels: int, out_channels: int, patch_size: Tuple[int, int, int], bias: bool = True + self, in_channels: int, out_channels: int, patch_size: tuple[int, int, int], bias: bool = True ) -> None: super().__init__() self.patch_size = patch_size @@ -291,10 +291,10 @@ class CosmosRotaryPosEmbed(nn.Module): def __init__( self, hidden_size: int, - max_size: Tuple[int, int, int] = (128, 240, 240), - patch_size: Tuple[int, int, int] = (1, 2, 2), + max_size: tuple[int, int, int] = (128, 240, 240), + patch_size: tuple[int, int, int] = (1, 2, 2), base_fps: int = 24, - rope_scale: Tuple[float, float, float] = (2.0, 1.0, 1.0), + rope_scale: tuple[float, float, float] = (2.0, 1.0, 1.0), ) -> None: super().__init__() @@ -310,7 +310,7 @@ def __init__( self.w_ntk_factor = rope_scale[2] ** (self.dim_w / (self.dim_w - 2)) self.t_ntk_factor = rope_scale[0] ** (self.dim_t / (self.dim_t - 2)) - def forward(self, hidden_states: torch.Tensor, fps: Optional[int] = None) -> Tuple[torch.Tensor, torch.Tensor]: + def forward(self, hidden_states: torch.Tensor, fps: Optional[int] = None) -> tuple[torch.Tensor, torch.Tensor]: batch_size, num_channels, num_frames, height, width = hidden_states.shape pe_size = [num_frames // self.patch_size[0], height // self.patch_size[1], width // self.patch_size[2]] device = hidden_states.device @@ -355,8 +355,8 @@ class CosmosLearnablePositionalEmbed(nn.Module): def __init__( self, hidden_size: int, - max_size: Tuple[int, int, int], - patch_size: Tuple[int, int, int], + max_size: tuple[int, int, int], + patch_size: tuple[int, int, int], eps: float = 1e-6, ) -> None: super().__init__() @@ -405,12 +405,12 @@ class CosmosTransformer3DModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): Input dimension of text embeddings from the text encoder. adaln_lora_dim (`int`, defaults to `256`): The hidden dimension of the Adaptive LayerNorm LoRA layer. - max_size (`Tuple[int, int, int]`, defaults to `(128, 240, 240)`): + max_size (`tuple[int, int, int]`, defaults to `(128, 240, 240)`): The maximum size of the input latent tensors in the temporal, height, and width dimensions. - patch_size (`Tuple[int, int, int]`, defaults to `(1, 2, 2)`): + patch_size (`tuple[int, int, int]`, defaults to `(1, 2, 2)`): The patch size to use for patchifying the input latent tensors in the temporal, height, and width dimensions. - rope_scale (`Tuple[float, float, float]`, defaults to `(2.0, 1.0, 1.0)`): + rope_scale (`tuple[float, float, float]`, defaults to `(2.0, 1.0, 1.0)`): The scaling factor to use for RoPE in the temporal, height, and width dimensions. concat_padding_mask (`bool`, defaults to `True`): Whether to concatenate the padding mask to the input latent tensors. @@ -434,9 +434,9 @@ def __init__( mlp_ratio: float = 4.0, text_embed_dim: int = 1024, adaln_lora_dim: int = 256, - max_size: Tuple[int, int, int] = (128, 240, 240), - patch_size: Tuple[int, int, int] = (1, 2, 2), - rope_scale: Tuple[float, float, float] = (2.0, 1.0, 1.0), + max_size: tuple[int, int, int] = (128, 240, 240), + patch_size: tuple[int, int, int] = (1, 2, 2), + rope_scale: tuple[float, float, float] = (2.0, 1.0, 1.0), concat_padding_mask: bool = True, extra_pos_embed_type: Optional[str] = "learnable", ) -> None: diff --git a/src/diffusers/models/transformers/transformer_easyanimate.py b/src/diffusers/models/transformers/transformer_easyanimate.py index 545fa29730db..8f9d9dd41a11 100755 --- a/src/diffusers/models/transformers/transformer_easyanimate.py +++ b/src/diffusers/models/transformers/transformer_easyanimate.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch import torch.nn.functional as F @@ -58,7 +58,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: shift, scale, gate, enc_shift, enc_scale, enc_gate = self.linear(self.silu(temb)).chunk(6, dim=1) hidden_states = self.norm(hidden_states) * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) encoder_hidden_states = self.norm(encoder_hidden_states) * (1 + enc_scale.unsqueeze(1)) + enc_shift.unsqueeze( @@ -68,7 +68,7 @@ def forward( class EasyAnimateRotaryPosEmbed(nn.Module): - def __init__(self, patch_size: int, rope_dim: List[int]) -> None: + def __init__(self, patch_size: int, rope_dim: list[int]) -> None: super().__init__() self.patch_size = patch_size @@ -280,8 +280,8 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: # 1. Attention norm_hidden_states, norm_encoder_hidden_states, gate_msa, enc_gate_msa = self.norm1( hidden_states, encoder_hidden_states, temb @@ -470,7 +470,7 @@ def forward( inpaint_latents: Optional[torch.Tensor] = None, control_latents: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: batch_size, channels, video_length, height, width = hidden_states.size() p = self.config.patch_size post_patch_height = height // p diff --git a/src/diffusers/models/transformers/transformer_flux.py b/src/diffusers/models/transformers/transformer_flux.py index 1a4464432425..f87e7315ec43 100644 --- a/src/diffusers/models/transformers/transformer_flux.py +++ b/src/diffusers/models/transformers/transformer_flux.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -187,7 +187,7 @@ def __call__( encoder_hidden_states: torch.Tensor = None, attention_mask: Optional[torch.Tensor] = None, image_rotary_emb: Optional[torch.Tensor] = None, - ip_hidden_states: Optional[List[torch.Tensor]] = None, + ip_hidden_states: Optional[list[torch.Tensor]] = None, ip_adapter_masks: Optional[torch.Tensor] = None, ) -> torch.Tensor: batch_size = hidden_states.shape[0] @@ -379,9 +379,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_len = encoder_hidden_states.shape[1] hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) @@ -439,9 +439,9 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( @@ -493,7 +493,7 @@ def forward( class FluxPosEmbed(nn.Module): # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 - def __init__(self, theta: int, axes_dim: List[int]): + def __init__(self, theta: int, axes_dim: list[int]): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -558,7 +558,7 @@ class FluxTransformer2DModel( The number of dimensions to use for the pooled projection. guidance_embeds (`bool`, defaults to `False`): Whether to use guidance embeddings for guidance-distilled variant of the model. - axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. """ @@ -589,7 +589,7 @@ def __init__( joint_attention_dim: int = 4096, pooled_projection_dim: int = 768, guidance_embeds: bool = False, - axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + axes_dims_rope: tuple[int, int, int] = (16, 56, 56), ): super().__init__() self.out_channels = out_channels or in_channels @@ -643,12 +643,12 @@ def forward( img_ids: torch.Tensor = None, txt_ids: torch.Tensor = None, guidance: torch.Tensor = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, controlnet_block_samples=None, controlnet_single_block_samples=None, return_dict: bool = True, controlnet_blocks_repeat: bool = False, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + ) -> torch.Tensor | Transformer2DModelOutput: """ The [`FluxTransformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_hidream_image.py b/src/diffusers/models/transformers/transformer_hidream_image.py index 4a5aee29abc4..d428b88dde51 100644 --- a/src/diffusers/models/transformers/transformer_hidream_image.py +++ b/src/diffusers/models/transformers/transformer_hidream_image.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -114,7 +114,7 @@ def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor: class HiDreamImageEmbedND(nn.Module): - def __init__(self, theta: int, axes_dim: List[int]): + def __init__(self, theta: int, axes_dim: list[int]): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -128,7 +128,7 @@ def forward(self, ids: torch.Tensor) -> torch.Tensor: return emb.unsqueeze(2) -def apply_rope(xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: +def apply_rope(xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2) xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2) xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] @@ -534,7 +534,7 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, temb: Optional[torch.Tensor] = None, image_rotary_emb: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: wtype = hidden_states.dtype ( shift_msa_i, @@ -581,7 +581,7 @@ def forward( class HiDreamBlock(nn.Module): - def __init__(self, block: Union[HiDreamImageTransformerBlock, HiDreamImageSingleTransformerBlock]): + def __init__(self, block: HiDreamImageTransformerBlock | HiDreamImageSingleTransformerBlock): super().__init__() self.block = block @@ -592,7 +592,7 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, temb: Optional[torch.Tensor] = None, image_rotary_emb: torch.Tensor = None, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: return self.block( hidden_states=hidden_states, hidden_states_masks=hidden_states_masks, @@ -616,13 +616,13 @@ def __init__( num_single_layers: int = 32, attention_head_dim: int = 128, num_attention_heads: int = 20, - caption_channels: List[int] = None, + caption_channels: list[int] = None, text_emb_dim: int = 2048, num_routed_experts: int = 4, num_activated_experts: int = 2, - axes_dims_rope: Tuple[int, int] = (32, 32), - max_resolution: Tuple[int, int] = (128, 128), - llama_layers: List[int] = None, + axes_dims_rope: tuple[int, int] = (32, 32), + max_resolution: tuple[int, int] = (128, 128), + llama_layers: list[int] = None, force_inference_output: bool = False, ): super().__init__() @@ -681,7 +681,7 @@ def __init__( self.gradient_checkpointing = False - def unpatchify(self, x: torch.Tensor, img_sizes: List[Tuple[int, int]], is_training: bool) -> List[torch.Tensor]: + def unpatchify(self, x: torch.Tensor, img_sizes: list[tuple[int, int]], is_training: bool) -> list[torch.Tensor]: if is_training and not self.config.force_inference_output: B, S, F = x.shape C = F // (self.config.patch_size * self.config.patch_size) @@ -781,12 +781,12 @@ def forward( encoder_hidden_states_llama3: torch.Tensor = None, pooled_embeds: torch.Tensor = None, img_ids: Optional[torch.Tensor] = None, - img_sizes: Optional[List[Tuple[int, int]]] = None, + img_sizes: Optional[list[tuple[int, int]]] = None, hidden_states_masks: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, **kwargs, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: encoder_hidden_states = kwargs.get("encoder_hidden_states", None) if encoder_hidden_states is not None: diff --git a/src/diffusers/models/transformers/transformer_hunyuan_video.py b/src/diffusers/models/transformers/transformer_hunyuan_video.py index bc857ccab463..c07968a360db 100644 --- a/src/diffusers/models/transformers/transformer_hunyuan_video.py +++ b/src/diffusers/models/transformers/transformer_hunyuan_video.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -143,7 +143,7 @@ def __call__( class HunyuanVideoPatchEmbed(nn.Module): def __init__( self, - patch_size: Union[int, Tuple[int, int, int]] = 16, + patch_size: int | tuple[int, int, int] = 16, in_chans: int = 3, embed_dim: int = 768, ) -> None: @@ -168,7 +168,7 @@ def __init__(self, in_features: int, out_features: Optional[int] = None) -> None def forward( self, temb: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: temb = self.linear(self.nonlinearity(temb)) gate_msa, gate_mlp = temb.chunk(2, dim=1) gate_msa, gate_mlp = gate_msa.unsqueeze(1), gate_mlp.unsqueeze(1) @@ -197,7 +197,7 @@ def forward( emb: torch.Tensor, token_replace_emb: torch.Tensor, first_frame_num_tokens: int, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: emb = self.linear(self.silu(emb)) token_replace_emb = self.linear(self.silu(token_replace_emb)) @@ -248,7 +248,7 @@ def forward( emb: torch.Tensor, token_replace_emb: torch.Tensor, first_frame_num_tokens: int, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: emb = self.linear(self.silu(emb)) token_replace_emb = self.linear(self.silu(token_replace_emb)) @@ -289,7 +289,7 @@ def __init__( def forward( self, timestep: torch.Tensor, pooled_projection: torch.Tensor, guidance: Optional[torch.Tensor] = None - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: timesteps_proj = self.time_proj(timestep) timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=pooled_projection.dtype)) # (N, D) pooled_projections = self.text_embedder(pooled_projection) @@ -457,7 +457,7 @@ def forward( class HunyuanVideoRotaryPosEmbed(nn.Module): - def __init__(self, patch_size: int, patch_size_t: int, rope_dim: List[int], theta: float = 256.0) -> None: + def __init__(self, patch_size: int, patch_size_t: int, rope_dim: list[int], theta: float = 256.0) -> None: super().__init__() self.patch_size = patch_size @@ -526,10 +526,10 @@ def forward( encoder_hidden_states: torch.Tensor, temb: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, *args, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_length = encoder_hidden_states.shape[1] hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) @@ -606,10 +606,10 @@ def forward( encoder_hidden_states: torch.Tensor, temb: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + freqs_cis: Optional[tuple[torch.Tensor, torch.Tensor]] = None, *args, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: # 1. Input normalization norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( @@ -681,10 +681,10 @@ def forward( encoder_hidden_states: torch.Tensor, temb: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, token_replace_emb: torch.Tensor = None, num_tokens: int = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: text_seq_length = encoder_hidden_states.shape[1] hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) @@ -765,10 +765,10 @@ def forward( encoder_hidden_states: torch.Tensor, temb: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + freqs_cis: Optional[tuple[torch.Tensor, torch.Tensor]] = None, token_replace_emb: torch.Tensor = None, num_tokens: int = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: # 1. Input normalization ( norm_hidden_states, @@ -854,7 +854,7 @@ class HunyuanVideoTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, The dimension of the pooled projection of the text embeddings. rope_theta (`float`, defaults to `256.0`): The value of theta to use in the RoPE layer. - rope_axes_dim (`Tuple[int]`, defaults to `(16, 56, 56)`): + rope_axes_dim (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions of the axes to use in the RoPE layer. image_condition_type (`str`, *optional*, defaults to `None`): The type of image conditioning to use. If `None`, no image conditioning is used. If `latent_concat`, the @@ -895,7 +895,7 @@ def __init__( text_embed_dim: int = 4096, pooled_projection_dim: int = 768, rope_theta: float = 256.0, - rope_axes_dim: Tuple[int] = (16, 56, 56), + rope_axes_dim: tuple[int] = (16, 56, 56), image_condition_type: Optional[str] = None, ) -> None: super().__init__() @@ -970,7 +970,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -979,7 +979,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -994,7 +994,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -1036,9 +1036,9 @@ def forward( encoder_attention_mask: torch.Tensor, pooled_projections: torch.Tensor, guidance: torch.Tensor = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/transformer_hunyuan_video_framepack.py b/src/diffusers/models/transformers/transformer_hunyuan_video_framepack.py index 60b40fff3cb8..19d9bce8fcd8 100644 --- a/src/diffusers/models/transformers/transformer_hunyuan_video_framepack.py +++ b/src/diffusers/models/transformers/transformer_hunyuan_video_framepack.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -39,7 +39,7 @@ class HunyuanVideoFramepackRotaryPosEmbed(nn.Module): - def __init__(self, patch_size: int, patch_size_t: int, rope_dim: List[int], theta: float = 256.0) -> None: + def __init__(self, patch_size: int, patch_size_t: int, rope_dim: list[int], theta: float = 256.0) -> None: super().__init__() self.patch_size = patch_size @@ -139,7 +139,7 @@ def __init__( text_embed_dim: int = 4096, pooled_projection_dim: int = 768, rope_theta: float = 256.0, - rope_axes_dim: Tuple[int] = (16, 56, 56), + rope_axes_dim: tuple[int] = (16, 56, 56), image_condition_type: Optional[str] = None, has_image_proj: int = False, image_proj_dim: int = 1152, @@ -214,9 +214,9 @@ def forward( indices_latents_history_2x: Optional[torch.Tensor] = None, latents_history_4x: Optional[torch.Tensor] = None, indices_latents_history_4x: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor] | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) @@ -351,10 +351,10 @@ def _pack_history_states( latents_clean: Optional[torch.Tensor] = None, latents_history_2x: Optional[torch.Tensor] = None, latents_history_4x: Optional[torch.Tensor] = None, - image_rotary_emb: Tuple[torch.Tensor, torch.Tensor] = None, - image_rotary_emb_clean: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - image_rotary_emb_history_2x: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - image_rotary_emb_history_4x: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor] = None, + image_rotary_emb_clean: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb_history_2x: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb_history_4x: Optional[tuple[torch.Tensor, torch.Tensor]] = None, height: int = None, width: int = None, ): @@ -381,10 +381,10 @@ def _pack_history_states( def _pad_rotary_emb( self, - image_rotary_emb: Tuple[torch.Tensor], + image_rotary_emb: tuple[torch.Tensor], height: int, width: int, - kernel_size: Tuple[int, int, int], + kernel_size: tuple[int, int, int], ): # freqs_cos, freqs_sin have shape [W * H * T, D / 2], where D is attention head dim freqs_cos, freqs_sin = image_rotary_emb diff --git a/src/diffusers/models/transformers/transformer_kandinsky.py b/src/diffusers/models/transformers/transformer_kandinsky.py index d4ba92acaf6e..def3688a0443 100644 --- a/src/diffusers/models/transformers/transformer_kandinsky.py +++ b/src/diffusers/models/transformers/transformer_kandinsky.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -368,7 +368,7 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, sparse_params: Optional[torch.Tensor] = None, - rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, **kwargs, ) -> torch.Tensor: attn_parameters = set(inspect.signature(self.processor.__call__).parameters.keys()) @@ -595,12 +595,12 @@ def forward( encoder_hidden_states: torch.Tensor, # text_embed timestep: torch.Tensor, # time pooled_projections: torch.Tensor, # pooled_text_embed - visual_rope_pos: Tuple[int, int, int], + visual_rope_pos: tuple[int, int, int], text_rope_pos: torch.LongTensor, - scale_factor: Tuple[float, float, float] = (1.0, 1.0, 1.0), - sparse_params: Optional[Dict[str, Any]] = None, + scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0), + sparse_params: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Transformer2DModelOutput, torch.FloatTensor]: + ) -> Transformer2DModelOutput | torch.FloatTensor: """ Forward pass of the Kandinsky5 3D Transformer. @@ -609,10 +609,10 @@ def forward( encoder_hidden_states (`torch.FloatTensor`): Text embeddings timestep (`torch.Tensor` or `float` or `int`): Current timestep pooled_projections (`torch.FloatTensor`): Pooled text embeddings - visual_rope_pos (`Tuple[int, int, int]`): Position for visual RoPE + visual_rope_pos (`tuple[int, int, int]`): Position for visual RoPE text_rope_pos (`torch.LongTensor`): Position for text RoPE - scale_factor (`Tuple[float, float, float]`, optional): Scale factor for RoPE - sparse_params (`Dict[str, Any]`, optional): Parameters for sparse attention + scale_factor (`tuple[float, float, float]`, optional): Scale factor for RoPE + sparse_params (`dict[str, Any]`, optional): Parameters for sparse attention return_dict (`bool`, optional): Whether to return a dictionary Returns: diff --git a/src/diffusers/models/transformers/transformer_ltx.py b/src/diffusers/models/transformers/transformer_ltx.py index 685c73c07c75..e668d5008dfb 100644 --- a/src/diffusers/models/transformers/transformer_ltx.py +++ b/src/diffusers/models/transformers/transformer_ltx.py @@ -15,7 +15,7 @@ import inspect import math -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -203,7 +203,7 @@ def _prepare_video_coords( num_frames: int, height: int, width: int, - rope_interpolation_scale: Tuple[torch.Tensor, float, float], + rope_interpolation_scale: tuple[torch.Tensor, float, float], device: torch.device, ) -> torch.Tensor: # Always compute rope in fp32 @@ -229,9 +229,9 @@ def forward( num_frames: Optional[int] = None, height: Optional[int] = None, width: Optional[int] = None, - rope_interpolation_scale: Optional[Tuple[torch.Tensor, float, float]] = None, + rope_interpolation_scale: Optional[tuple[torch.Tensor, float, float]] = None, video_coords: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: batch_size = hidden_states.size(0) if video_coords is None: @@ -346,7 +346,7 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: batch_size = hidden_states.size(0) @@ -500,9 +500,9 @@ def forward( num_frames: Optional[int] = None, height: Optional[int] = None, width: Optional[int] = None, - rope_interpolation_scale: Optional[Union[Tuple[float, float, float], torch.Tensor]] = None, + rope_interpolation_scale: Optional[tuple[float, float, float] | torch.Tensor] = None, video_coords: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, ) -> torch.Tensor: if attention_kwargs is not None: diff --git a/src/diffusers/models/transformers/transformer_lumina2.py b/src/diffusers/models/transformers/transformer_lumina2.py index 77121edb9fc9..f69bd5623390 100644 --- a/src/diffusers/models/transformers/transformer_lumina2.py +++ b/src/diffusers/models/transformers/transformer_lumina2.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -58,7 +58,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, timestep: torch.Tensor, encoder_hidden_states: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: timestep_proj = self.time_proj(timestep).type_as(hidden_states) time_embed = self.timestep_embedder(timestep_proj) caption_embed = self.caption_embedder(encoder_hidden_states) @@ -231,7 +231,7 @@ def forward( class Lumina2RotaryPosEmbed(nn.Module): - def __init__(self, theta: int, axes_dim: List[int], axes_lens: List[int] = (300, 512, 512), patch_size: int = 2): + def __init__(self, theta: int, axes_dim: list[int], axes_lens: list[int] = (300, 512, 512), patch_size: int = 2): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -240,7 +240,7 @@ def __init__(self, theta: int, axes_dim: List[int], axes_lens: List[int] = (300, self.freqs_cis = self._precompute_freqs_cis(axes_dim, axes_lens, theta) - def _precompute_freqs_cis(self, axes_dim: List[int], axes_lens: List[int], theta: int) -> List[torch.Tensor]: + def _precompute_freqs_cis(self, axes_dim: list[int], axes_lens: list[int], theta: int) -> list[torch.Tensor]: freqs_cis = [] freqs_dtype = torch.float32 if torch.backends.mps.is_available() else torch.float64 for i, (d, e) in enumerate(zip(axes_dim, axes_lens)): @@ -378,8 +378,8 @@ def __init__( ffn_dim_multiplier: Optional[float] = None, norm_eps: float = 1e-5, scaling_factor: float = 1.0, - axes_dim_rope: Tuple[int, int, int] = (32, 32, 32), - axes_lens: Tuple[int, int, int] = (300, 512, 512), + axes_dim_rope: tuple[int, int, int] = (32, 32, 32), + axes_lens: tuple[int, int, int] = (300, 512, 512), cap_feat_dim: int = 1024, ) -> None: super().__init__() @@ -461,9 +461,9 @@ def forward( timestep: torch.Tensor, encoder_hidden_states: torch.Tensor, encoder_attention_mask: torch.Tensor, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + ) -> torch.Tensor | Transformer2DModelOutput: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/transformer_mochi.py b/src/diffusers/models/transformers/transformer_mochi.py index 63911fe7c10d..9389952fdc72 100644 --- a/src/diffusers/models/transformers/transformer_mochi.py +++ b/src/diffusers/models/transformers/transformer_mochi.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional import torch import torch.nn as nn @@ -104,7 +104,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, emb: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: hidden_states_dtype = hidden_states.dtype emb = self.linear(self.silu(emb)) @@ -206,7 +206,7 @@ def forward( temb: torch.Tensor, encoder_attention_mask: torch.Tensor, image_rotary_emb: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: norm_hidden_states, gate_msa, scale_mlp, gate_mlp = self.norm1(hidden_states, temb) if not self.context_pre_only: @@ -299,7 +299,7 @@ def forward( width: int, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: pos = self._get_positions(num_frames, height, width, device, dtype) rope_cos, rope_sin = self._create_rope(pos_frequencies, pos) return rope_cos, rope_sin @@ -410,7 +410,7 @@ def forward( encoder_hidden_states: torch.Tensor, timestep: torch.LongTensor, encoder_attention_mask: torch.Tensor, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, ) -> torch.Tensor: if attention_kwargs is not None: diff --git a/src/diffusers/models/transformers/transformer_omnigen.py b/src/diffusers/models/transformers/transformer_omnigen.py index 6939cac0a3a7..9954be6283f3 100644 --- a/src/diffusers/models/transformers/transformer_omnigen.py +++ b/src/diffusers/models/transformers/transformer_omnigen.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Optional import torch import torch.nn as nn @@ -387,7 +387,7 @@ def __init__( self.gradient_checkpointing = False def _get_multimodal_embeddings( - self, input_ids: torch.Tensor, input_img_latents: List[torch.Tensor], input_image_sizes: Dict + self, input_ids: torch.Tensor, input_img_latents: list[torch.Tensor], input_image_sizes: Dict ) -> Optional[torch.Tensor]: if input_ids is None: return None @@ -408,14 +408,14 @@ def _get_multimodal_embeddings( def forward( self, hidden_states: torch.Tensor, - timestep: Union[int, float, torch.FloatTensor], + timestep: int | float | torch.FloatTensor, input_ids: torch.Tensor, - input_img_latents: List[torch.Tensor], - input_image_sizes: Dict[int, List[int]], + input_img_latents: list[torch.Tensor], + input_image_sizes: dict[int, list[int]], attention_mask: torch.Tensor, position_ids: torch.Tensor, return_dict: bool = True, - ) -> Union[Transformer2DModelOutput, Tuple[torch.Tensor]]: + ) -> Transformer2DModelOutput | tuple[torch.Tensor]: batch_size, num_channels, height, width = hidden_states.shape p = self.config.patch_size post_patch_height, post_patch_width = height // p, width // p diff --git a/src/diffusers/models/transformers/transformer_prx.py b/src/diffusers/models/transformers/transformer_prx.py index 9b2664b9cb26..507a7656c171 100644 --- a/src/diffusers/models/transformers/transformer_prx.py +++ b/src/diffusers/models/transformers/transformer_prx.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from torch import nn @@ -264,10 +264,10 @@ class PRXEmbedND(nn.Module): theta (int): Scaling factor that controls the frequency spectrum of the rotary embeddings. axes_dim (list[int]): - List of embedding dimensions for each axis (each must be even). + list of embedding dimensions for each axis (each must be even). """ - def __init__(self, dim: int, theta: int, axes_dim: List[int]): + def __init__(self, dim: int, theta: int, axes_dim: list[int]): super().__init__() self.dim = dim self.theta = theta @@ -342,7 +342,7 @@ def __init__(self, dim: int): def forward( self, vec: torch.Tensor - ) -> Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: + ) -> tuple[tuple[torch.Tensor, torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: out = self.lin(nn.functional.silu(vec))[:, None, :].chunk(6, dim=-1) return tuple(out[:3]), tuple(out[3:]) @@ -427,7 +427,7 @@ def forward( temb: torch.Tensor, image_rotary_emb: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - **kwargs: Dict[str, Any], + **kwargs: dict[str, Any], ) -> torch.Tensor: r""" Runs modulation-gated cross-attention and MLP, with residual connections. @@ -577,7 +577,7 @@ class PRXTransformer2DModel(ModelMixin, ConfigMixin, AttentionMixin): depth (`int`, *optional*, defaults to 16): Number of transformer blocks. axes_dim (`list[int]`, *optional*): - List of dimensions for each positional embedding axis. Defaults to `[32, 32]`. + list of dimensions for each positional embedding axis. Defaults to `[32, 32]`. theta (`int`, *optional*, defaults to 10000): Frequency scaling factor for rotary embeddings. time_factor (`float`, *optional*, defaults to 1000.0): @@ -698,9 +698,9 @@ def forward( timestep: torch.Tensor, encoder_hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[Tuple[torch.Tensor, ...], Transformer2DModelOutput]: + ) -> tuple[torch.Tensor, ...] | Transformer2DModelOutput: r""" Forward pass of the PRXTransformer2DModel. diff --git a/src/diffusers/models/transformers/transformer_qwenimage.py b/src/diffusers/models/transformers/transformer_qwenimage.py index c0fa031b9faf..eca1dcf1b9e9 100644 --- a/src/diffusers/models/transformers/transformer_qwenimage.py +++ b/src/diffusers/models/transformers/transformer_qwenimage.py @@ -14,7 +14,7 @@ import functools import math -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -95,10 +95,10 @@ def get_timestep_embedding( def apply_rotary_emb_qwen( x: torch.Tensor, - freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + freqs_cis: torch.Tensor | tuple[torch.Tensor], use_real: bool = True, use_real_unbind_dim: int = -1, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> tuple[torch.Tensor, torch.Tensor]: """ Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are @@ -108,10 +108,10 @@ def apply_rotary_emb_qwen( Args: x (`torch.Tensor`): Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply - freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) Returns: - Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings. """ if use_real: cos, sin = freqs_cis # [S, D] @@ -158,7 +158,7 @@ def forward(self, timestep, hidden_states): class QwenEmbedRope(nn.Module): - def __init__(self, theta: int, axes_dim: List[int], scale_rope=False): + def __init__(self, theta: int, axes_dim: list[int], scale_rope=False): super().__init__() self.theta = theta self.axes_dim = axes_dim @@ -196,15 +196,15 @@ def rope_params(self, index, dim, theta=10000): def forward( self, - video_fhw: Union[Tuple[int, int, int], List[Tuple[int, int, int]]], - txt_seq_lens: List[int], + video_fhw: tuple[int, int, int] | list[tuple[int, int, int]], + txt_seq_lens: list[int], device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: """ Args: - video_fhw (`Tuple[int, int, int]` or `List[Tuple[int, int, int]]`): + video_fhw (`tuple[int, int, int]` or `list[tuple[int, int, int]]`): A list of 3 integers [frame, height, width] representing the shape of the video. - txt_seq_lens (`List[int]`): + txt_seq_lens (`list[int]`): A list of integers of length batch_size representing the length of each text prompt. device: (`torch.device`): The device on which to perform the RoPE computation. @@ -414,9 +414,9 @@ def forward( encoder_hidden_states: torch.Tensor, encoder_hidden_states_mask: torch.Tensor, temb: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: # Get modulation parameters for both streams img_mod_params = self.img_mod(temb) # [B, 6*dim] txt_mod_params = self.txt_mod(temb) # [B, 6*dim] @@ -500,7 +500,7 @@ class QwenImageTransformer2DModel( `encoder_hidden_states`). guidance_embeds (`bool`, defaults to `False`): Whether to use guidance embeddings for guidance-distilled variant of the model. - axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. """ @@ -532,7 +532,7 @@ def __init__( num_attention_heads: int = 24, joint_attention_dim: int = 3584, guidance_embeds: bool = False, # TODO: this should probably be removed - axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + axes_dims_rope: tuple[int, int, int] = (16, 56, 56), ): super().__init__() self.out_channels = out_channels or in_channels @@ -569,13 +569,13 @@ def forward( encoder_hidden_states: torch.Tensor = None, encoder_hidden_states_mask: torch.Tensor = None, timestep: torch.LongTensor = None, - img_shapes: Optional[List[Tuple[int, int, int]]] = None, - txt_seq_lens: Optional[List[int]] = None, + img_shapes: Optional[list[tuple[int, int, int]]] = None, + txt_seq_lens: Optional[list[int]] = None, guidance: torch.Tensor = None, # TODO: this should probably be removed - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, controlnet_block_samples=None, return_dict: bool = True, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + ) -> torch.Tensor | Transformer2DModelOutput: """ The [`QwenTransformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_sd3.py b/src/diffusers/models/transformers/transformer_sd3.py index 762d89c303d7..c5565400567b 100644 --- a/src/diffusers/models/transformers/transformer_sd3.py +++ b/src/diffusers/models/transformers/transformer_sd3.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -107,7 +107,7 @@ class SD3Transformer2DModel( The number of latent channels in the output. pos_embed_max_size (`int`, defaults to `96`): The maximum latent height/width of positional embeddings. - dual_attention_layers (`Tuple[int, ...]`, defaults to `()`): + dual_attention_layers (`tuple[int, ...]`, defaults to `()`): The number of dual-stream transformer blocks to use. qk_norm (`str`, *optional*, defaults to `None`): The normalization to use for query and key in the attention layer. If `None`, no normalization is used. @@ -131,7 +131,7 @@ def __init__( pooled_projection_dim: int = 2048, out_channels: int = 16, pos_embed_max_size: int = 96, - dual_attention_layers: Tuple[ + dual_attention_layers: tuple[ int, ... ] = (), # () for sd3.0; (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) for sd3.5 qk_norm: Optional[str] = None, @@ -216,7 +216,7 @@ def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -225,7 +225,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -240,7 +240,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -312,11 +312,11 @@ def forward( encoder_hidden_states: torch.Tensor = None, pooled_projections: torch.Tensor = None, timestep: torch.LongTensor = None, - block_controlnet_hidden_states: List = None, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + block_controlnet_hidden_states: list = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - skip_layers: Optional[List[int]] = None, - ) -> Union[torch.Tensor, Transformer2DModelOutput]: + skip_layers: Optional[list[int]] = None, + ) -> torch.Tensor | Transformer2DModelOutput: """ The [`SD3Transformer2DModel`] forward method. diff --git a/src/diffusers/models/transformers/transformer_skyreels_v2.py b/src/diffusers/models/transformers/transformer_skyreels_v2.py index 6b600aa22487..a7d152456d08 100644 --- a/src/diffusers/models/transformers/transformer_skyreels_v2.py +++ b/src/diffusers/models/transformers/transformer_skyreels_v2.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -87,7 +87,7 @@ def __call__( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, ) -> torch.Tensor: encoder_hidden_states_img = None if attn.add_k_proj is not None: @@ -277,7 +277,7 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, **kwargs, ) -> torch.Tensor: return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, rotary_emb, **kwargs) @@ -375,7 +375,7 @@ class SkyReelsV2RotaryPosEmbed(nn.Module): def __init__( self, attention_head_dim: int, - patch_size: Tuple[int, int, int], + patch_size: tuple[int, int, int], max_seq_len: int, theta: float = 10000.0, ): @@ -522,7 +522,7 @@ class SkyReelsV2Transformer3DModel( A Transformer model for video-like data used in the Wan-based SkyReels-V2 model. Args: - patch_size (`Tuple[int]`, defaults to `(1, 2, 2)`): + patch_size (`tuple[int]`, defaults to `(1, 2, 2)`): 3D patch dimensions for video embedding (t_patch, h_patch, w_patch). num_attention_heads (`int`, defaults to `16`): Fixed length for text embeddings. @@ -540,7 +540,7 @@ class SkyReelsV2Transformer3DModel( Intermediate dimension in feed-forward network. num_layers (`int`, defaults to `32`): The number of layers of transformer blocks to use. - window_size (`Tuple[int]`, defaults to `(-1, -1)`): + window_size (`tuple[int]`, defaults to `(-1, -1)`): Window size for local attention (-1 indicates global attention). cross_attn_norm (`bool`, defaults to `True`): Enable cross-attention normalization. @@ -570,7 +570,7 @@ class SkyReelsV2Transformer3DModel( @register_to_config def __init__( self, - patch_size: Tuple[int] = (1, 2, 2), + patch_size: tuple[int] = (1, 2, 2), num_attention_heads: int = 16, attention_head_dim: int = 128, in_channels: int = 16, @@ -639,8 +639,8 @@ def forward( enable_diffusion_forcing: bool = False, fps: Optional[torch.Tensor] = None, return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> torch.Tensor | dict[str, torch.Tensor]: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/transformer_temporal.py b/src/diffusers/models/transformers/transformer_temporal.py index ffaf31d04570..76f71deebced 100644 --- a/src/diffusers/models/transformers/transformer_temporal.py +++ b/src/diffusers/models/transformers/transformer_temporal.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, Optional +from typing import Any, Optional import torch from torch import nn @@ -127,7 +127,7 @@ def forward( timestep: Optional[torch.LongTensor] = None, class_labels: torch.LongTensor = None, num_frames: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, ) -> TransformerTemporalModelOutput: """ diff --git a/src/diffusers/models/transformers/transformer_wan.py b/src/diffusers/models/transformers/transformer_wan.py index dd75fb124f1a..e5821d3e8ed3 100644 --- a/src/diffusers/models/transformers/transformer_wan.py +++ b/src/diffusers/models/transformers/transformer_wan.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -81,7 +81,7 @@ def __call__( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, ) -> torch.Tensor: encoder_hidden_states_img = None if attn.add_k_proj is not None: @@ -270,7 +270,7 @@ def forward( hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, **kwargs, ) -> torch.Tensor: return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, rotary_emb, **kwargs) @@ -350,7 +350,7 @@ class WanRotaryPosEmbed(nn.Module): def __init__( self, attention_head_dim: int, - patch_size: Tuple[int, int, int], + patch_size: tuple[int, int, int], max_seq_len: int, theta: float = 10000.0, ): @@ -505,7 +505,7 @@ class WanTransformer3DModel( A Transformer model for video-like data used in the Wan model. Args: - patch_size (`Tuple[int]`, defaults to `(1, 2, 2)`): + patch_size (`tuple[int]`, defaults to `(1, 2, 2)`): 3D patch dimensions for video embedding (t_patch, h_patch, w_patch). num_attention_heads (`int`, defaults to `40`): Fixed length for text embeddings. @@ -523,7 +523,7 @@ class WanTransformer3DModel( Intermediate dimension in feed-forward network. num_layers (`int`, defaults to `40`): The number of layers of transformer blocks to use. - window_size (`Tuple[int]`, defaults to `(-1, -1)`): + window_size (`tuple[int]`, defaults to `(-1, -1)`): Window size for local attention (-1 indicates global attention). cross_attn_norm (`bool`, defaults to `True`): Enable cross-attention normalization. @@ -560,7 +560,7 @@ class WanTransformer3DModel( @register_to_config def __init__( self, - patch_size: Tuple[int] = (1, 2, 2), + patch_size: tuple[int] = (1, 2, 2), num_attention_heads: int = 40, attention_head_dim: int = 128, in_channels: int = 16, @@ -621,8 +621,8 @@ def forward( encoder_hidden_states: torch.Tensor, encoder_hidden_states_image: Optional[torch.Tensor] = None, return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> torch.Tensor | dict[str, torch.Tensor]: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/transformers/transformer_wan_vace.py b/src/diffusers/models/transformers/transformer_wan_vace.py index 30c38c244ad8..2f3f4b5f9ee3 100644 --- a/src/diffusers/models/transformers/transformer_wan_vace.py +++ b/src/diffusers/models/transformers/transformer_wan_vace.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -141,7 +141,7 @@ class WanVACETransformer3DModel( A Transformer model for video-like data used in the Wan model. Args: - patch_size (`Tuple[int]`, defaults to `(1, 2, 2)`): + patch_size (`tuple[int]`, defaults to `(1, 2, 2)`): 3D patch dimensions for video embedding (t_patch, h_patch, w_patch). num_attention_heads (`int`, defaults to `40`): Fixed length for text embeddings. @@ -159,7 +159,7 @@ class WanVACETransformer3DModel( Intermediate dimension in feed-forward network. num_layers (`int`, defaults to `40`): The number of layers of transformer blocks to use. - window_size (`Tuple[int]`, defaults to `(-1, -1)`): + window_size (`tuple[int]`, defaults to `(-1, -1)`): Window size for local attention (-1 indicates global attention). cross_attn_norm (`bool`, defaults to `True`): Enable cross-attention normalization. @@ -182,7 +182,7 @@ class WanVACETransformer3DModel( @register_to_config def __init__( self, - patch_size: Tuple[int] = (1, 2, 2), + patch_size: tuple[int] = (1, 2, 2), num_attention_heads: int = 40, attention_head_dim: int = 128, in_channels: int = 16, @@ -198,7 +198,7 @@ def __init__( added_kv_proj_dim: Optional[int] = None, rope_max_seq_len: int = 1024, pos_embed_seq_len: Optional[int] = None, - vace_layers: List[int] = [0, 5, 10, 15, 20, 25, 30, 35], + vace_layers: list[int] = [0, 5, 10, 15, 20, 25, 30, 35], vace_in_channels: int = 96, ) -> None: super().__init__() @@ -270,8 +270,8 @@ def forward( control_hidden_states: torch.Tensor = None, control_hidden_states_scale: torch.Tensor = None, return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + attention_kwargs: Optional[dict[str, Any]] = None, + ) -> torch.Tensor | dict[str, torch.Tensor]: if attention_kwargs is not None: attention_kwargs = attention_kwargs.copy() lora_scale = attention_kwargs.pop("scale", 1.0) diff --git a/src/diffusers/models/unets/unet_1d.py b/src/diffusers/models/unets/unet_1d.py index 4c4c528a59ad..0cbfadf662d3 100644 --- a/src/diffusers/models/unets/unet_1d.py +++ b/src/diffusers/models/unets/unet_1d.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -56,12 +56,12 @@ class UNet1DModel(ModelMixin, ConfigMixin): freq_shift (`float`, *optional*, defaults to 0.0): Frequency shift for Fourier time embedding. flip_sin_to_cos (`bool`, *optional*, defaults to `False`): Whether to flip sin to cos for Fourier time embedding. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")`): - Tuple of downsample block types. - up_block_types (`Tuple[str]`, *optional*, defaults to `("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")`): - Tuple of upsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(32, 32, 64)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")`): + tuple of downsample block types. + up_block_types (`tuple[str]`, *optional*, defaults to `("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")`): + tuple of upsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(32, 32, 64)`): + tuple of block output channels. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock1D"`): Block type for middle of UNet. out_block_type (`str`, *optional*, defaults to `None`): Optional output processing block of UNet. act_fn (`str`, *optional*, defaults to `None`): Optional activation function in UNet blocks. @@ -86,11 +86,11 @@ def __init__( flip_sin_to_cos: bool = True, use_timestep_embedding: bool = False, freq_shift: float = 0.0, - down_block_types: Tuple[str] = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D"), - up_block_types: Tuple[str] = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip"), - mid_block_type: Tuple[str] = "UNetMidBlock1D", + down_block_types: tuple[str] = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D"), + up_block_types: tuple[str] = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip"), + mid_block_type: tuple[str] = "UNetMidBlock1D", out_block_type: str = None, - block_out_channels: Tuple[int] = (32, 32, 64), + block_out_channels: tuple[int] = (32, 32, 64), act_fn: str = None, norm_num_groups: int = 8, layers_per_block: int = 1, @@ -206,9 +206,9 @@ def __init__( def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, return_dict: bool = True, - ) -> Union[UNet1DOutput, Tuple]: + ) -> UNet1DOutput | tuple: r""" The [`UNet1DModel`] forward method. diff --git a/src/diffusers/models/unets/unet_1d_blocks.py b/src/diffusers/models/unets/unet_1d_blocks.py index 58cbdfd005b6..3760fc99ce9f 100644 --- a/src/diffusers/models/unets/unet_1d_blocks.py +++ b/src/diffusers/models/unets/unet_1d_blocks.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import math -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn.functional as F @@ -129,7 +129,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Optional[Tuple[torch.Tensor, ...]] = None, + res_hidden_states_tuple: Optional[tuple[torch.Tensor, ...]] = None, temb: Optional[torch.Tensor] = None, ) -> torch.Tensor: if res_hidden_states_tuple is not None: @@ -543,7 +543,7 @@ def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[i def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, ) -> torch.Tensor: res_hidden_states = res_hidden_states_tuple[-1] @@ -575,7 +575,7 @@ def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[i def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, ) -> torch.Tensor: res_hidden_states = res_hidden_states_tuple[-1] @@ -605,7 +605,7 @@ def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[i def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, ) -> torch.Tensor: res_hidden_states = res_hidden_states_tuple[-1] @@ -617,10 +617,10 @@ def forward( return hidden_states -DownBlockType = Union[DownResnetBlock1D, DownBlock1D, AttnDownBlock1D, DownBlock1DNoSkip] -MidBlockType = Union[MidResTemporalBlock1D, ValueFunctionMidBlock1D, UNetMidBlock1D] -OutBlockType = Union[OutConv1DBlock, OutValueFunctionBlock] -UpBlockType = Union[UpResnetBlock1D, UpBlock1D, AttnUpBlock1D, UpBlock1DNoSkip] +DownBlockType = DownResnetBlock1D | DownBlock1D | AttnDownBlock1D | DownBlock1DNoSkip +MidBlockType = MidResTemporalBlock1D | ValueFunctionMidBlock1D | UNetMidBlock1D +OutBlockType = OutConv1DBlock | OutValueFunctionBlock +UpBlockType = UpResnetBlock1D | UpBlock1D | AttnUpBlock1D | UpBlock1DNoSkip def get_down_block( diff --git a/src/diffusers/models/unets/unet_2d.py b/src/diffusers/models/unets/unet_2d.py index 2588a9c518bd..b9253154150a 100644 --- a/src/diffusers/models/unets/unet_2d.py +++ b/src/diffusers/models/unets/unet_2d.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -45,7 +45,7 @@ class UNet2DModel(ModelMixin, ConfigMixin): for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. Dimensions must be a multiple of `2 ** (len(block_out_channels) - 1)`. in_channels (`int`, *optional*, defaults to 3): Number of channels in the input sample. @@ -55,14 +55,14 @@ class UNet2DModel(ModelMixin, ConfigMixin): freq_shift (`int`, *optional*, defaults to 0): Frequency shift for Fourier time embedding. flip_sin_to_cos (`bool`, *optional*, defaults to `True`): Whether to flip sin to cos for Fourier time embedding. - down_block_types (`Tuple[str]`, *optional*, defaults to `("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D")`): - Tuple of downsample block types. + down_block_types (`tuple[str]`, *optional*, defaults to `("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D")`): + tuple of downsample block types. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2D"`): Block type for middle of UNet, it can be either `UNetMidBlock2D` or `None`. - up_block_types (`Tuple[str]`, *optional*, defaults to `("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D")`): - Tuple of upsample block types. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(224, 448, 672, 896)`): - Tuple of block output channels. + up_block_types (`tuple[str]`, *optional*, defaults to `("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D")`): + tuple of upsample block types. + block_out_channels (`tuple[int]`, *optional*, defaults to `(224, 448, 672, 896)`): + tuple of block output channels. layers_per_block (`int`, *optional*, defaults to `2`): The number of layers per block. mid_block_scale_factor (`float`, *optional*, defaults to `1`): The scale factor for the mid block. downsample_padding (`int`, *optional*, defaults to `1`): The padding for the downsample convolution. @@ -95,7 +95,7 @@ class UNet2DModel(ModelMixin, ConfigMixin): @register_to_config def __init__( self, - sample_size: Optional[Union[int, Tuple[int, int]]] = None, + sample_size: Optional[int | tuple[int, int]] = None, in_channels: int = 3, out_channels: int = 3, center_input_sample: bool = False, @@ -103,10 +103,10 @@ def __init__( time_embedding_dim: Optional[int] = None, freq_shift: int = 0, flip_sin_to_cos: bool = True, - down_block_types: Tuple[str, ...] = ("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D"), + down_block_types: tuple[str, ...] = ("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D"), mid_block_type: Optional[str] = "UNetMidBlock2D", - up_block_types: Tuple[str, ...] = ("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D"), - block_out_channels: Tuple[int, ...] = (224, 448, 672, 896), + up_block_types: tuple[str, ...] = ("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D"), + block_out_channels: tuple[int, ...] = (224, 448, 672, 896), layers_per_block: int = 2, mid_block_scale_factor: float = 1, downsample_padding: int = 1, @@ -250,10 +250,10 @@ def __init__( def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, class_labels: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[UNet2DOutput, Tuple]: + ) -> UNet2DOutput | tuple: r""" The [`UNet2DModel`] forward method. diff --git a/src/diffusers/models/unets/unet_2d_blocks.py b/src/diffusers/models/unets/unet_2d_blocks.py index 94a9245e567c..8ed72cf5bdcf 100644 --- a/src/diffusers/models/unets/unet_2d_blocks.py +++ b/src/diffusers/models/unets/unet_2d_blocks.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -756,7 +756,7 @@ def __init__( out_channels: Optional[int] = None, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -857,7 +857,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: if cross_attention_kwargs is not None: @@ -981,7 +981,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} @@ -1114,8 +1114,8 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + cross_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -1152,7 +1152,7 @@ def __init__( temb_channels: int, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -1242,10 +1242,10 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, additional_residuals: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: if cross_attention_kwargs is not None: if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -1345,7 +1345,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, *args, **kwargs - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -1639,7 +1639,7 @@ def forward( skip_sample: Optional[torch.Tensor] = None, *args, **kwargs, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...], torch.Tensor]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...], torch.Tensor]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -1730,7 +1730,7 @@ def forward( skip_sample: Optional[torch.Tensor] = None, *args, **kwargs, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...], torch.Tensor]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...], torch.Tensor]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -1819,7 +1819,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, *args, **kwargs - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -1944,9 +1944,9 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -2042,7 +2042,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, *args, **kwargs - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -2135,9 +2135,9 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} if cross_attention_kwargs.get("scale", None) is not None: logger.warning("Passing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.") @@ -2278,7 +2278,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, *args, @@ -2321,7 +2321,7 @@ def __init__( resolution_idx: Optional[int] = None, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -2405,10 +2405,10 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, @@ -2524,7 +2524,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, *args, @@ -2841,12 +2841,12 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, skip_sample=None, *args, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -2953,12 +2953,12 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, skip_sample=None, *args, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -3060,7 +3060,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, *args, @@ -3190,12 +3190,12 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} @@ -3298,7 +3298,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, *args, @@ -3414,10 +3414,10 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, @@ -3541,7 +3541,7 @@ def forward( # requires assessing impact of change to positional param interface. emb: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} diff --git a/src/diffusers/models/unets/unet_2d_condition.py b/src/diffusers/models/unets/unet_2d_condition.py index f04d3dfa0136..14ef7bad9564 100644 --- a/src/diffusers/models/unets/unet_2d_condition.py +++ b/src/diffusers/models/unets/unet_2d_condition.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -78,7 +78,7 @@ class UNet2DConditionModel( for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. @@ -86,17 +86,17 @@ class UNet2DConditionModel( flip_sin_to_cos (`bool`, *optional*, defaults to `True`): Whether to flip the sin to cos in the time embedding. freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): The tuple of upsample blocks to use. - only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + only_cross_attention(`bool` or `tuple[bool]`, *optional*, default to `False`): Whether to include self-attention in the basic transformer blocks, see [`~models.attention.BasicTransformerBlock`]. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. @@ -106,15 +106,15 @@ class UNet2DConditionModel( norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. If `None`, normalization and activation layers is skipped in post-processing. norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. - cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + cross_attention_dim (`int` or `tuple[int]`, *optional*, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + transformer_layers_per_block (`int`, `tuple[int]`, or `tuple[tuple]` , *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unets.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unets.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unets.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. - reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None): + reverse_transformer_layers_per_block : (`tuple[tuple]`, *optional*, defaults to None): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling - blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for + blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `tuple[tuple]` and for [`~models.unets.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unets.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unets.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. encoder_hid_dim (`int`, *optional*, defaults to None): @@ -171,36 +171,36 @@ class conditioning with `class_embed_type` equal to `None`. @register_to_config def __init__( self, - sample_size: Optional[Union[int, Tuple[int, int]]] = None, + sample_size: Optional[int | tuple[int, int]] = None, in_channels: int = 4, out_channels: int = 4, center_input_sample: bool = False, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", - up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int] = (320, 640, 1280, 1280), - layers_per_block: Union[int, Tuple[int]] = 2, + up_block_types: tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int] = (320, 640, 1280, 1280), + layers_per_block: int | tuple[int] = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, dropout: float = 0.0, act_fn: str = "silu", norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, - cross_attention_dim: Union[int, Tuple[int]] = 1280, - transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, - reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None, + cross_attention_dim: int | tuple[int] = 1280, + transformer_layers_per_block: int | tuple[int] | tuple[tuple] = 1, + reverse_transformer_layers_per_block: Optional[tuple[tuple[int]]] = None, encoder_hid_dim: Optional[int] = None, encoder_hid_dim_type: Optional[str] = None, - attention_head_dim: Union[int, Tuple[int]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + attention_head_dim: int | tuple[int] = 8, + num_attention_heads: Optional[int | tuple[int]] = None, dual_cross_attention: bool = False, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, @@ -486,16 +486,16 @@ def __init__( def _check_config( self, - down_block_types: Tuple[str], - up_block_types: Tuple[str], - only_cross_attention: Union[bool, Tuple[bool]], - block_out_channels: Tuple[int], - layers_per_block: Union[int, Tuple[int]], - cross_attention_dim: Union[int, Tuple[int]], - transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple[int]]], + down_block_types: tuple[str], + up_block_types: tuple[str], + only_cross_attention: bool | tuple[bool], + block_out_channels: tuple[int], + layers_per_block: int | tuple[int], + cross_attention_dim: int | tuple[int], + transformer_layers_per_block: int | tuple[int] | tuple[tuple[int]], reverse_transformer_layers_per_block: bool, attention_head_dim: int, - num_attention_heads: Optional[Union[int, Tuple[int]]], + num_attention_heads: Optional[int | tuple[int]], ): if len(down_block_types) != len(up_block_types): raise ValueError( @@ -543,7 +543,7 @@ def _set_time_proj( flip_sin_to_cos: bool, freq_shift: float, time_embedding_dim: int, - ) -> Tuple[int, int]: + ) -> tuple[int, int]: if time_embedding_type == "fourier": time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 if time_embed_dim % 2 != 0: @@ -567,7 +567,7 @@ def _set_time_proj( def _set_encoder_hid_proj( self, encoder_hid_dim_type: Optional[str], - cross_attention_dim: Union[int, Tuple[int]], + cross_attention_dim: int | tuple[int], encoder_hid_dim: Optional[int], ): if encoder_hid_dim_type is None and encoder_hid_dim is not None: @@ -697,7 +697,7 @@ def _set_pos_net_if_use_gligen(self, attention_type: str, cross_attention_dim: i ) @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -706,7 +706,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -720,7 +720,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -769,7 +769,7 @@ def set_default_attn_processor(self): self.set_attn_processor(processor) - def set_attention_slice(self, slice_size: Union[str, int, List[int]] = "auto"): + def set_attention_slice(self, slice_size: str | int | list[int] = "auto"): r""" Enable sliced attention computation. @@ -823,7 +823,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -896,9 +896,7 @@ def unfuse_qkv_projections(self): if self.original_attn_processors is not None: self.set_attn_processor(self.original_attn_processors) - def get_time_embed( - self, sample: torch.Tensor, timestep: Union[torch.Tensor, float, int] - ) -> Optional[torch.Tensor]: + def get_time_embed(self, sample: torch.Tensor, timestep: torch.Tensor | float | int) -> Optional[torch.Tensor]: timesteps = timestep if not torch.is_tensor(timesteps): # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can @@ -940,7 +938,7 @@ def get_class_embed(self, sample: torch.Tensor, class_labels: Optional[torch.Ten return class_emb def get_aug_embed( - self, emb: torch.Tensor, encoder_hidden_states: torch.Tensor, added_cond_kwargs: Dict[str, Any] + self, emb: torch.Tensor, encoder_hidden_states: torch.Tensor, added_cond_kwargs: dict[str, Any] ) -> Optional[torch.Tensor]: aug_emb = None if self.config.addition_embed_type == "text": @@ -992,7 +990,7 @@ def get_aug_embed( return aug_emb def process_encoder_hidden_states( - self, encoder_hidden_states: torch.Tensor, added_cond_kwargs: Dict[str, Any] + self, encoder_hidden_states: torch.Tensor, added_cond_kwargs: dict[str, Any] ) -> torch.Tensor: if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj": encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states) @@ -1030,19 +1028,19 @@ def process_encoder_hidden_states( def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[tuple[torch.Tensor]] = None, mid_block_additional_residual: Optional[torch.Tensor] = None, - down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + down_intrablock_additional_residuals: Optional[tuple[torch.Tensor]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[UNet2DConditionOutput, Tuple]: + ) -> UNet2DConditionOutput | tuple: r""" The [`UNet2DConditionModel`] forward method. diff --git a/src/diffusers/models/unets/unet_2d_condition_flax.py b/src/diffusers/models/unets/unet_2d_condition_flax.py index 8d9a309afbcc..32a9003e1179 100644 --- a/src/diffusers/models/unets/unet_2d_condition_flax.py +++ b/src/diffusers/models/unets/unet_2d_condition_flax.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional import flax import flax.linen as nn @@ -74,20 +74,20 @@ class FlaxUNet2DConditionModel(nn.Module, FlaxModelMixin, ConfigMixin): The number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): The tuple of downsample blocks to use. - up_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("FlaxUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D")`): The tuple of upsample blocks to use. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`. If `None`, the mid block layer is skipped. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. - attention_head_dim (`int` or `Tuple[int]`, *optional*, defaults to 8): + attention_head_dim (`int` or `tuple[int]`, *optional*, defaults to 8): The dimension of the attention heads. - num_attention_heads (`int` or `Tuple[int]`, *optional*): + num_attention_heads (`int` or `tuple[int]`, *optional*): The number of attention heads. cross_attention_dim (`int`, *optional*, defaults to 768): The dimension of the cross attention features. @@ -106,19 +106,19 @@ class FlaxUNet2DConditionModel(nn.Module, FlaxModelMixin, ConfigMixin): sample_size: int = 32 in_channels: int = 4 out_channels: int = 4 - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ) - up_block_types: Tuple[str, ...] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D") + up_block_types: tuple[str, ...] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D") mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn" - only_cross_attention: Union[bool, Tuple[bool]] = False - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280) + only_cross_attention: bool | tuple[bool] = False + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280) layers_per_block: int = 2 - attention_head_dim: Union[int, Tuple[int, ...]] = 8 - num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None + attention_head_dim: int | tuple[int, ...] = 8 + num_attention_heads: Optional[int | tuple[int, ...]] = None cross_attention_dim: int = 1280 dropout: float = 0.0 use_linear_projection: bool = False @@ -127,7 +127,7 @@ class FlaxUNet2DConditionModel(nn.Module, FlaxModelMixin, ConfigMixin): freq_shift: int = 0 use_memory_efficient_attention: bool = False split_head_dim: bool = False - transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1 + transformer_layers_per_block: int | tuple[int, ...] = 1 addition_embed_type: Optional[str] = None addition_time_embed_dim: Optional[int] = None addition_embed_type_num_heads: int = 64 @@ -338,14 +338,14 @@ def setup(self) -> None: def __call__( self, sample: jnp.ndarray, - timesteps: Union[jnp.ndarray, float, int], + timesteps: jnp.ndarray | float | int, encoder_hidden_states: jnp.ndarray, - added_cond_kwargs: Optional[Union[Dict, FrozenDict]] = None, - down_block_additional_residuals: Optional[Tuple[jnp.ndarray, ...]] = None, + added_cond_kwargs: Optional[Dict | FrozenDict] = None, + down_block_additional_residuals: Optional[tuple[jnp.ndarray, ...]] = None, mid_block_additional_residual: Optional[jnp.ndarray] = None, return_dict: bool = True, train: bool = False, - ) -> Union[FlaxUNet2DConditionOutput, Tuple[jnp.ndarray]]: + ) -> FlaxUNet2DConditionOutput | tuple[jnp.ndarray]: r""" Args: sample (`jnp.ndarray`): (batch, channel, height, width) noisy inputs tensor diff --git a/src/diffusers/models/unets/unet_3d_blocks.py b/src/diffusers/models/unets/unet_3d_blocks.py index 53c0f4bae38b..5b6c2f9c59ea 100644 --- a/src/diffusers/models/unets/unet_3d_blocks.py +++ b/src/diffusers/models/unets/unet_3d_blocks.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from __future__ import annotations + +from typing import Any, Optional import torch from torch import nn @@ -99,15 +101,10 @@ def get_down_block( resnet_time_scale_shift: str = "default", temporal_num_attention_heads: int = 8, temporal_max_seq_length: int = 32, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, dropout: float = 0.0, -) -> Union[ - "DownBlock3D", - "CrossAttnDownBlock3D", - "DownBlockSpatioTemporal", - "CrossAttnDownBlockSpatioTemporal", -]: +) -> "DownBlock3D" | "CrossAttnDownBlock3D" | "DownBlockSpatioTemporal" | "CrossAttnDownBlockSpatioTemporal": if down_block_type == "DownBlock3D": return DownBlock3D( num_layers=num_layers, @@ -193,15 +190,10 @@ def get_up_block( temporal_num_attention_heads: int = 8, temporal_cross_attention_dim: Optional[int] = None, temporal_max_seq_length: int = 32, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, dropout: float = 0.0, -) -> Union[ - "UpBlock3D", - "CrossAttnUpBlock3D", - "UpBlockSpatioTemporal", - "CrossAttnUpBlockSpatioTemporal", -]: +) -> "UpBlock3D" | "CrossAttnUpBlock3D" | "UpBlockSpatioTemporal" | "CrossAttnUpBlockSpatioTemporal": if up_block_type == "UpBlock3D": return UpBlock3D( num_layers=num_layers, @@ -380,7 +372,7 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, num_frames: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ) -> torch.Tensor: hidden_states = self.resnets[0](hidden_states, temb) hidden_states = self.temp_convs[0](hidden_states, num_frames=num_frames) @@ -513,8 +505,8 @@ def forward( encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, num_frames: int = 1, - cross_attention_kwargs: Dict[str, Any] = None, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, ...]]: + cross_attention_kwargs: dict[str, Any] = None, + ) -> torch.Tensor | tuple[torch.Tensor, ...]: # TODO(Patrick, William) - attention mask is not used output_states = () @@ -618,7 +610,7 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, num_frames: int = 1, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> torch.Tensor | tuple[torch.Tensor, ...]: output_states = () for resnet, temp_conv in zip(self.resnets, self.temp_convs): @@ -734,13 +726,13 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, num_frames: int = 1, - cross_attention_kwargs: Dict[str, Any] = None, + cross_attention_kwargs: dict[str, Any] = None, ) -> torch.Tensor: is_freeu_enabled = ( getattr(self, "s1", None) @@ -856,7 +848,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, num_frames: int = 1, @@ -1015,7 +1007,7 @@ def __init__( in_channels: int, temb_channels: int, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, num_attention_heads: int = 1, cross_attention_dim: int = 1280, ): @@ -1144,7 +1136,7 @@ def forward( hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None, image_only_indicator: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: output_states = () for resnet in self.resnets: if torch.is_grad_enabled() and self.gradient_checkpointing: @@ -1170,7 +1162,7 @@ def __init__( out_channels: int, temb_channels: int, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, num_attention_heads: int = 1, cross_attention_dim: int = 1280, add_downsample: bool = True, @@ -1230,7 +1222,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, image_only_indicator: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: output_states = () blocks = list(zip(self.resnets, self.attentions)) @@ -1305,7 +1297,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, image_only_indicator: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, @@ -1338,7 +1330,7 @@ def __init__( temb_channels: int, resolution_idx: Optional[int] = None, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, num_attention_heads: int = 1, cross_attention_dim: int = 1280, @@ -1390,7 +1382,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, image_only_indicator: Optional[torch.Tensor] = None, diff --git a/src/diffusers/models/unets/unet_3d_condition.py b/src/diffusers/models/unets/unet_3d_condition.py index 6a119185b812..1c5ee36cd9b7 100644 --- a/src/diffusers/models/unets/unet_3d_condition.py +++ b/src/diffusers/models/unets/unet_3d_condition.py @@ -14,7 +14,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -67,15 +67,15 @@ class UNet3DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D")`): The tuple of downsample blocks to use. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D")`): The tuple of upsample blocks to use. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. @@ -100,19 +100,19 @@ def __init__( sample_size: Optional[int] = None, in_channels: int = 4, out_channels: int = 4, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D", ), - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", ), - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, @@ -120,8 +120,8 @@ def __init__( norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 1024, - attention_head_dim: Union[int, Tuple[int]] = 64, - num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + attention_head_dim: int | tuple[int] = 64, + num_attention_heads: Optional[int | tuple[int]] = None, time_cond_proj_dim: Optional[int] = None, ): super().__init__() @@ -287,7 +287,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -296,7 +296,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -311,7 +311,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice - def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + def set_attention_slice(self, slice_size: str | int | list[int]) -> None: r""" Enable sliced attention computation. @@ -365,7 +365,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -377,7 +377,7 @@ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[i fn_recursive_set_attention_slice(module, reversed_slice_size) # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -536,16 +536,16 @@ def unfuse_qkv_projections(self): def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + down_block_additional_residuals: Optional[tuple[torch.Tensor]] = None, mid_block_additional_residual: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[UNet3DConditionOutput, Tuple[torch.Tensor]]: + ) -> UNet3DConditionOutput | tuple[torch.Tensor]: r""" The [`UNet3DConditionModel`] forward method. diff --git a/src/diffusers/models/unets/unet_i2vgen_xl.py b/src/diffusers/models/unets/unet_i2vgen_xl.py index 3dba8edca79d..34b1243214ee 100644 --- a/src/diffusers/models/unets/unet_i2vgen_xl.py +++ b/src/diffusers/models/unets/unet_i2vgen_xl.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -102,15 +102,15 @@ class I2VGenXLUNet(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): The tuple of upsample blocks to use. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. @@ -128,24 +128,24 @@ def __init__( sample_size: Optional[int] = None, in_channels: int = 4, out_channels: int = 4, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D", ), - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", ), - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), layers_per_block: int = 2, norm_num_groups: Optional[int] = 32, cross_attention_dim: int = 1024, - attention_head_dim: Union[int, Tuple[int]] = 64, - num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + attention_head_dim: int | tuple[int] = 64, + num_attention_heads: Optional[int | tuple[int]] = None, ): super().__init__() @@ -315,7 +315,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -324,7 +324,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -339,7 +339,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -500,15 +500,15 @@ def unfuse_qkv_projections(self): def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, fps: torch.Tensor, image_latents: torch.Tensor, image_embeddings: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, return_dict: bool = True, - ) -> Union[UNet3DConditionOutput, Tuple[torch.Tensor]]: + ) -> UNet3DConditionOutput | tuple[torch.Tensor]: r""" The [`I2VGenXLUNet`] forward method. diff --git a/src/diffusers/models/unets/unet_kandinsky3.py b/src/diffusers/models/unets/unet_kandinsky3.py index 27241ce2e674..056f9ad249cd 100644 --- a/src/diffusers/models/unets/unet_kandinsky3.py +++ b/src/diffusers/models/unets/unet_kandinsky3.py @@ -13,7 +13,6 @@ # limitations under the License. from dataclasses import dataclass -from typing import Dict, Tuple, Union import torch from torch import nn @@ -53,9 +52,9 @@ def __init__( time_embedding_dim: int = 1536, groups: int = 32, attention_head_dim: int = 64, - layers_per_block: Union[int, Tuple[int]] = 3, - block_out_channels: Tuple[int] = (384, 768, 1536, 3072), - cross_attention_dim: Union[int, Tuple[int]] = 4096, + layers_per_block: int | tuple[int] = 3, + block_out_channels: tuple[int] = (384, 768, 1536, 3072), + cross_attention_dim: int | tuple[int] = 4096, encoder_hid_dim: int = 4096, ): super().__init__() @@ -141,7 +140,7 @@ def __init__( self.conv_out = nn.Conv2d(init_channels, out_channels, kernel_size=3, padding=1) @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -150,7 +149,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "set_processor"): processors[f"{name}.processor"] = module.processor @@ -164,7 +163,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. diff --git a/src/diffusers/models/unets/unet_motion_model.py b/src/diffusers/models/unets/unet_motion_model.py index 18d5eb917f48..9a015b3a93d1 100644 --- a/src/diffusers/models/unets/unet_motion_model.py +++ b/src/diffusers/models/unets/unet_motion_model.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -147,7 +147,7 @@ def forward( timestep: Optional[torch.LongTensor] = None, class_labels: Optional[torch.LongTensor] = None, num_frames: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ) -> torch.Tensor: """ The [`AnimateDiffTransformer3D`] forward method. @@ -228,10 +228,10 @@ def __init__( output_scale_factor: float = 1.0, add_downsample: bool = True, downsample_padding: int = 1, - temporal_num_attention_heads: Union[int, Tuple[int]] = 1, + temporal_num_attention_heads: int | tuple[int] = 1, temporal_cross_attention_dim: Optional[int] = None, temporal_max_seq_length: int = 32, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, temporal_double_self_attention: bool = True, ): super().__init__() @@ -313,7 +313,7 @@ def forward( num_frames: int = 1, *args, **kwargs, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> torch.Tensor | tuple[torch.Tensor, ...]: if len(args) > 0 or kwargs.get("scale", None) is not None: deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." deprecate("scale", "1.0.0", deprecation_message) @@ -348,7 +348,7 @@ def __init__( temb_channels: int, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -367,7 +367,7 @@ def __init__( temporal_cross_attention_dim: Optional[int] = None, temporal_num_attention_heads: int = 8, temporal_max_seq_length: int = 32, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, temporal_double_self_attention: bool = True, ): super().__init__() @@ -483,7 +483,7 @@ def forward( attention_mask: Optional[torch.Tensor] = None, num_frames: int = 1, encoder_attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, additional_residuals: Optional[torch.Tensor] = None, ): if cross_attention_kwargs is not None: @@ -535,7 +535,7 @@ def __init__( resolution_idx: Optional[int] = None, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -553,7 +553,7 @@ def __init__( temporal_cross_attention_dim: Optional[int] = None, temporal_num_attention_heads: int = 8, temporal_max_seq_length: int = 32, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, ): super().__init__() resnets = [] @@ -654,10 +654,10 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, @@ -737,7 +737,7 @@ def __init__( temporal_cross_attention_dim: Optional[int] = None, temporal_num_attention_heads: int = 8, temporal_max_seq_length: int = 32, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, ): super().__init__() resnets = [] @@ -799,7 +799,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size=None, num_frames: int = 1, @@ -859,7 +859,7 @@ def __init__( temb_channels: int, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -875,7 +875,7 @@ def __init__( temporal_num_attention_heads: int = 1, temporal_cross_attention_dim: Optional[int] = None, temporal_max_seq_length: int = 32, - temporal_transformer_layers_per_block: Union[int, Tuple[int]] = 1, + temporal_transformer_layers_per_block: int | tuple[int] = 1, ): super().__init__() @@ -984,7 +984,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, num_frames: int = 1, ) -> torch.Tensor: @@ -1022,8 +1022,8 @@ def __init__( self, in_channels: int, layers_per_block: int = 2, - transformer_layers_per_block: Union[int, Tuple[int]] = 8, - num_attention_heads: Union[int, Tuple[int]] = 8, + transformer_layers_per_block: int | tuple[int] = 8, + num_attention_heads: int | tuple[int] = 8, attention_bias: bool = False, cross_attention_dim: Optional[int] = None, activation_fn: str = "geglu", @@ -1062,12 +1062,12 @@ class MotionAdapter(ModelMixin, ConfigMixin, FromOriginalModelMixin): @register_to_config def __init__( self, - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), - motion_layers_per_block: Union[int, Tuple[int]] = 2, - motion_transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple[int]]] = 1, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), + motion_layers_per_block: int | tuple[int] = 2, + motion_transformer_layers_per_block: int | tuple[int] | tuple[tuple[int]] = 1, motion_mid_block_layers_per_block: int = 1, - motion_transformer_layers_per_mid_block: Union[int, Tuple[int]] = 1, - motion_num_attention_heads: Union[int, Tuple[int]] = 8, + motion_transformer_layers_per_mid_block: int | tuple[int] = 1, + motion_num_attention_heads: int | tuple[int] = 8, motion_norm_num_groups: int = 32, motion_max_seq_length: int = 32, use_motion_mid_block: bool = True, @@ -1076,17 +1076,17 @@ def __init__( """Container to store AnimateDiff Motion Modules Args: - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each UNet block. - motion_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 2): + motion_layers_per_block (`int` or `tuple[int]`, *optional*, defaults to 2): The number of motion layers per UNet block. - motion_transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple[int]]`, *optional*, defaults to 1): + motion_transformer_layers_per_block (`int`, `tuple[int]`, or `tuple[tuple[int]]`, *optional*, defaults to 1): The number of transformer layers to use in each motion layer in each block. motion_mid_block_layers_per_block (`int`, *optional*, defaults to 1): The number of motion layers in the middle UNet block. - motion_transformer_layers_per_mid_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + motion_transformer_layers_per_mid_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer layers to use in each motion layer in the middle block. - motion_num_attention_heads (`int` or `Tuple[int]`, *optional*, defaults to 8): + motion_num_attention_heads (`int` or `tuple[int]`, *optional*, defaults to 8): The number of heads to use in each attention layer of the motion module. motion_norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use in each group normalization layer of the motion module. @@ -1213,37 +1213,37 @@ def __init__( sample_size: Optional[int] = None, in_channels: int = 4, out_channels: int = 4, - down_block_types: Tuple[str, ...] = ( + down_block_types: tuple[str, ...] = ( "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "CrossAttnDownBlockMotion", "DownBlockMotion", ), - up_block_types: Tuple[str, ...] = ( + up_block_types: tuple[str, ...] = ( "UpBlockMotion", "CrossAttnUpBlockMotion", "CrossAttnUpBlockMotion", "CrossAttnUpBlockMotion", ), - block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), - layers_per_block: Union[int, Tuple[int]] = 2, + block_out_channels: tuple[int, ...] = (320, 640, 1280, 1280), + layers_per_block: int | tuple[int] = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, act_fn: str = "silu", norm_num_groups: int = 32, norm_eps: float = 1e-5, cross_attention_dim: int = 1280, - transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, - reverse_transformer_layers_per_block: Optional[Union[int, Tuple[int], Tuple[Tuple]]] = None, - temporal_transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, - reverse_temporal_transformer_layers_per_block: Optional[Union[int, Tuple[int], Tuple[Tuple]]] = None, - transformer_layers_per_mid_block: Optional[Union[int, Tuple[int]]] = None, - temporal_transformer_layers_per_mid_block: Optional[Union[int, Tuple[int]]] = 1, + transformer_layers_per_block: int | tuple[int] | tuple[tuple] = 1, + reverse_transformer_layers_per_block: Optional[int | tuple[int] | tuple[tuple]] = None, + temporal_transformer_layers_per_block: int | tuple[int] | tuple[tuple] = 1, + reverse_temporal_transformer_layers_per_block: Optional[int | tuple[int] | tuple[tuple]] = None, + transformer_layers_per_mid_block: Optional[int | tuple[int]] = None, + temporal_transformer_layers_per_mid_block: Optional[int | tuple[int]] = 1, use_linear_projection: bool = False, - num_attention_heads: Union[int, Tuple[int, ...]] = 8, + num_attention_heads: int | tuple[int, ...] = 8, motion_max_seq_length: int = 32, - motion_num_attention_heads: Union[int, Tuple[int, ...]] = 8, - reverse_motion_num_attention_heads: Optional[Union[int, Tuple[int, ...], Tuple[Tuple[int, ...], ...]]] = None, + motion_num_attention_heads: int | tuple[int, ...] = 8, + reverse_motion_num_attention_heads: Optional[int | tuple[int, ...] | tuple[tuple[int, ...], ...]] = None, use_motion_mid_block: bool = True, mid_block_layers: int = 1, encoder_hid_dim: Optional[int] = None, @@ -1756,7 +1756,7 @@ def save_motion_modules( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -1765,7 +1765,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -1780,7 +1780,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -1939,16 +1939,16 @@ def unfuse_qkv_projections(self): def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[tuple[torch.Tensor]] = None, mid_block_additional_residual: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[UNetMotionOutput, Tuple[torch.Tensor]]: + ) -> UNetMotionOutput | tuple[torch.Tensor]: r""" The [`UNetMotionModel`] forward method. diff --git a/src/diffusers/models/unets/unet_spatio_temporal_condition.py b/src/diffusers/models/unets/unet_spatio_temporal_condition.py index 059a6e807c8e..5c12c8afb95e 100644 --- a/src/diffusers/models/unets/unet_spatio_temporal_condition.py +++ b/src/diffusers/models/unets/unet_spatio_temporal_condition.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -38,29 +38,29 @@ class UNetSpatioTemporalConditionModel(ModelMixin, ConfigMixin, UNet2DConditionL for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 8): Number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "DownBlockSpatioTemporal")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "DownBlockSpatioTemporal")`): The tuple of downsample blocks to use. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal")`): The tuple of upsample blocks to use. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. addition_time_embed_dim: (`int`, defaults to 256): Dimension to to encode the additional time ids. projection_class_embeddings_input_dim (`int`, defaults to 768): The dimension of the projection of encoded `added_time_ids`. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. - cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + cross_attention_dim (`int` or `tuple[int]`, *optional*, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + transformer_layers_per_block (`int`, `tuple[int]`, or `tuple[tuple]` , *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unets.unet_3d_blocks.CrossAttnDownBlockSpatioTemporal`], [`~models.unets.unet_3d_blocks.CrossAttnUpBlockSpatioTemporal`], [`~models.unets.unet_3d_blocks.UNetMidBlockSpatioTemporal`]. - num_attention_heads (`int`, `Tuple[int]`, defaults to `(5, 10, 10, 20)`): + num_attention_heads (`int`, `tuple[int]`, defaults to `(5, 10, 10, 20)`): The number of attention heads. dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. """ @@ -73,25 +73,25 @@ def __init__( sample_size: Optional[int] = None, in_channels: int = 8, out_channels: int = 4, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "DownBlockSpatioTemporal", ), - up_block_types: Tuple[str] = ( + up_block_types: tuple[str] = ( "UpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", ), - block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + block_out_channels: tuple[int] = (320, 640, 1280, 1280), addition_time_embed_dim: int = 256, projection_class_embeddings_input_dim: int = 768, - layers_per_block: Union[int, Tuple[int]] = 2, - cross_attention_dim: Union[int, Tuple[int]] = 1024, - transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, - num_attention_heads: Union[int, Tuple[int]] = (5, 10, 20, 20), + layers_per_block: int | tuple[int] = 2, + cross_attention_dim: int | tuple[int] = 1024, + transformer_layers_per_block: int | tuple[int] | tuple[tuple] = 1, + num_attention_heads: int | tuple[int] = (5, 10, 20, 20), num_frames: int = 25, ): super().__init__() @@ -246,7 +246,7 @@ def __init__( ) @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -258,7 +258,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: def fn_recursive_add_processors( name: str, module: torch.nn.Module, - processors: Dict[str, AttentionProcessor], + processors: dict[str, AttentionProcessor], ): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -273,7 +273,7 @@ def fn_recursive_add_processors( return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -353,11 +353,11 @@ def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, added_time_ids: torch.Tensor, return_dict: bool = True, - ) -> Union[UNetSpatioTemporalConditionOutput, Tuple]: + ) -> UNetSpatioTemporalConditionOutput | tuple: r""" The [`UNetSpatioTemporalConditionModel`] forward method. diff --git a/src/diffusers/models/unets/unet_stable_cascade.py b/src/diffusers/models/unets/unet_stable_cascade.py index e79ce8ac1da4..9a42c21bf38a 100644 --- a/src/diffusers/models/unets/unet_stable_cascade.py +++ b/src/diffusers/models/unets/unet_stable_cascade.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -145,16 +145,16 @@ def __init__( timestep_ratio_embedding_dim: int = 64, patch_size: int = 1, conditioning_dim: int = 2048, - block_out_channels: Tuple[int] = (2048, 2048), - num_attention_heads: Tuple[int] = (32, 32), - down_num_layers_per_block: Tuple[int] = (8, 24), - up_num_layers_per_block: Tuple[int] = (24, 8), - down_blocks_repeat_mappers: Optional[Tuple[int]] = ( + block_out_channels: tuple[int] = (2048, 2048), + num_attention_heads: tuple[int] = (32, 32), + down_num_layers_per_block: tuple[int] = (8, 24), + up_num_layers_per_block: tuple[int] = (24, 8), + down_blocks_repeat_mappers: Optional[tuple[int]] = ( 1, 1, ), - up_blocks_repeat_mappers: Optional[Tuple[int]] = (1, 1), - block_types_per_layer: Tuple[Tuple[str]] = ( + up_blocks_repeat_mappers: Optional[tuple[int]] = (1, 1), + block_types_per_layer: tuple[tuple[str]] = ( ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), ), @@ -165,10 +165,10 @@ def __init__( effnet_in_channels: Optional[int] = None, pixel_mapper_in_channels: Optional[int] = None, kernel_size=3, - dropout: Union[float, Tuple[float]] = (0.1, 0.1), - self_attn: Union[bool, Tuple[bool]] = True, - timestep_conditioning_type: Tuple[str] = ("sca", "crp"), - switch_level: Optional[Tuple[bool]] = None, + dropout: float | tuple[float] = (0.1, 0.1), + self_attn: bool | tuple[bool] = True, + timestep_conditioning_type: tuple[str] = ("sca", "crp"), + switch_level: Optional[tuple[bool]] = None, ): """ @@ -183,20 +183,20 @@ def __init__( Patch size to use for pixel unshuffling layer conditioning_dim (`int`, defaults to 2048): Dimension of the image and text conditional embedding. - block_out_channels (Tuple[int], defaults to (2048, 2048)): - Tuple of output channels for each block. - num_attention_heads (Tuple[int], defaults to (32, 32)): + block_out_channels (tuple[int], defaults to (2048, 2048)): + tuple of output channels for each block. + num_attention_heads (tuple[int], defaults to (32, 32)): Number of attention heads in each attention block. Set to -1 to if block types in a layer do not have attention. - down_num_layers_per_block (Tuple[int], defaults to [8, 24]): + down_num_layers_per_block (tuple[int], defaults to [8, 24]): Number of layers in each down block. - up_num_layers_per_block (Tuple[int], defaults to [24, 8]): + up_num_layers_per_block (tuple[int], defaults to [24, 8]): Number of layers in each up block. - down_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]): + down_blocks_repeat_mappers (tuple[int], optional, defaults to [1, 1]): Number of 1x1 Convolutional layers to repeat in each down block. - up_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]): + up_blocks_repeat_mappers (tuple[int], optional, defaults to [1, 1]): Number of 1x1 Convolutional layers to repeat in each up block. - block_types_per_layer (Tuple[Tuple[str]], optional, + block_types_per_layer (tuple[tuple[str]], optional, defaults to ( ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock") @@ -214,14 +214,14 @@ def __init__( Number of input channels for pixel mapper conditioning. kernel_size (`int`, *optional*, defaults to 3): Kernel size to use in the block convolutional layers. - dropout (Tuple[float], *optional*, defaults to (0.1, 0.1)): + dropout (tuple[float], *optional*, defaults to (0.1, 0.1)): Dropout to use per block. - self_attn (Union[bool, Tuple[bool]]): - Tuple of booleans that determine whether to use self attention in a block or not. - timestep_conditioning_type (Tuple[str], defaults to ("sca", "crp")): + self_attn (Union[bool, tuple[bool]]): + tuple of booleans that determine whether to use self attention in a block or not. + timestep_conditioning_type (tuple[str], defaults to ("sca", "crp")): Timestep conditioning type. - switch_level (Optional[Tuple[bool]], *optional*, defaults to `None`): - Tuple that indicates whether upsampling or downsampling should be applied in a block + switch_level (Optional[tuple[bool]], *optional*, defaults to `None`): + tuple that indicates whether upsampling or downsampling should be applied in a block """ super().__init__() diff --git a/src/diffusers/models/unets/uvit_2d.py b/src/diffusers/models/unets/uvit_2d.py index 94b39c84f055..e7b6559562f3 100644 --- a/src/diffusers/models/unets/uvit_2d.py +++ b/src/diffusers/models/unets/uvit_2d.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Union import torch import torch.nn.functional as F @@ -211,7 +210,7 @@ def layer_(*args): @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -220,7 +219,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -235,7 +234,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. diff --git a/src/diffusers/models/upsampling.py b/src/diffusers/models/upsampling.py index 8a47c69f1264..26bbea60adbe 100644 --- a/src/diffusers/models/upsampling.py +++ b/src/diffusers/models/upsampling.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Optional import torch import torch.nn as nn @@ -211,7 +211,7 @@ def __init__( channels: Optional[int] = None, out_channels: Optional[int] = None, use_conv: bool = False, - fir_kernel: Tuple[int, int, int, int] = (1, 3, 3, 1), + fir_kernel: tuple[int, int, int, int] = (1, 3, 3, 1), ): super().__init__() out_channels = out_channels if out_channels else channels @@ -425,7 +425,7 @@ def upfirdn2d_native( kernel: torch.Tensor, up: int = 1, down: int = 1, - pad: Tuple[int, int] = (0, 0), + pad: tuple[int, int] = (0, 0), ) -> torch.Tensor: up_x = up_y = up down_x = down_y = down diff --git a/src/diffusers/models/vae_flax.py b/src/diffusers/models/vae_flax.py index 13653b90372a..6b920d20e004 100644 --- a/src/diffusers/models/vae_flax.py +++ b/src/diffusers/models/vae_flax.py @@ -16,7 +16,6 @@ import math from functools import partial -from typing import Tuple import flax import flax.linen as nn @@ -514,10 +513,10 @@ class FlaxEncoder(nn.Module): Input channels out_channels (:obj:`int`, *optional*, defaults to 3): Output channels - down_block_types (:obj:`Tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): + down_block_types (:obj:`tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): DownEncoder block type - block_out_channels (:obj:`Tuple[str]`, *optional*, defaults to `(64,)`): - Tuple containing the number of output channels for each block + block_out_channels (:obj:`tuple[str]`, *optional*, defaults to `(64,)`): + tuple containing the number of output channels for each block layers_per_block (:obj:`int`, *optional*, defaults to `2`): Number of Resnet layer for each block norm_num_groups (:obj:`int`, *optional*, defaults to `32`): @@ -532,8 +531,8 @@ class FlaxEncoder(nn.Module): in_channels: int = 3 out_channels: int = 3 - down_block_types: Tuple[str] = ("DownEncoderBlock2D",) - block_out_channels: Tuple[int] = (64,) + down_block_types: tuple[str] = ("DownEncoderBlock2D",) + block_out_channels: tuple[int] = (64,) layers_per_block: int = 2 norm_num_groups: int = 32 act_fn: str = "silu" @@ -632,10 +631,10 @@ class FlaxDecoder(nn.Module): Input channels out_channels (:obj:`int`, *optional*, defaults to 3): Output channels - up_block_types (:obj:`Tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): + up_block_types (:obj:`tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): UpDecoder block type - block_out_channels (:obj:`Tuple[str]`, *optional*, defaults to `(64,)`): - Tuple containing the number of output channels for each block + block_out_channels (:obj:`tuple[str]`, *optional*, defaults to `(64,)`): + tuple containing the number of output channels for each block layers_per_block (:obj:`int`, *optional*, defaults to `2`): Number of Resnet layer for each block norm_num_groups (:obj:`int`, *optional*, defaults to `32`): @@ -650,7 +649,7 @@ class FlaxDecoder(nn.Module): in_channels: int = 3 out_channels: int = 3 - up_block_types: Tuple[str] = ("UpDecoderBlock2D",) + up_block_types: tuple[str] = ("UpDecoderBlock2D",) block_out_channels: int = (64,) layers_per_block: int = 2 norm_num_groups: int = 32 @@ -794,12 +793,12 @@ class FlaxAutoencoderKL(nn.Module, FlaxModelMixin, ConfigMixin): Number of channels in the input image. out_channels (`int`, *optional*, defaults to 3): Number of channels in the output. - down_block_types (`Tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): - Tuple of downsample block types. - up_block_types (`Tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): - Tuple of upsample block types. - block_out_channels (`Tuple[str]`, *optional*, defaults to `(64,)`): - Tuple of block output channels. + down_block_types (`tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): + tuple of downsample block types. + up_block_types (`tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): + tuple of upsample block types. + block_out_channels (`tuple[str]`, *optional*, defaults to `(64,)`): + tuple of block output channels. layers_per_block (`int`, *optional*, defaults to `2`): Number of ResNet layer for each block. act_fn (`str`, *optional*, defaults to `silu`): @@ -823,9 +822,9 @@ class FlaxAutoencoderKL(nn.Module, FlaxModelMixin, ConfigMixin): in_channels: int = 3 out_channels: int = 3 - down_block_types: Tuple[str] = ("DownEncoderBlock2D",) - up_block_types: Tuple[str] = ("UpDecoderBlock2D",) - block_out_channels: Tuple[int] = (64,) + down_block_types: tuple[str] = ("DownEncoderBlock2D",) + up_block_types: tuple[str] = ("UpDecoderBlock2D",) + block_out_channels: tuple[int] = (64,) layers_per_block: int = 1 act_fn: str = "silu" latent_channels: int = 4 diff --git a/src/diffusers/modular_pipelines/components_manager.py b/src/diffusers/modular_pipelines/components_manager.py index 9dd8035c44e7..58edfabb1119 100644 --- a/src/diffusers/modular_pipelines/components_manager.py +++ b/src/diffusers/modular_pipelines/components_manager.py @@ -16,7 +16,7 @@ import time from collections import OrderedDict from itertools import combinations -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional import torch @@ -53,8 +53,8 @@ class CustomOffloadHook(ModelHook): def __init__( self, - execution_device: Optional[Union[str, int, torch.device]] = None, - other_hooks: Optional[List["UserCustomOffloadHook"]] = None, + execution_device: Optional[str | int | torch.device] = None, + other_hooks: Optional[list["UserCustomOffloadHook"]] = None, offload_strategy: Optional["AutoOffloadStrategy"] = None, ): self.execution_device = execution_device if execution_device is not None else PartialState().default_device @@ -135,7 +135,7 @@ def add_other_hook(self, hook: "UserCustomOffloadHook"): def custom_offload_with_hook( model_id: str, model: torch.nn.Module, - execution_device: Union[str, int, torch.device] = None, + execution_device: str | int | torch.device = None, offload_strategy: Optional["AutoOffloadStrategy"] = None, ): hook = CustomOffloadHook(execution_device=execution_device, offload_strategy=offload_strategy) @@ -219,7 +219,7 @@ def search_best_candidate(module_sizes, min_memory_offload): # utils for display component info in a readable format # TODO: move to a different file -def summarize_dict_by_value_and_parts(d: Dict[str, Any]) -> Dict[str, Any]: +def summarize_dict_by_value_and_parts(d: dict[str, Any]) -> dict[str, Any]: """Summarizes a dictionary by finding common prefixes that share the same value. For a dictionary with dot-separated keys like: { @@ -240,7 +240,7 @@ def summarize_dict_by_value_and_parts(d: Dict[str, Any]) -> Dict[str, Any]: value_to_keys[value_tuple] = [] value_to_keys[value_tuple].append(key) - def find_common_prefix(keys: List[str]) -> str: + def find_common_prefix(keys: list[str]) -> str: """Find the shortest common prefix among a list of dot-separated keys.""" if not keys: return "" @@ -679,7 +679,7 @@ def matches_pattern(component_id, pattern, exact_match=False): return get_return_dict(matches, return_dict_with_names) - def enable_auto_cpu_offload(self, device: Union[str, int, torch.device] = None, memory_reserve_margin="3GB"): + def enable_auto_cpu_offload(self, device: str | int | torch.device = None, memory_reserve_margin="3GB"): """ Enable automatic CPU offloading for all components. @@ -746,13 +746,13 @@ def disable_auto_cpu_offload(self): def get_model_info( self, component_id: str, - fields: Optional[Union[str, List[str]]] = None, - ) -> Optional[Dict[str, Any]]: + fields: Optional[str | list[str]] = None, + ) -> Optional[dict[str, Any]]: """Get comprehensive information about a component. Args: component_id (str): Name of the component to get info for - fields (Optional[Union[str, List[str]]]): + fields (Optional[Union[str, list[str]]]): Field(s) to return. Can be a string for single field or list of fields. If None, uses the available_info_fields setting. @@ -1002,16 +1002,16 @@ def get_one( return next(iter(results.values())) - def get_ids(self, names: Union[str, List[str]] = None, collection: Optional[str] = None): + def get_ids(self, names: str | list[str] = None, collection: Optional[str] = None): """ Get component IDs by a list of names, optionally filtered by collection. Args: - names (Union[str, List[str]]): List of component names + names (Union[str, list[str]]): list of component names collection (Optional[str]): Optional collection to filter by Returns: - List[str]: List of component IDs + list[str]: list of component IDs """ ids = set() if not isinstance(names, list): @@ -1020,18 +1020,18 @@ def get_ids(self, names: Union[str, List[str]] = None, collection: Optional[str] ids.update(self._lookup_ids(name=name, collection=collection)) return list(ids) - def get_components_by_ids(self, ids: List[str], return_dict_with_names: Optional[bool] = True): + def get_components_by_ids(self, ids: list[str], return_dict_with_names: Optional[bool] = True): """ Get components by a list of IDs. Args: - ids (List[str]): - List of component IDs + ids (list[str]): + list of component IDs return_dict_with_names (Optional[bool]): Whether to return a dictionary with component names as keys: Returns: - Dict[str, Any]: Dictionary of components. + dict[str, Any]: Dictionary of components. - If return_dict_with_names=True, keys are component names. - If return_dict_with_names=False, keys are component IDs. @@ -1053,16 +1053,16 @@ def get_components_by_ids(self, ids: List[str], return_dict_with_names: Optional else: return components - def get_components_by_names(self, names: List[str], collection: Optional[str] = None): + def get_components_by_names(self, names: list[str], collection: Optional[str] = None): """ Get components by a list of names, optionally filtered by collection. Args: - names (List[str]): List of component names + names (list[str]): list of component names collection (Optional[str]): Optional collection to filter by Returns: - Dict[str, Any]: Dictionary of components with component names as keys + dict[str, Any]: Dictionary of components with component names as keys Raises: ValueError: If duplicate component names are found in the search results diff --git a/src/diffusers/modular_pipelines/flux/before_denoise.py b/src/diffusers/modular_pipelines/flux/before_denoise.py index c098b7d4f1e5..b45ced1f8d52 100644 --- a/src/diffusers/modular_pipelines/flux/before_denoise.py +++ b/src/diffusers/modular_pipelines/flux/before_denoise.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Union +from typing import Optional import numpy as np import torch @@ -34,9 +34,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -51,15 +51,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -156,7 +156,7 @@ class FluxSetTimestepsStep(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)] @property @@ -164,7 +164,7 @@ def description(self) -> str: return "Step that sets the scheduler's timesteps for inference" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_inference_steps", default=50), InputParam("timesteps"), @@ -183,7 +183,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("timesteps", type_hint=torch.Tensor, description="The timesteps to use for inference"), OutputParam( @@ -232,7 +232,7 @@ class FluxImg2ImgSetTimestepsStep(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)] @property @@ -240,7 +240,7 @@ def description(self) -> str: return "Step that sets the scheduler's timesteps for inference" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_inference_steps", default=50), InputParam("timesteps"), @@ -259,7 +259,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("timesteps", type_hint=torch.Tensor, description="The timesteps to use for inference"), OutputParam( @@ -322,7 +322,7 @@ class FluxPrepareLatentsStep(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [] @property @@ -330,7 +330,7 @@ def description(self) -> str: return "Prepare latents step that prepares the latents for the text-to-image generation process" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("height", type_hint=int), InputParam("width", type_hint=int), @@ -347,7 +347,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" @@ -431,11 +431,11 @@ def description(self) -> str: " `prepare_latents`. Both noise and image latents should already be patchified." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( name="latents", @@ -458,7 +458,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="initial_noise", @@ -507,7 +507,7 @@ def description(self) -> str: return "Step that prepares the RoPE inputs for the denoising process. Should be placed after text encoder and latent preparation steps." @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="height", required=True), InputParam(name="width", required=True), @@ -515,18 +515,18 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="txt_ids", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the prompt embeds, used for RoPE calculation.", ), OutputParam( name="img_ids", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the image latents, used for RoPE calculation.", ), ] @@ -557,7 +557,7 @@ def description(self) -> str: return "Step that prepares the RoPE inputs for the denoising process of Flux Kontext. Should be placed after text encoder and latent preparation steps." @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="image_height"), InputParam(name="image_width"), @@ -567,18 +567,18 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="txt_ids", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the prompt embeds, used for RoPE calculation.", ), OutputParam( name="img_ids", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the image latents, used for RoPE calculation.", ), ] diff --git a/src/diffusers/modular_pipelines/flux/decoders.py b/src/diffusers/modular_pipelines/flux/decoders.py index 846549b1a376..5da861e78fcb 100644 --- a/src/diffusers/modular_pipelines/flux/decoders.py +++ b/src/diffusers/modular_pipelines/flux/decoders.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Tuple, Union +from typing import Any import numpy as np import PIL @@ -49,7 +49,7 @@ class FluxDecodeStep(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKL), ComponentSpec( @@ -65,7 +65,7 @@ def description(self) -> str: return "Step that decodes the denoised latents into images" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("output_type", default="pil"), InputParam("height", default=1024), @@ -79,11 +79,11 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "images", - type_hint=Union[List[PIL.Image.Image], torch.Tensor, np.ndarray], + type_hint=list[PIL.Image.Image] | torch.Tensor | np.ndarray, description="The generated images, can be a list of PIL.Image.Image, torch.Tensor or a numpy array", ) ] diff --git a/src/diffusers/modular_pipelines/flux/denoise.py b/src/diffusers/modular_pipelines/flux/denoise.py index b1796bb63cb0..f6741f3f2158 100644 --- a/src/diffusers/modular_pipelines/flux/denoise.py +++ b/src/diffusers/modular_pipelines/flux/denoise.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Tuple +from typing import Any import torch @@ -36,7 +36,7 @@ class FluxLoopDenoiser(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("transformer", FluxTransformer2DModel)] @property @@ -48,7 +48,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("joint_attention_kwargs"), InputParam( @@ -113,7 +113,7 @@ class FluxKontextLoopDenoiser(ModularPipelineBlocks): model_name = "flux-kontext" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("transformer", FluxTransformer2DModel)] @property @@ -125,7 +125,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("joint_attention_kwargs"), InputParam( @@ -203,7 +203,7 @@ class FluxLoopAfterDenoiser(ModularPipelineBlocks): model_name = "flux" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)] @property @@ -215,15 +215,15 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [] @property - def intermediate_inputs(self) -> List[str]: + def intermediate_inputs(self) -> list[str]: return [InputParam("generator")] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents")] @torch.no_grad() @@ -254,14 +254,14 @@ def description(self) -> str: ) @property - def loop_expected_components(self) -> List[ComponentSpec]: + def loop_expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ComponentSpec("transformer", FluxTransformer2DModel), ] @property - def loop_inputs(self) -> List[InputParam]: + def loop_inputs(self) -> list[InputParam]: return [ InputParam( "timesteps", diff --git a/src/diffusers/modular_pipelines/flux/encoders.py b/src/diffusers/modular_pipelines/flux/encoders.py index b71962bd9313..014d883c9479 100644 --- a/src/diffusers/modular_pipelines/flux/encoders.py +++ b/src/diffusers/modular_pipelines/flux/encoders.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import List, Optional, Union +from typing import Optional import regex as re import torch @@ -90,7 +90,7 @@ def description(self) -> str: return "Image Preprocess step." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", @@ -101,11 +101,11 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [InputParam("resized_image"), InputParam("image"), InputParam("height"), InputParam("width")] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam(name="processed_image")] @staticmethod @@ -155,7 +155,7 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", @@ -166,11 +166,11 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [InputParam("image")] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam(name="processed_image")] @torch.no_grad() @@ -244,17 +244,17 @@ def description(self) -> str: return f"Dynamic VAE Encoder step that converts {self._image_input_name} into latent representations {self._image_latents_output_name}.\n" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: components = [ComponentSpec("vae", AutoencoderKL)] return components @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [InputParam(self._image_input_name), InputParam("generator")] return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( self._image_latents_output_name, @@ -294,7 +294,7 @@ def description(self) -> str: return "Text Encoder step that generate text_embeddings to guide the image generation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", CLIPTextModel), ComponentSpec("tokenizer", CLIPTokenizer), @@ -303,7 +303,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("prompt"), InputParam("prompt_2"), @@ -312,7 +312,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "prompt_embeds", @@ -335,9 +335,7 @@ def check_inputs(block_state): raise ValueError(f"`prompt` or `prompt_2` has to be of type `str` or `list` but is {type(prompt)}") @staticmethod - def _get_t5_prompt_embeds( - components, prompt: Union[str, List[str]], max_sequence_length: int, device: torch.device - ): + def _get_t5_prompt_embeds(components, prompt: str | list[str], max_sequence_length: int, device: torch.device): dtype = components.text_encoder_2.dtype prompt = [prompt] if isinstance(prompt, str) else prompt @@ -368,7 +366,7 @@ def _get_t5_prompt_embeds( return prompt_embeds @staticmethod - def _get_clip_prompt_embeds(components, prompt: Union[str, List[str]], device: torch.device): + def _get_clip_prompt_embeds(components, prompt: str | list[str], device: torch.device): prompt = [prompt] if isinstance(prompt, str) else prompt if isinstance(components, TextualInversionLoaderMixin): @@ -404,8 +402,8 @@ def _get_clip_prompt_embeds(components, prompt: Union[str, List[str]], device: t @staticmethod def encode_prompt( components, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], device: Optional[torch.device] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, diff --git a/src/diffusers/modular_pipelines/flux/inputs.py b/src/diffusers/modular_pipelines/flux/inputs.py index e1bc17f5ff4e..77ecd8451684 100644 --- a/src/diffusers/modular_pipelines/flux/inputs.py +++ b/src/diffusers/modular_pipelines/flux/inputs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List import torch @@ -42,7 +41,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_images_per_prompt", default=1), InputParam( @@ -62,7 +61,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "batch_size", @@ -123,8 +122,8 @@ class FluxInputsDynamicStep(ModularPipelineBlocks): def __init__( self, - image_latent_inputs: List[str] = ["image_latents"], - additional_batch_inputs: List[str] = [], + image_latent_inputs: list[str] = ["image_latents"], + additional_batch_inputs: list[str] = [], ): if not isinstance(image_latent_inputs, list): image_latent_inputs = [image_latent_inputs] @@ -159,7 +158,7 @@ def description(self) -> str: return summary_section + inputs_info + placement_section @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [ InputParam(name="num_images_per_prompt", default=1), InputParam(name="batch_size", required=True), @@ -178,7 +177,7 @@ def inputs(self) -> List[InputParam]: return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam(name="image_height", type_hint=int, description="The height of the image latents"), OutputParam(name="image_width", type_hint=int, description="The width of the image latents"), @@ -307,7 +306,7 @@ def description(self): ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [ InputParam(name="height"), InputParam(name="width"), @@ -316,7 +315,7 @@ def inputs(self) -> List[InputParam]: return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam(name="height", type_hint=int, description="The height of the initial noisy latents"), OutputParam(name="width", type_hint=int, description="The width of the initial noisy latents"), diff --git a/src/diffusers/modular_pipelines/mellon_node_utils.py b/src/diffusers/modular_pipelines/mellon_node_utils.py index a405aebee221..bb18b62dccc8 100644 --- a/src/diffusers/modular_pipelines/mellon_node_utils.py +++ b/src/diffusers/modular_pipelines/mellon_node_utils.py @@ -4,7 +4,7 @@ # Simple typed wrapper for parameter overrides from dataclasses import asdict, dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional from huggingface_hub import create_repo, hf_hub_download from huggingface_hub.utils import ( @@ -323,12 +323,12 @@ class MellonParam: step: Optional[float] = None options: Any = None value: Any = None - fieldOptions: Optional[Dict[str, Any]] = None + fieldOptions: Optional[dict[str, Any]] = None onChange: Any = None onSignal: Any = None _map_to_input: Any = None # the block input name this parameter maps to - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: data = asdict(self) return {k: v for k, v in data.items() if not k.startswith("_") and v is not None} @@ -345,9 +345,9 @@ class MellonNodeConfig(PushToHubMixin): """ - inputs: List[Union[str, MellonParam]] - model_inputs: List[Union[str, MellonParam]] - outputs: List[Union[str, MellonParam]] + inputs: list[str | MellonParam] + model_inputs: list[str | MellonParam] + outputs: list[str | MellonParam] blocks_names: list[str] node_type: str config_name = "mellon_config.json" @@ -362,11 +362,11 @@ def __post_init__(self): @staticmethod def _resolve_params_list( - params: List[Union[str, MellonParam]], default_map: Dict[str, Dict[str, Any]] - ) -> Dict[str, Dict[str, Any]]: + params: list[str | MellonParam], default_map: dict[str, dict[str, Any]] + ) -> dict[str, dict[str, Any]]: def _resolve_param( - param: Union[str, MellonParam], default_params_map: Dict[str, Dict[str, Any]] - ) -> Tuple[str, Dict[str, Any]]: + param: str | MellonParam, default_params_map: dict[str, dict[str, Any]] + ) -> tuple[str, dict[str, Any]]: if isinstance(param, str): if param not in default_params_map: raise ValueError(f"Unknown param '{param}', please define a `MellonParam` object instead") @@ -393,11 +393,11 @@ def _resolve_param( @validate_hf_hub_args def load_mellon_config( cls, - pretrained_model_name_or_path: Union[str, os.PathLike], + pretrained_model_name_or_path: str | os.PathLike, return_unused_kwargs=False, return_commit_hash=False, **kwargs, - ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + ) -> tuple[dict[str, Any], dict[str, Any]]: r""" Load a model or scheduler configuration. @@ -416,7 +416,7 @@ def load_mellon_config( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -541,7 +541,7 @@ def load_mellon_config( return outputs - def save_mellon_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_mellon_config(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save the Mellon node definition to a JSON file. @@ -552,7 +552,7 @@ def save_mellon_config(self, save_directory: Union[str, os.PathLike], push_to_hu Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -584,7 +584,7 @@ def save_mellon_config(self, save_directory: Union[str, os.PathLike], push_to_hu subfolder=subfolder, ) - def to_json_file(self, json_file_path: Union[str, os.PathLike]): + def to_json_file(self, json_file_path: str | os.PathLike): """ Save the Mellon schema dictionary to a JSON file. @@ -607,7 +607,7 @@ def to_json_string(self) -> str: mellon_dict = self.to_mellon_dict() return json.dumps(mellon_dict, indent=2, sort_keys=True) + "\n" - def to_mellon_dict(self) -> Dict[str, Any]: + def to_mellon_dict(self) -> dict[str, Any]: """Return a JSON-serializable dict focusing on the Mellon schema fields only. params is a single flat dict composed as: {**inputs, **model_inputs, **outputs}. @@ -625,7 +625,7 @@ def to_mellon_dict(self) -> Dict[str, Any]: } @classmethod - def from_mellon_dict(cls, mellon_dict: Dict[str, Any]) -> "MellonNodeConfig": + def from_mellon_dict(cls, mellon_dict: dict[str, Any]) -> "MellonNodeConfig": """Create a config from a Mellon schema dict produced by to_mellon_dict(). Splits the flat params dict back into inputs/model_inputs/outputs using the known key spaces from @@ -634,9 +634,9 @@ def from_mellon_dict(cls, mellon_dict: Dict[str, Any]) -> "MellonNodeConfig": """ flat_params = mellon_dict.get("params", {}) - inputs: Dict[str, Any] = {} - model_inputs: Dict[str, Any] = {} - outputs: Dict[str, Any] = {} + inputs: dict[str, Any] = {} + model_inputs: dict[str, Any] = {} + outputs: dict[str, Any] = {} for param_name, param_dict in flat_params.items(): if param_dict.get("display", "") == "output": @@ -667,9 +667,9 @@ def from_blocks(cls, blocks: ModularPipelineBlocks, node_type: str) -> "MellonNo blocks_names = list(blocks.sub_blocks.keys()) default_node_config = NODE_TYPE_PARAMS_MAP[node_type] - inputs_list: List[Union[str, MellonParam]] = default_node_config.get("inputs", []) - model_inputs_list: List[Union[str, MellonParam]] = default_node_config.get("model_inputs", []) - outputs_list: List[Union[str, MellonParam]] = default_node_config.get("outputs", []) + inputs_list: list[str | MellonParam] = default_node_config.get("inputs", []) + model_inputs_list: list[str | MellonParam] = default_node_config.get("model_inputs", []) + outputs_list: list[str | MellonParam] = default_node_config.get("outputs", []) for required_input_name in blocks.required_inputs: if required_input_name not in inputs_list: @@ -707,7 +707,7 @@ def __init__(self): self._registry = {} self._initialized = False - def register(self, pipeline_cls: type, node_params: Dict[str, MellonNodeConfig]): + def register(self, pipeline_cls: type, node_params: dict[str, MellonNodeConfig]): if not self._initialized: _initialize_registry(self) self._registry[pipeline_cls] = node_params @@ -717,14 +717,14 @@ def get(self, pipeline_cls: type) -> MellonNodeConfig: _initialize_registry(self) return self._registry.get(pipeline_cls, None) - def get_all(self) -> Dict[type, Dict[str, MellonNodeConfig]]: + def get_all(self) -> dict[type, dict[str, MellonNodeConfig]]: if not self._initialized: _initialize_registry(self) return self._registry def _register_preset_node_types( - pipeline_cls, params_map: Dict[str, Dict[str, Any]], registry: ModularMellonNodeRegistry + pipeline_cls, params_map: dict[str, dict[str, Any]], registry: ModularMellonNodeRegistry ): """Register all node-type presets for a given pipeline class from a params map.""" node_configs = {} diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index cfbca48a981a..7dc37eda32ac 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -19,7 +19,7 @@ from collections import OrderedDict from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch from huggingface_hub import create_repo @@ -71,8 +71,8 @@ class PipelineState: [`PipelineState`] stores the state of a pipeline. It is used to pass data between pipeline blocks. """ - values: Dict[str, Any] = field(default_factory=dict) - kwargs_mapping: Dict[str, List[str]] = field(default_factory=dict) + values: dict[str, Any] = field(default_factory=dict) + kwargs_mapping: dict[str, list[str]] = field(default_factory=dict) def set(self, key: str, value: Any, kwargs_type: str = None): """ @@ -91,22 +91,22 @@ def set(self, key: str, value: Any, kwargs_type: str = None): else: self.kwargs_mapping[kwargs_type].append(key) - def get(self, keys: Union[str, List[str]], default: Any = None) -> Union[Any, Dict[str, Any]]: + def get(self, keys: str | list[str], default: Any = None) -> Any | dict[str, Any]: """ Get one or multiple values from the pipeline state. Args: - keys (Union[str, List[str]]): Key or list of keys for the values + keys (Union[str, list[str]]): Key or list of keys for the values default (Any): The default value to return if not found Returns: - Union[Any, Dict[str, Any]]: Single value if keys is str, dictionary of values if keys is list + Union[Any, dict[str, Any]]: Single value if keys is str, dictionary of values if keys is list """ if isinstance(keys, str): return self.values.get(keys, default) return {key: self.values.get(key, default) for key in keys} - def get_by_kwargs(self, kwargs_type: str) -> Dict[str, Any]: + def get_by_kwargs(self, kwargs_type: str) -> dict[str, Any]: """ Get all values with matching kwargs_type. @@ -114,12 +114,12 @@ def get_by_kwargs(self, kwargs_type: str) -> Dict[str, Any]: kwargs_type (str): The kwargs_type to filter by Returns: - Dict[str, Any]: Dictionary of values with matching kwargs_type + dict[str, Any]: Dictionary of values with matching kwargs_type """ value_names = self.kwargs_mapping.get(kwargs_type, []) return self.get(value_names) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Convert PipelineState to a dictionary. """ @@ -172,7 +172,7 @@ def as_dict(self): Convert BlockState to a dictionary. Returns: - Dict[str, Any]: Dictionary containing all attributes of the BlockState + dict[str, Any]: Dictionary containing all attributes of the BlockState """ return dict(self.__dict__.items()) @@ -186,14 +186,14 @@ def format_value(v): elif isinstance(v, list): if len(v) > 0 and hasattr(v[0], "shape") and hasattr(v[0], "dtype"): shapes = [t.shape for t in v] - return f"List[{len(v)}] of Tensors with shapes {shapes}" + return f"list[{len(v)}] of Tensors with shapes {shapes}" return repr(v) # Handle tuples of tensors elif isinstance(v, tuple): if len(v) > 0 and hasattr(v[0], "shape") and hasattr(v[0], "dtype"): shapes = [t.shape for t in v] - return f"Tuple[{len(v)}] of Tensors with shapes {shapes}" + return f"tuple[{len(v)}] of Tensors with shapes {shapes}" return repr(v) # Handle dicts with tensor values @@ -209,7 +209,7 @@ def format_value(v): and hasattr(val[0], "dtype") ): shapes = [t.shape for t in val] - formatted_dict[k] = f"List[{len(val)}] of Tensors with shapes {shapes}" + formatted_dict[k] = f"list[{len(val)}] of Tensors with shapes {shapes}" else: formatted_dict[k] = repr(val) return formatted_dict @@ -252,16 +252,16 @@ def description(self) -> str: return "" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [] @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [] @property - def inputs(self) -> List[InputParam]: - """List of input parameters. Must be implemented by subclasses.""" + def inputs(self) -> list[InputParam]: + """list of input parameters. Must be implemented by subclasses.""" return [] def _get_required_inputs(self): @@ -273,19 +273,19 @@ def _get_required_inputs(self): return input_names @property - def required_inputs(self) -> List[InputParam]: + def required_inputs(self) -> list[InputParam]: return self._get_required_inputs() @property - def intermediate_outputs(self) -> List[OutputParam]: - """List of intermediate output parameters. Must be implemented by subclasses.""" + def intermediate_outputs(self) -> list[OutputParam]: + """list of intermediate output parameters. Must be implemented by subclasses.""" return [] def _get_outputs(self): return self.intermediate_outputs @property - def outputs(self) -> List[OutputParam]: + def outputs(self) -> list[OutputParam]: return self._get_outputs() @classmethod @@ -350,7 +350,7 @@ def save_pretrained(self, save_directory, push_to_hub=False, **kwargs): def init_pipeline( self, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: Optional[str | os.PathLike] = None, components_manager: Optional[ComponentsManager] = None, collection: Optional[str] = None, ) -> "ModularPipeline": @@ -427,17 +427,17 @@ def set_block_state(self, state: PipelineState, block_state: BlockState): state.set(param_name, param, input_param.kwargs_type) @staticmethod - def combine_inputs(*named_input_lists: List[Tuple[str, List[InputParam]]]) -> List[InputParam]: + def combine_inputs(*named_input_lists: list[tuple[str, list[InputParam]]]) -> list[InputParam]: """ Combines multiple lists of InputParam objects from different blocks. For duplicate inputs, updates only if current default value is None and new default value is not None. Warns if multiple non-None default values exist for the same input. Args: - named_input_lists: List of tuples containing (block_name, input_param_list) pairs + named_input_lists: list of tuples containing (block_name, input_param_list) pairs Returns: - List[InputParam]: Combined list of unique InputParam objects + list[InputParam]: Combined list of unique InputParam objects """ combined_dict = {} # name -> InputParam value_sources = {} # name -> block_name @@ -470,16 +470,16 @@ def combine_inputs(*named_input_lists: List[Tuple[str, List[InputParam]]]) -> Li return list(combined_dict.values()) @staticmethod - def combine_outputs(*named_output_lists: List[Tuple[str, List[OutputParam]]]) -> List[OutputParam]: + def combine_outputs(*named_output_lists: list[tuple[str, list[OutputParam]]]) -> list[OutputParam]: """ Combines multiple lists of OutputParam objects from different blocks. For duplicate outputs, keeps the first occurrence of each output name. Args: - named_output_lists: List of tuples containing (block_name, output_param_list) pairs + named_output_lists: list of tuples containing (block_name, output_param_list) pairs Returns: - List[OutputParam]: Combined list of unique OutputParam objects + list[OutputParam]: Combined list of unique OutputParam objects """ combined_dict = {} # name -> OutputParam @@ -493,15 +493,15 @@ def combine_outputs(*named_output_lists: List[Tuple[str, List[OutputParam]]]) -> return list(combined_dict.values()) @property - def input_names(self) -> List[str]: + def input_names(self) -> list[str]: return [input_param.name for input_param in self.inputs] @property - def intermediate_output_names(self) -> List[str]: + def intermediate_output_names(self) -> list[str]: return [output_param.name for output_param in self.intermediate_outputs] @property - def output_names(self) -> List[str]: + def output_names(self) -> list[str]: return [output_param.name for output_param in self.outputs] @property @@ -526,9 +526,9 @@ class AutoPipelineBlocks(ModularPipelineBlocks): > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: - block_classes: List of block classes to be used - block_names: List of prefixes for each block - block_trigger_inputs: List of input names that trigger specific blocks, with None for default + block_classes: list of block classes to be used + block_names: list of prefixes for each block + block_trigger_inputs: list of input names that trigger specific blocks, with None for default """ block_classes = [] @@ -590,7 +590,7 @@ def expected_configs(self): return expected_configs @property - def required_inputs(self) -> List[str]: + def required_inputs(self) -> list[str]: if None not in self.block_trigger_inputs: return [] first_block = next(iter(self.sub_blocks.values())) @@ -605,7 +605,7 @@ def required_inputs(self) -> List[str]: # YiYi TODO: add test for this @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: named_inputs = [(name, block.inputs) for name, block in self.sub_blocks.items()] combined_inputs = self.combine_inputs(*named_inputs) # mark Required inputs only if that input is required by all the blocks @@ -617,13 +617,13 @@ def inputs(self) -> List[Tuple[str, Any]]: return combined_inputs @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = self.combine_outputs(*named_outputs) return combined_outputs @property - def outputs(self) -> List[str]: + def outputs(self) -> list[str]: named_outputs = [(name, block.outputs) for name, block in self.sub_blocks.items()] combined_outputs = self.combine_outputs(*named_outputs) return combined_outputs @@ -784,8 +784,8 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: - block_classes: List of block classes to be used - block_names: List of prefixes for each block + block_classes: list of block classes to be used + block_names: list of prefixes for each block """ block_classes = [] @@ -819,7 +819,7 @@ def expected_configs(self): @classmethod def from_blocks_dict( - cls, blocks_dict: Dict[str, Any], description: Optional[str] = None + cls, blocks_dict: dict[str, Any], description: Optional[str] = None ) -> "SequentialPipelineBlocks": """Creates a SequentialPipelineBlocks instance from a dictionary of blocks. @@ -882,11 +882,11 @@ def _get_inputs(self): # YiYi TODO: add test for this @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return self._get_inputs() @property - def required_inputs(self) -> List[str]: + def required_inputs(self) -> list[str]: # Get the first block from the dictionary first_block = next(iter(self.sub_blocks.values())) required_by_any = set(getattr(first_block, "required_inputs", set())) @@ -899,7 +899,7 @@ def required_inputs(self) -> List[str]: return list(required_by_any) @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: named_outputs = [] for name, block in self.sub_blocks.items(): inp_names = {inp.name for inp in block.inputs} @@ -912,7 +912,7 @@ def intermediate_outputs(self) -> List[str]: # YiYi TODO: I think we can remove the outputs property @property - def outputs(self) -> List[str]: + def outputs(self) -> list[str]: # return next(reversed(self.sub_blocks.values())).intermediate_outputs return self.intermediate_outputs @@ -1139,8 +1139,8 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: - block_classes: List of block classes to be used - block_names: List of prefixes for each block + block_classes: list of block classes to be used + block_names: list of prefixes for each block """ model_name = None @@ -1153,20 +1153,20 @@ def description(self) -> str: raise NotImplementedError("description method must be implemented in subclasses") @property - def loop_expected_components(self) -> List[ComponentSpec]: + def loop_expected_components(self) -> list[ComponentSpec]: return [] @property - def loop_expected_configs(self) -> List[ConfigSpec]: + def loop_expected_configs(self) -> list[ConfigSpec]: return [] @property - def loop_inputs(self) -> List[InputParam]: - """List of input parameters. Must be implemented by subclasses.""" + def loop_inputs(self) -> list[InputParam]: + """list of input parameters. Must be implemented by subclasses.""" return [] @property - def loop_required_inputs(self) -> List[str]: + def loop_required_inputs(self) -> list[str]: input_names = [] for input_param in self.loop_inputs: if input_param.required: @@ -1174,8 +1174,8 @@ def loop_required_inputs(self) -> List[str]: return input_names @property - def loop_intermediate_outputs(self) -> List[OutputParam]: - """List of intermediate output parameters. Must be implemented by subclasses.""" + def loop_intermediate_outputs(self) -> list[OutputParam]: + """list of intermediate output parameters. Must be implemented by subclasses.""" return [] # modified from SequentialPipelineBlocks to include loop_expected_components @@ -1240,7 +1240,7 @@ def inputs(self): # modified from SequentialPipelineBlocks, if any additionan input required by the loop is required by the block @property - def required_inputs(self) -> List[str]: + def required_inputs(self) -> list[str]: # Get the first block from the dictionary first_block = next(iter(self.sub_blocks.values())) required_by_any = set(getattr(first_block, "required_inputs", set())) @@ -1258,7 +1258,7 @@ def required_inputs(self) -> List[str]: # YiYi TODO: this need to be thought about more # modified from SequentialPipelineBlocks to include loop_intermediate_outputs @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = self.combine_outputs(*named_outputs) for output in self.loop_intermediate_outputs: @@ -1268,7 +1268,7 @@ def intermediate_outputs(self) -> List[str]: # YiYi TODO: this need to be thought about more @property - def outputs(self) -> List[str]: + def outputs(self) -> list[str]: return next(reversed(self.sub_blocks.values())).intermediate_outputs def __init__(self): @@ -1281,7 +1281,7 @@ def __init__(self): self.sub_blocks = sub_blocks @classmethod - def from_blocks_dict(cls, blocks_dict: Dict[str, Any]) -> "LoopSequentialPipelineBlocks": + def from_blocks_dict(cls, blocks_dict: dict[str, Any]) -> "LoopSequentialPipelineBlocks": """ Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. @@ -1433,7 +1433,7 @@ class ModularPipeline(ConfigMixin, PushToHubMixin): def __init__( self, blocks: Optional[ModularPipelineBlocks] = None, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: Optional[str | os.PathLike] = None, components_manager: Optional[ComponentsManager] = None, collection: Optional[str] = None, **kwargs, @@ -1586,7 +1586,7 @@ def __init__( self.register_to_config(_blocks_class_name=self.blocks.__class__.__name__ if self.blocks is not None else None) @property - def default_call_parameters(self) -> Dict[str, Any]: + def default_call_parameters(self) -> dict[str, Any]: """ Returns: - Dictionary mapping input names to their default values @@ -1600,7 +1600,7 @@ def default_call_parameters(self) -> Dict[str, Any]: @validate_hf_hub_args def from_pretrained( cls, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], + pretrained_model_name_or_path: Optional[str | os.PathLike], trust_remote_code: Optional[bool] = None, components_manager: Optional[ComponentsManager] = None, collection: Optional[str] = None, @@ -1690,7 +1690,7 @@ def from_pretrained( ) return pipeline - def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save the pipeline to a directory. It does not save components, you need to save them separately. @@ -1890,26 +1890,26 @@ def dtype(self) -> torch.dtype: return torch.float32 @property - def null_component_names(self) -> List[str]: + def null_component_names(self) -> list[str]: """ Returns: - - List of names for components that needs to be loaded + - list of names for components that needs to be loaded """ return [name for name in self._component_specs.keys() if hasattr(self, name) and getattr(self, name) is None] @property - def component_names(self) -> List[str]: + def component_names(self) -> list[str]: """ Returns: - - List of names for all components + - list of names for all components """ return list(self.components.keys()) @property - def pretrained_component_names(self) -> List[str]: + def pretrained_component_names(self) -> list[str]: """ Returns: - - List of names for from_pretrained components + - list of names for from_pretrained components """ return [ name @@ -1918,10 +1918,10 @@ def pretrained_component_names(self) -> List[str]: ] @property - def config_component_names(self) -> List[str]: + def config_component_names(self) -> list[str]: """ Returns: - - List of names for from_config components + - list of names for from_config components """ return [ name @@ -1930,7 +1930,7 @@ def config_component_names(self) -> List[str]: ] @property - def components(self) -> Dict[str, Any]: + def components(self) -> dict[str, Any]: """ Returns: - Dictionary mapping component names to their objects (include both from_pretrained and from_config @@ -2077,12 +2077,12 @@ def update_components(self, **kwargs): self.register_to_config(**config_to_register) # YiYi TODO: support map for additional from_pretrained kwargs - def load_components(self, names: Optional[Union[List[str], str]] = None, **kwargs): + def load_components(self, names: Optional[list[str] | str] = None, **kwargs): """ Load selected components from specs. Args: - names: List of component names to load. If None, will load all components with + names: list of component names to load. If None, will load all components with default_creation_method == "from_pretrained". If provided as a list or string, will load only the specified components. **kwargs: additional kwargs to be passed to `from_pretrained()`.Can be: @@ -2345,7 +2345,7 @@ def _component_spec_to_dict(component_spec: ComponentSpec) -> Any: the `default_creation_method` is not `from_pretrained`, return None. This dict contains: - - "type_hint": Tuple[str, str] + - "type_hint": tuple[str, str] Library name and class name of the component. (e.g. ("diffusers", "UNet2DConditionModel")) - All loading fields defined by `component_spec.loading_fields()`, typically: - "repo": Optional[str] @@ -2363,7 +2363,7 @@ def _component_spec_to_dict(component_spec: ComponentSpec) -> Any: The spec object describing one pipeline component. Returns: - Dict[str, Any]: A mapping suitable for JSON serialization. + dict[str, Any]: A mapping suitable for JSON serialization. Example: >>> from diffusers.pipelines.modular_pipeline_utils import ComponentSpec >>> from diffusers import @@ -2393,13 +2393,13 @@ def _component_spec_to_dict(component_spec: ComponentSpec) -> Any: @staticmethod def _dict_to_component_spec( name: str, - spec_dict: Dict[str, Any], + spec_dict: dict[str, Any], ) -> ComponentSpec: """ Reconstruct a ComponentSpec from a loading specdict. This method converts a dictionary representation back into a ComponentSpec object. The dict should contain: - - "type_hint": Tuple[str, str] + - "type_hint": tuple[str, str] Library name and class name of the component. (e.g. ("diffusers", "UNet2DConditionModel")) - All loading fields defined by `component_spec.loading_fields()`, typically: - "repo": Optional[str] @@ -2415,7 +2415,7 @@ def _dict_to_component_spec( Args: name (str): The name of the component. - specdict (Dict[str, Any]): + specdict (dict[str, Any]): A dictionary containing the component specification data. Returns: @@ -2450,7 +2450,7 @@ def set_progress_bar_config(self, **kwargs): if hasattr(sub_block, "set_progress_bar_config"): sub_block.set_progress_bar_config(**kwargs) - def __call__(self, state: PipelineState = None, output: Union[str, List[str]] = None, **kwargs): + def __call__(self, state: PipelineState = None, output: str | list[str] = None, **kwargs): """ Execute the pipeline by running the pipeline blocks with the given inputs. @@ -2458,11 +2458,11 @@ def __call__(self, state: PipelineState = None, output: Union[str, List[str]] = state (`PipelineState`, optional): PipelineState instance contains inputs and intermediate values. If None, a new `PipelineState` will be created based on the user inputs and the pipeline blocks's requirement. - output (`str` or `List[str]`, optional): + output (`str` or `list[str]`, optional): Optional specification of what to return: - None: Returns the complete `PipelineState` with all inputs and intermediates (default) - str: Returns a specific intermediate value from the state (e.g. `output="image"`) - - List[str]: Returns a dictionary of specific intermediate values (e.g. `output=["image", + - list[str]: Returns a dictionary of specific intermediate values (e.g. `output=["image", "latents"]`) @@ -2487,7 +2487,7 @@ def __call__(self, state: PipelineState = None, output: Union[str, List[str]] = Returns: - If `output` is None: Complete `PipelineState` containing all inputs and intermediates - If `output` is str: The specific intermediate value from the state (e.g. `output="image"`) - - If `output` is List[str]: Dictionary mapping output names to their values from the state (e.g. + - If `output` is list[str]: Dictionary mapping output names to their values from the state (e.g. `output=["image", "latents"]`) """ if state is None: diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index b15126868634..7e03968b1e3a 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -16,7 +16,7 @@ import re from collections import OrderedDict from dataclasses import dataclass, field, fields -from typing import Any, Dict, List, Literal, Optional, Type, Union +from typing import Any, Literal, Optional, Type, Union import torch @@ -92,7 +92,7 @@ class ComponentSpec: description: Optional[str] = None config: Optional[FrozenDict] = None # YiYi Notes: should we change it to pretrained_model_name_or_path for consistency? a bit long for a field name - repo: Optional[Union[str, List[str]]] = field(default=None, metadata={"loading": True}) + repo: Optional[str | list[str]] = field(default=None, metadata={"loading": True}) subfolder: Optional[str] = field(default="", metadata={"loading": True}) variant: Optional[str] = field(default=None, metadata={"loading": True}) revision: Optional[str] = field(default=None, metadata={"loading": True}) @@ -173,7 +173,7 @@ def from_component(cls, name: str, component: Any) -> Any: ) @classmethod - def loading_fields(cls) -> List[str]: + def loading_fields(cls) -> list[str]: """ Return the names of all loading‐related fields (i.e. those whose field.metadata["loading"] is True). """ @@ -192,7 +192,7 @@ def load_id(self) -> str: return "|".join(p for p in parts if p) @classmethod - def decode_load_id(cls, load_id: str) -> Dict[str, Optional[str]]: + def decode_load_id(cls, load_id: str) -> dict[str, Optional[str]]: """ Decode a load_id string back into a dictionary of loading fields and values. @@ -229,7 +229,7 @@ def decode_load_id(cls, load_id: str) -> Dict[str, Optional[str]]: # otherwise we cannot do spec -> spec.create() -> component -> ComponentSpec.from_component(component) # the config info is lost in the process # remove error check in from_component spec and ModularPipeline.update_components() if we remove support for non configmixin in `create()` method - def create(self, config: Optional[Union[FrozenDict, Dict[str, Any]]] = None, **kwargs) -> Any: + def create(self, config: Optional[FrozenDict | dict[str, Any]] = None, **kwargs) -> Any: """Create component using from_config with config.""" if self.type_hint is None or not isinstance(self.type_hint, type): @@ -343,7 +343,7 @@ def format_inputs_short(inputs): Format input parameters into a string representation, with required params first followed by optional ones. Args: - inputs: List of input parameters with 'required' and 'name' attributes, and 'default' for optional params + inputs: list of input parameters with 'required' and 'name' attributes, and 'default' for optional params Returns: str: Formatted string of input parameters @@ -372,9 +372,9 @@ def format_intermediates_short(intermediate_inputs, required_intermediate_inputs Formats intermediate inputs and outputs of a block into a string representation. Args: - intermediate_inputs: List of intermediate input parameters - required_intermediate_inputs: List of required intermediate input names - intermediate_outputs: List of intermediate output parameters + intermediate_inputs: list of intermediate input parameters + required_intermediate_inputs: list of required intermediate input names + intermediate_outputs: list of intermediate output parameters Returns: str: Formatted string like: @@ -421,7 +421,7 @@ def format_params(params, header="Args", indent_level=4, max_line_length=115): """Format a list of InputParam or OutputParam objects into a readable string representation. Args: - params: List of InputParam or OutputParam objects to format + params: list of InputParam or OutputParam objects to format header: Header text to use (e.g. "Args" or "Returns") indent_level: Number of spaces to indent each parameter line (default: 4) max_line_length: Maximum length for each line before wrapping (default: 115) @@ -499,7 +499,7 @@ def format_input_params(input_params, indent_level=4, max_line_length=115): """Format a list of InputParam objects into a readable string representation. Args: - input_params: List of InputParam objects to format + input_params: list of InputParam objects to format indent_level: Number of spaces to indent each parameter line (default: 4) max_line_length: Maximum length for each line before wrapping (default: 115) @@ -513,7 +513,7 @@ def format_output_params(output_params, indent_level=4, max_line_length=115): """Format a list of OutputParam objects into a readable string representation. Args: - output_params: List of OutputParam objects to format + output_params: list of OutputParam objects to format indent_level: Number of spaces to indent each parameter line (default: 4) max_line_length: Maximum length for each line before wrapping (default: 115) @@ -527,7 +527,7 @@ def format_components(components, indent_level=4, max_line_length=115, add_empty """Format a list of ComponentSpec objects into a readable string representation. Args: - components: List of ComponentSpec objects to format + components: list of ComponentSpec objects to format indent_level: Number of spaces to indent each component line (default: 4) max_line_length: Maximum length for each line before wrapping (default: 115) add_empty_lines: Whether to add empty lines between components (default: True) @@ -582,7 +582,7 @@ def format_configs(configs, indent_level=4, max_line_length=115, add_empty_lines """Format a list of ConfigSpec objects into a readable string representation. Args: - configs: List of ConfigSpec objects to format + configs: list of ConfigSpec objects to format indent_level: Number of spaces to indent each config line (default: 4) max_line_length: Maximum length for each line before wrapping (default: 115) add_empty_lines: Whether to add empty lines between configs (default: True) @@ -628,13 +628,13 @@ def make_doc_string( Generates a formatted documentation string describing the pipeline block's parameters and structure. Args: - inputs: List of input parameters - intermediate_inputs: List of intermediate input parameters - outputs: List of output parameters + inputs: list of input parameters + intermediate_inputs: list of intermediate input parameters + outputs: list of output parameters description (str, *optional*): Description of the block class_name (str, *optional*): Name of the class to include in the documentation - expected_components (List[ComponentSpec], *optional*): List of expected components - expected_configs (List[ConfigSpec], *optional*): List of expected configurations + expected_components (list[ComponentSpec], *optional*): list of expected components + expected_configs (list[ConfigSpec], *optional*): list of expected configurations Returns: str: A formatted string containing information about components, configs, call parameters, diff --git a/src/diffusers/modular_pipelines/node_utils.py b/src/diffusers/modular_pipelines/node_utils.py index f7ee1dd3097b..dabcfe7a2532 100644 --- a/src/diffusers/modular_pipelines/node_utils.py +++ b/src/diffusers/modular_pipelines/node_utils.py @@ -2,7 +2,7 @@ import logging import os from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import PIL @@ -19,21 +19,21 @@ # YiYi Notes: this is actually for SDXL, put it here for now SDXL_INPUTS_SCHEMA = { "prompt": InputParam( - "prompt", type_hint=Union[str, List[str]], description="The prompt or prompts to guide the image generation" + "prompt", type_hint=str | list[str], description="The prompt or prompts to guide the image generation" ), "prompt_2": InputParam( "prompt_2", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The prompt or prompts to be sent to the tokenizer_2 and text_encoder_2", ), "negative_prompt": InputParam( "negative_prompt", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The prompt or prompts not to guide the image generation", ), "negative_prompt_2": InputParam( "negative_prompt_2", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The negative prompt or prompts for text_encoder_2", ), "cross_attention_kwargs": InputParam( @@ -58,7 +58,7 @@ ), "generator": InputParam( "generator", - type_hint=Optional[Union[torch.Generator, List[torch.Generator]]], + type_hint=Optional[torch.Generator | list[torch.Generator]], description="Generator(s) for deterministic generation", ), "height": InputParam("height", type_hint=Optional[int], description="Height in pixels of the generated image"), @@ -92,36 +92,36 @@ ), "padding_mask_crop": InputParam( "padding_mask_crop", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Size of margin in crop for image and mask", ), "original_size": InputParam( "original_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Original size of the image for SDXL's micro-conditioning", ), "target_size": InputParam( - "target_size", type_hint=Optional[Tuple[int, int]], description="Target size for SDXL's micro-conditioning" + "target_size", type_hint=Optional[tuple[int, int]], description="Target size for SDXL's micro-conditioning" ), "negative_original_size": InputParam( "negative_original_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Negative conditioning based on image resolution", ), "negative_target_size": InputParam( "negative_target_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Negative conditioning based on target resolution", ), "crops_coords_top_left": InputParam( "crops_coords_top_left", - type_hint=Tuple[int, int], + type_hint=tuple[int, int], default=(0, 0), description="Top-left coordinates for SDXL's micro-conditioning", ), "negative_crops_coords_top_left": InputParam( "negative_crops_coords_top_left", - type_hint=Tuple[int, int], + type_hint=tuple[int, int], default=(0, 0), description="Negative conditioning crop coordinates", ), @@ -146,19 +146,19 @@ ), "control_guidance_start": InputParam( "control_guidance_start", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=0.0, description="When ControlNet starts applying", ), "control_guidance_end": InputParam( "control_guidance_end", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=1.0, description="When ControlNet stops applying", ), "controlnet_conditioning_scale": InputParam( "controlnet_conditioning_scale", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=1.0, description="Scale factor for ControlNet outputs", ), @@ -169,7 +169,7 @@ description="Enables ControlNet encoder to recognize input without prompts", ), "control_mode": InputParam( - "control_mode", type_hint=List[int], required=True, description="Control mode for union controlnet" + "control_mode", type_hint=list[int], required=True, description="Control mode for union controlnet" ), } @@ -219,18 +219,18 @@ ), "timestep_cond": InputParam("timestep_cond", type_hint=torch.Tensor, description="Timestep conditioning for LCM"), "noise": InputParam("noise", type_hint=torch.Tensor, description="Noise added to image latents"), - "crops_coords": InputParam("crops_coords", type_hint=Optional[Tuple[int]], description="Crop coordinates"), + "crops_coords": InputParam("crops_coords", type_hint=Optional[tuple[int]], description="Crop coordinates"), "ip_adapter_embeds": InputParam( - "ip_adapter_embeds", type_hint=List[torch.Tensor], description="Image embeddings for IP-Adapter" + "ip_adapter_embeds", type_hint=list[torch.Tensor], description="Image embeddings for IP-Adapter" ), "negative_ip_adapter_embeds": InputParam( "negative_ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], description="Negative image embeddings for IP-Adapter", ), "images": InputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], required=True, description="Generated images", ), diff --git a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py index fdec95dc506e..c6d62b2cafee 100644 --- a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -44,9 +44,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -61,15 +61,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -124,13 +124,13 @@ def description(self) -> str: return "Prepare initial random noise for the generation process" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("pachifier", QwenImagePachifier, default_creation_method="from_config"), ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="height"), InputParam(name="width"), @@ -151,7 +151,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="latents", @@ -214,13 +214,13 @@ def description(self) -> str: return "Step that adds noise to image latents for image-to-image/inpainting. Should be run after set_timesteps, prepare_latents. Both noise and image latents should alreadybe patchified." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( name="latents", @@ -243,7 +243,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="initial_noise", @@ -295,13 +295,13 @@ def description(self) -> str: return "Step that creates mask latents from preprocessed mask_image by interpolating to latent space." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("pachifier", QwenImagePachifier, default_creation_method="from_config"), ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( name="processed_mask_image", @@ -315,7 +315,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="mask", type_hint=torch.Tensor, description="The mask to use for the inpainting process." @@ -361,13 +361,13 @@ def description(self) -> str: return "Step that sets the the scheduler's timesteps for text-to-image generation. Should be run after prepare latents step." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="num_inference_steps", default=50), InputParam(name="sigmas"), @@ -380,7 +380,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="timesteps", type_hint=torch.Tensor, description="The timesteps to use for the denoising process" @@ -427,13 +427,13 @@ def description(self) -> str: return "Step that sets the the scheduler's timesteps for image-to-image generation, and inpainting. Should be run after prepare latents step." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="num_inference_steps", default=50), InputParam(name="sigmas"), @@ -447,7 +447,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="timesteps", @@ -507,7 +507,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="batch_size", required=True), InputParam(name="height", required=True), @@ -517,23 +517,23 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="img_shapes", - type_hint=List[List[Tuple[int, int, int]]], + type_hint=list[list[tuple[int, int, int]]], description="The shapes of the images latents, used for RoPE calculation", ), OutputParam( name="txt_seq_lens", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the prompt embeds, used for RoPE calculation", ), OutputParam( name="negative_txt_seq_lens", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the negative prompt embeds, used for RoPE calculation", ), ] @@ -573,7 +573,7 @@ def description(self) -> str: return "Step that prepares the RoPE inputs for denoising process. This is used in QwenImage Edit. Should be placed after prepare_latents step" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="batch_size", required=True), InputParam(name="image_height", required=True), @@ -585,23 +585,23 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="img_shapes", - type_hint=List[List[Tuple[int, int, int]]], + type_hint=list[list[tuple[int, int, int]]], description="The shapes of the images latents, used for RoPE calculation", ), OutputParam( name="txt_seq_lens", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the prompt embeds, used for RoPE calculation", ), OutputParam( name="negative_txt_seq_lens", kwargs_type="denoiser_input_fields", - type_hint=List[int], + type_hint=list[int], description="The sequence lengths of the negative prompt embeds, used for RoPE calculation", ), ] @@ -645,7 +645,7 @@ class QwenImageControlNetBeforeDenoiserStep(ModularPipelineBlocks): model_name = "qwenimage" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("controlnet", QwenImageControlNetModel), ] @@ -655,7 +655,7 @@ def description(self) -> str: return "step that prepare inputs for controlnet. Insert before the Denoise Step, after set_timesteps step." @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("control_guidance_start", default=0.0), InputParam("control_guidance_end", default=1.0), @@ -670,9 +670,9 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ - OutputParam("controlnet_keep", type_hint=List[float], description="The controlnet keep values"), + OutputParam("controlnet_keep", type_hint=list[float], description="The controlnet keep values"), ] @torch.no_grad() diff --git a/src/diffusers/modular_pipelines/qwenimage/decoders.py b/src/diffusers/modular_pipelines/qwenimage/decoders.py index 6c82fe989e55..ebd92c6e45b6 100644 --- a/src/diffusers/modular_pipelines/qwenimage/decoders.py +++ b/src/diffusers/modular_pipelines/qwenimage/decoders.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Union import numpy as np import PIL @@ -38,7 +37,7 @@ def description(self) -> str: return "Step that decodes the latents to images" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: components = [ ComponentSpec("vae", AutoencoderKLQwenImage), ComponentSpec("pachifier", QwenImagePachifier, default_creation_method="from_config"), @@ -47,7 +46,7 @@ def expected_components(self) -> List[ComponentSpec]: return components @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="height", required=True), InputParam(name="width", required=True), @@ -60,11 +59,11 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], description="The generated images, can be a PIL.Image.Image, torch.Tensor or a numpy array", ) ] @@ -102,7 +101,7 @@ def description(self) -> str: return "postprocess the generated image" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", @@ -113,7 +112,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("images", required=True, description="the generated image from decoders step"), InputParam( @@ -152,7 +151,7 @@ def description(self) -> str: return "postprocess the generated image, optional apply the mask overally to the original image.." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_mask_processor", @@ -163,7 +162,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("images", required=True, description="the generated image from decoders step"), InputParam( diff --git a/src/diffusers/modular_pipelines/qwenimage/denoise.py b/src/diffusers/modular_pipelines/qwenimage/denoise.py index d0704ee6e071..f8b0776d5fc2 100644 --- a/src/diffusers/modular_pipelines/qwenimage/denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage/denoise.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple import torch @@ -41,7 +40,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( "latents", @@ -71,7 +70,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( "latents", @@ -100,7 +99,7 @@ class QwenImageLoopBeforeDenoiserControlNet(ModularPipelineBlocks): model_name = "qwenimage" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -120,7 +119,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( "control_image_latents", @@ -136,7 +135,7 @@ def inputs(self) -> List[InputParam]: InputParam( "controlnet_keep", required=True, - type_hint=List[float], + type_hint=list[float], description="The controlnet keep values to use for the denoising process. Can be generated in prepare_controlnet_inputs step.", ), InputParam( @@ -197,7 +196,7 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -209,7 +208,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("attention_kwargs"), InputParam( @@ -231,7 +230,7 @@ def inputs(self) -> List[InputParam]: InputParam( "img_shapes", required=True, - type_hint=List[Tuple[int, int]], + type_hint=list[tuple[int, int]], description="The shape of the image latents for RoPE calculation. Can be generated in prepare_additional_inputs step.", ), ] @@ -287,7 +286,7 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -299,7 +298,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("attention_kwargs"), InputParam( @@ -321,7 +320,7 @@ def inputs(self) -> List[InputParam]: InputParam( "img_shapes", required=True, - type_hint=List[Tuple[int, int]], + type_hint=list[tuple[int, int]], description="The shape of the image latents for RoPE calculation. Can be generated in prepare_additional_inputs step.", ), ] @@ -380,13 +379,13 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents."), ] @@ -421,7 +420,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( "mask", @@ -476,13 +475,13 @@ def description(self) -> str: ) @property - def loop_expected_components(self) -> List[ComponentSpec]: + def loop_expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler), ] @property - def loop_inputs(self) -> List[InputParam]: + def loop_inputs(self) -> list[InputParam]: return [ InputParam( "timesteps", diff --git a/src/diffusers/modular_pipelines/qwenimage/encoders.py b/src/diffusers/modular_pipelines/qwenimage/encoders.py index 04fb3fdc947b..6feff99df4a0 100644 --- a/src/diffusers/modular_pipelines/qwenimage/encoders.py +++ b/src/diffusers/modular_pipelines/qwenimage/encoders.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import PIL import torch @@ -44,7 +44,7 @@ def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): def get_qwen_prompt_embeds( text_encoder, tokenizer, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, prompt_template_encode: str = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", prompt_template_encode_start_idx: int = 34, tokenizer_max_length: int = 1024, @@ -84,7 +84,7 @@ def get_qwen_prompt_embeds( def get_qwen_prompt_embeds_edit( text_encoder, processor, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: Optional[torch.Tensor] = None, prompt_template_encode: str = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n", prompt_template_encode_start_idx: int = 64, @@ -131,8 +131,8 @@ def get_qwen_prompt_embeds_edit( def get_qwen_prompt_embeds_edit_plus( text_encoder, processor, - prompt: Union[str, List[str]] = None, - image: Optional[Union[torch.Tensor, List[PIL.Image.Image], PIL.Image.Image]] = None, + prompt: str | list[str] = None, + image: Optional[torch.Tensor | list[PIL.Image.Image] | PIL.Image.Image] = None, prompt_template_encode: str = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", img_template_encode: str = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>", prompt_template_encode_start_idx: int = 64, @@ -270,7 +270,7 @@ def description(self) -> str: return f"Image Resize step that resize the {self._image_input_name} to the target area (1024 * 1024) while maintaining the aspect ratio." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_resize_processor", @@ -281,7 +281,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( name=self._image_input_name, required=True, type_hint=torch.Tensor, description="The image to resize" @@ -289,10 +289,10 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( - name=self._resized_image_output_name, type_hint=List[PIL.Image.Image], description="The resized images" + name=self._resized_image_output_name, type_hint=list[PIL.Image.Image], description="The resized images" ), ] @@ -356,11 +356,11 @@ def __init__( super().__init__() @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return super().intermediate_outputs + [ OutputParam( name=self._vae_image_output_name, - type_hint=List[PIL.Image.Image], + type_hint=list[PIL.Image.Image], description="The images to be processed which will be further used by the VAE encoder.", ), ] @@ -406,7 +406,7 @@ def description(self) -> str: return "Text Encoder step that generate text_embeddings to guide the image generation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", Qwen2_5_VLForConditionalGeneration, description="The text encoder to use"), ComponentSpec("tokenizer", Qwen2Tokenizer, description="The tokenizer to use"), @@ -419,7 +419,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [ ConfigSpec( name="prompt_template_encode", @@ -430,7 +430,7 @@ def expected_configs(self) -> List[ConfigSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="prompt", required=True, type_hint=str, description="The prompt to encode"), InputParam(name="negative_prompt", type_hint=str, description="The negative prompt to encode"), @@ -440,7 +440,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="prompt_embeds", @@ -533,7 +533,7 @@ def description(self) -> str: return "Text Encoder step that processes both prompt and image together to generate text embeddings for guiding image generation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", Qwen2_5_VLForConditionalGeneration), ComponentSpec("processor", Qwen2VLProcessor), @@ -546,7 +546,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [ ConfigSpec( name="prompt_template_encode", @@ -556,7 +556,7 @@ def expected_configs(self) -> List[ConfigSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="prompt", required=True, type_hint=str, description="The prompt to encode"), InputParam(name="negative_prompt", type_hint=str, description="The negative prompt to encode"), @@ -569,7 +569,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="prompt_embeds", @@ -647,7 +647,7 @@ class QwenImageEditPlusTextEncoderStep(QwenImageEditTextEncoderStep): model_name = "qwenimage" @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [ ConfigSpec( name="prompt_template_encode", @@ -706,7 +706,7 @@ def description(self) -> str: return "Image Preprocess step for inpainting task. This processes the image and mask inputs together. Images can be resized first using QwenImageEditResizeDynamicStep." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_mask_processor", @@ -717,7 +717,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("mask_image", required=True), InputParam("resized_image"), @@ -728,7 +728,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam(name="processed_image"), OutputParam(name="processed_mask_image"), @@ -787,7 +787,7 @@ def description(self) -> str: return "Image Preprocess step. Images can be resized first using QwenImageEditResizeDynamicStep." @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", @@ -798,11 +798,11 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [InputParam("resized_image"), InputParam("image"), InputParam("height"), InputParam("width")] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam(name="processed_image"), ] @@ -852,7 +852,7 @@ def description(self) -> str: return "Image Preprocess step for QwenImage Edit Plus. Unlike QwenImage Edit, QwenImage Edit Plus doesn't use the same resized image for further preprocessing." @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [InputParam("vae_image"), InputParam("image"), InputParam("height"), InputParam("width")] @torch.no_grad() @@ -919,14 +919,14 @@ def description(self) -> str: return f"Dynamic VAE Encoder step that converts {self._image_input_name} into latent representations {self._image_latents_output_name}.\n" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: components = [ ComponentSpec("vae", AutoencoderKLQwenImage), ] return components @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [ InputParam(self._image_input_name, required=True), InputParam("generator"), @@ -934,7 +934,7 @@ def inputs(self) -> List[InputParam]: return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( self._image_latents_output_name, @@ -976,7 +976,7 @@ def description(self) -> str: return "VAE Encoder step that converts `control_image` into latent representations control_image_latents.\n" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: components = [ ComponentSpec("vae", AutoencoderKLQwenImage), ComponentSpec("controlnet", QwenImageControlNetModel), @@ -990,7 +990,7 @@ def expected_components(self) -> List[ComponentSpec]: return components @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [ InputParam("control_image", required=True), InputParam("height"), @@ -1000,7 +1000,7 @@ def inputs(self) -> List[InputParam]: return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "control_image_latents", diff --git a/src/diffusers/modular_pipelines/qwenimage/inputs.py b/src/diffusers/modular_pipelines/qwenimage/inputs.py index 2b229c040b89..e6f2edcb6013 100644 --- a/src/diffusers/modular_pipelines/qwenimage/inputs.py +++ b/src/diffusers/modular_pipelines/qwenimage/inputs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple import torch @@ -78,7 +77,7 @@ def repeat_tensor_to_batch_size( return input_tensor -def calculate_dimension_from_latents(latents: torch.Tensor, vae_scale_factor: int) -> Tuple[int, int]: +def calculate_dimension_from_latents(latents: torch.Tensor, vae_scale_factor: int) -> tuple[int, int]: """Calculate image dimensions from latent tensor dimensions. This function converts latent space dimensions to image space dimensions by multiplying the latent height and width @@ -91,7 +90,7 @@ def calculate_dimension_from_latents(latents: torch.Tensor, vae_scale_factor: in Typically 8 for most VAEs (image is 8x larger than latents in each dimension) Returns: - Tuple[int, int]: The calculated image dimensions as (height, width) + tuple[int, int]: The calculated image dimensions as (height, width) Raises: ValueError: If latents tensor doesn't have 4 or 5 dimensions @@ -127,7 +126,7 @@ def description(self) -> str: return summary_section + placement_section @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="num_images_per_prompt", default=1), InputParam(name="prompt_embeds", required=True, kwargs_type="denoiser_input_fields"), @@ -137,7 +136,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "batch_size", @@ -226,8 +225,8 @@ class QwenImageInputsDynamicStep(ModularPipelineBlocks): def __init__( self, - image_latent_inputs: List[str] = ["image_latents"], - additional_batch_inputs: List[str] = [], + image_latent_inputs: list[str] = ["image_latents"], + additional_batch_inputs: list[str] = [], ): """Initialize a configurable step that standardizes the inputs for the denoising step. It:\n" @@ -238,10 +237,10 @@ def __init__( This is a dynamic block that allows you to configure which inputs to process. Args: - image_latent_inputs (List[str], optional): Names of image latent tensors to process. + image_latent_inputs (list[str], optional): Names of image latent tensors to process. These will be used to determine height/width, patchified, and batch-expanded. Can be a single string or list of strings. Defaults to ["image_latents"]. Examples: ["image_latents"], ["control_image_latents"] - additional_batch_inputs (List[str], optional): + additional_batch_inputs (list[str], optional): Names of additional conditional input tensors to expand batch size. These tensors will only have their batch dimensions adjusted to match the final batch size. Can be a single string or list of strings. Defaults to []. Examples: ["processed_mask_image"] @@ -289,7 +288,7 @@ def description(self) -> str: return summary_section + inputs_info + placement_section @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: inputs = [ InputParam(name="num_images_per_prompt", default=1), InputParam(name="batch_size", required=True), @@ -308,14 +307,14 @@ def inputs(self) -> List[InputParam]: return inputs @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam(name="image_height", type_hint=int, description="The height of the image latents"), OutputParam(name="image_width", type_hint=int, description="The width of the image latents"), ] @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("pachifier", QwenImagePachifier, default_creation_method="from_config"), ] @@ -380,7 +379,7 @@ def description(self) -> str: return "prepare the `control_image_latents` for controlnet. Insert after all the other inputs steps." @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam(name="control_image_latents", required=True), InputParam(name="batch_size", required=True), diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py index 70cbf0c1c78d..fb4cc86657ca 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, List, Optional, Tuple, Union +from typing import Any, Optional import PIL import torch @@ -47,9 +47,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -64,15 +64,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -210,7 +210,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_images_per_prompt", default=1), InputParam( @@ -236,18 +236,18 @@ def inputs(self) -> List[InputParam]: ), InputParam( "ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], description="Pre-generated image embeddings for IP-Adapter. Can be generated from ip_adapter step.", ), InputParam( "negative_ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], description="Pre-generated negative image embeddings for IP-Adapter. Can be generated from ip_adapter step.", ), ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "batch_size", @@ -285,13 +285,13 @@ def intermediate_outputs(self) -> List[str]: ), OutputParam( "ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], kwargs_type="denoiser_input_fields", # already in intermedites state but declare here again for denoiser_input_fields description="image embeddings for IP-Adapter", ), OutputParam( "negative_ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], kwargs_type="denoiser_input_fields", # already in intermedites state but declare here again for denoiser_input_fields description="negative image embeddings for IP-Adapter", ), @@ -393,7 +393,7 @@ class StableDiffusionXLImg2ImgSetTimestepsStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ] @@ -406,7 +406,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_inference_steps", default=50), InputParam("timesteps"), @@ -425,7 +425,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam("timesteps", type_hint=torch.Tensor, description="The timesteps to use for inference"), OutputParam( @@ -537,7 +537,7 @@ class StableDiffusionXLSetTimestepsStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ] @@ -547,7 +547,7 @@ def description(self) -> str: return "Step that sets the scheduler's timesteps for inference" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_inference_steps", default=50), InputParam("timesteps"), @@ -556,7 +556,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("timesteps", type_hint=torch.Tensor, description="The timesteps to use for inference"), OutputParam( @@ -605,7 +605,7 @@ class StableDiffusionXLInpaintPrepareLatentsStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ] @@ -615,7 +615,7 @@ def description(self) -> str: return "Step that prepares the latents for the inpainting process" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("latents"), InputParam("num_images_per_prompt", default=1), @@ -664,7 +664,7 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" @@ -879,7 +879,7 @@ class StableDiffusionXLImg2ImgPrepareLatentsStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKL), ComponentSpec("scheduler", EulerDiscreteScheduler), @@ -890,7 +890,7 @@ def description(self) -> str: return "Step that prepares the latents for the image-to-image generation process" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("latents"), InputParam("num_images_per_prompt", default=1), @@ -918,7 +918,7 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" @@ -955,7 +955,7 @@ class StableDiffusionXLPrepareLatentsStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ComponentSpec("vae", AutoencoderKL), @@ -966,7 +966,7 @@ def description(self) -> str: return "Prepare latents step that prepares the latents for the text-to-image generation process" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("height"), InputParam("width"), @@ -983,7 +983,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" @@ -1061,13 +1061,13 @@ class StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep(ModularPipelineB model_name = "stable-diffusion-xl" @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [ ConfigSpec("requires_aesthetics_score", False), ] @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("unet", UNet2DConditionModel), ComponentSpec( @@ -1083,7 +1083,7 @@ def description(self) -> str: return "Step that prepares the additional conditioning for the image-to-image/inpainting generation process" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("original_size"), InputParam("target_size"), @@ -1115,7 +1115,7 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "add_time_ids", @@ -1284,7 +1284,7 @@ def description(self) -> str: return "Step that prepares the additional conditioning for the text-to-image generation process" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("unet", UNet2DConditionModel), ComponentSpec( @@ -1296,7 +1296,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("original_size"), InputParam("target_size"), @@ -1326,7 +1326,7 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "add_time_ids", @@ -1458,7 +1458,7 @@ class StableDiffusionXLControlNetInputStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("controlnet", ControlNetModel), ComponentSpec( @@ -1474,7 +1474,7 @@ def description(self) -> str: return "step that prepare inputs for controlnet" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("control_image", required=True), InputParam("control_guidance_start", default=0.0), @@ -1502,26 +1502,26 @@ def inputs(self) -> List[Tuple[str, Any]]: ), InputParam( "crops_coords", - type_hint=Optional[Tuple[int]], + type_hint=Optional[tuple[int]], description="The crop coordinates to use for preprocess/postprocess the image and mask, for inpainting task only. Can be generated in vae_encode step.", ), ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("controlnet_cond", type_hint=torch.Tensor, description="The processed control image"), OutputParam( - "control_guidance_start", type_hint=List[float], description="The controlnet guidance start values" + "control_guidance_start", type_hint=list[float], description="The controlnet guidance start values" ), OutputParam( - "control_guidance_end", type_hint=List[float], description="The controlnet guidance end values" + "control_guidance_end", type_hint=list[float], description="The controlnet guidance end values" ), OutputParam( - "conditioning_scale", type_hint=List[float], description="The controlnet conditioning scale values" + "conditioning_scale", type_hint=list[float], description="The controlnet conditioning scale values" ), OutputParam("guess_mode", type_hint=bool, description="Whether guess mode is used"), - OutputParam("controlnet_keep", type_hint=List[float], description="The controlnet keep values"), + OutputParam("controlnet_keep", type_hint=list[float], description="The controlnet keep values"), ] # Modified from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl.StableDiffusionXLControlNetPipeline.prepare_image @@ -1672,7 +1672,7 @@ class StableDiffusionXLControlNetUnionInputStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("controlnet", ControlNetUnionModel), ComponentSpec( @@ -1688,7 +1688,7 @@ def description(self) -> str: return "step that prepares inputs for the ControlNetUnion model" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("control_image", required=True), InputParam("control_mode", required=True), @@ -1723,18 +1723,18 @@ def inputs(self) -> List[Tuple[str, Any]]: ), InputParam( "crops_coords", - type_hint=Optional[Tuple[int]], + type_hint=Optional[tuple[int]], description="The crop coordinates to use for preprocess/postprocess the image and mask, for inpainting task only. Can be generated in vae_encode step.", ), ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ - OutputParam("controlnet_cond", type_hint=List[torch.Tensor], description="The processed control images"), + OutputParam("controlnet_cond", type_hint=list[torch.Tensor], description="The processed control images"), OutputParam( "control_type_idx", - type_hint=List[int], + type_hint=list[int], description="The control mode indices", kwargs_type="controlnet_kwargs", ), @@ -1747,10 +1747,10 @@ def intermediate_outputs(self) -> List[OutputParam]: OutputParam("control_guidance_start", type_hint=float, description="The controlnet guidance start value"), OutputParam("control_guidance_end", type_hint=float, description="The controlnet guidance end value"), OutputParam( - "conditioning_scale", type_hint=List[float], description="The controlnet conditioning scale values" + "conditioning_scale", type_hint=list[float], description="The controlnet conditioning scale values" ), OutputParam("guess_mode", type_hint=bool, description="Whether guess mode is used"), - OutputParam("controlnet_keep", type_hint=List[float], description="The controlnet keep values"), + OutputParam("controlnet_keep", type_hint=list[float], description="The controlnet keep values"), ] # Modified from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl.StableDiffusionXLControlNetPipeline.prepare_image diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/decoders.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/decoders.py index feb78e1ef11b..c3151aeb1fcf 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/decoders.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/decoders.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Tuple, Union +from typing import Any import numpy as np import PIL @@ -37,7 +37,7 @@ class StableDiffusionXLDecodeStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKL), ComponentSpec( @@ -53,7 +53,7 @@ def description(self) -> str: return "Step that decodes the denoised latents into images" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("output_type", default="pil"), InputParam( @@ -65,11 +65,11 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], description="The generated images, can be a PIL.Image.Image, torch.Tensor or a numpy array", ) ] @@ -163,7 +163,7 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", @@ -174,19 +174,19 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("image"), InputParam("mask_image"), InputParam("padding_mask_crop"), InputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], description="The generated images from the decode step", ), InputParam( "crops_coords", - type_hint=Tuple[int, int], + type_hint=tuple[int, int], description="The crop coordinates to use for preprocess/postprocess the image and mask, for inpainting task only. Can be generated in vae_encode step.", ), ] diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/denoise.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/denoise.py index 83e6007e3fc1..009df37f6ff3 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/denoise.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/denoise.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, List, Optional, Tuple +from typing import Any, Optional import torch @@ -41,7 +41,7 @@ class StableDiffusionXLLoopBeforeDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ] @@ -55,7 +55,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[str]: + def inputs(self) -> list[str]: return [ InputParam( "latents", @@ -77,7 +77,7 @@ class StableDiffusionXLInpaintLoopBeforeDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ComponentSpec("unet", UNet2DConditionModel), @@ -91,7 +91,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[str]: + def inputs(self) -> list[str]: return [ InputParam( "latents", @@ -148,7 +148,7 @@ class StableDiffusionXLLoopDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -168,7 +168,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("cross_attention_kwargs"), InputParam( @@ -248,7 +248,7 @@ class StableDiffusionXLControlNetLoopDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -269,7 +269,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("cross_attention_kwargs"), InputParam( @@ -292,7 +292,7 @@ def inputs(self) -> List[Tuple[str, Any]]: InputParam( "controlnet_keep", required=True, - type_hint=List[float], + type_hint=list[float], description="The controlnet keep values to use for the denoising process. Can be generated in prepare_controlnet_inputs step.", ), InputParam( @@ -443,7 +443,7 @@ class StableDiffusionXLLoopAfterDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ] @@ -457,14 +457,14 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("eta", default=0.0), InputParam("generator"), ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents")] # YiYi TODO: move this out of here @@ -508,7 +508,7 @@ class StableDiffusionXLInpaintLoopAfterDenoiser(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", EulerDiscreteScheduler), ComponentSpec("unet", UNet2DConditionModel), @@ -523,7 +523,7 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("eta", default=0.0), InputParam("generator"), @@ -551,7 +551,7 @@ def inputs(self) -> List[Tuple[str, Any]]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents")] @staticmethod @@ -625,7 +625,7 @@ def description(self) -> str: ) @property - def loop_expected_components(self) -> List[ComponentSpec]: + def loop_expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -638,7 +638,7 @@ def loop_expected_components(self) -> List[ComponentSpec]: ] @property - def loop_inputs(self) -> List[InputParam]: + def loop_inputs(self) -> list[InputParam]: return [ InputParam( "timesteps", diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py index 90b254b6f5d4..901bf5e97f1f 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple +from typing import Optional import torch from transformers import ( @@ -71,7 +71,7 @@ def description(self) -> str: ) @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("image_encoder", CLIPVisionModelWithProjection), ComponentSpec( @@ -90,7 +90,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam( "ip_adapter_image", @@ -101,7 +101,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("ip_adapter_embeds", type_hint=torch.Tensor, description="IP adapter image embeddings"), OutputParam( @@ -223,7 +223,7 @@ def description(self) -> str: return "Text Encoder step that generate text_embeddings to guide the image generation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", CLIPTextModel), ComponentSpec("text_encoder_2", CLIPTextModelWithProjection), @@ -238,11 +238,11 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [ConfigSpec("force_zeros_for_empty_prompt", True)] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("prompt"), InputParam("prompt_2"), @@ -253,7 +253,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "prompt_embeds", @@ -313,9 +313,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -324,11 +324,11 @@ def encode_prompt( number of images that should be generated per prompt prepare_unconditional_embeds (`bool`): whether to use prepare unconditional embeddings or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -450,7 +450,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -584,7 +584,7 @@ def description(self) -> str: return "Vae Encoder step that encode the input image into a latent representation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKL), ComponentSpec( @@ -596,7 +596,7 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("image", required=True), InputParam("height"), @@ -611,7 +611,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "image_latents", @@ -687,7 +687,7 @@ class StableDiffusionXLInpaintVaeEncoderStep(ModularPipelineBlocks): model_name = "stable-diffusion-xl" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKL), ComponentSpec( @@ -711,7 +711,7 @@ def description(self) -> str: return "Vae encoder step that prepares the image and mask for the inpainting process" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("height"), InputParam("width"), @@ -723,7 +723,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "image_latents", type_hint=torch.Tensor, description="The latents representation of the input image" @@ -736,7 +736,7 @@ def intermediate_outputs(self) -> List[OutputParam]: ), OutputParam( "crops_coords", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="The crop coordinates to use for the preprocess/postprocess of the image and mask", ), ] diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_pipeline.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_pipeline.py index f2a4c96073ea..1a8f12ccf3c5 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_pipeline.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import PIL @@ -94,21 +94,21 @@ def num_channels_latents(self): # auto_docstring SDXL_INPUTS_SCHEMA = { "prompt": InputParam( - "prompt", type_hint=Union[str, List[str]], description="The prompt or prompts to guide the image generation" + "prompt", type_hint=str | list[str], description="The prompt or prompts to guide the image generation" ), "prompt_2": InputParam( "prompt_2", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The prompt or prompts to be sent to the tokenizer_2 and text_encoder_2", ), "negative_prompt": InputParam( "negative_prompt", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The prompt or prompts not to guide the image generation", ), "negative_prompt_2": InputParam( "negative_prompt_2", - type_hint=Union[str, List[str]], + type_hint=str | list[str], description="The negative prompt or prompts for text_encoder_2", ), "cross_attention_kwargs": InputParam( @@ -133,7 +133,7 @@ def num_channels_latents(self): ), "generator": InputParam( "generator", - type_hint=Optional[Union[torch.Generator, List[torch.Generator]]], + type_hint=Optional[torch.Generator | list[torch.Generator]], description="Generator(s) for deterministic generation", ), "height": InputParam("height", type_hint=Optional[int], description="Height in pixels of the generated image"), @@ -167,36 +167,36 @@ def num_channels_latents(self): ), "padding_mask_crop": InputParam( "padding_mask_crop", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Size of margin in crop for image and mask", ), "original_size": InputParam( "original_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Original size of the image for SDXL's micro-conditioning", ), "target_size": InputParam( - "target_size", type_hint=Optional[Tuple[int, int]], description="Target size for SDXL's micro-conditioning" + "target_size", type_hint=Optional[tuple[int, int]], description="Target size for SDXL's micro-conditioning" ), "negative_original_size": InputParam( "negative_original_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Negative conditioning based on image resolution", ), "negative_target_size": InputParam( "negative_target_size", - type_hint=Optional[Tuple[int, int]], + type_hint=Optional[tuple[int, int]], description="Negative conditioning based on target resolution", ), "crops_coords_top_left": InputParam( "crops_coords_top_left", - type_hint=Tuple[int, int], + type_hint=tuple[int, int], default=(0, 0), description="Top-left coordinates for SDXL's micro-conditioning", ), "negative_crops_coords_top_left": InputParam( "negative_crops_coords_top_left", - type_hint=Tuple[int, int], + type_hint=tuple[int, int], default=(0, 0), description="Negative conditioning crop coordinates", ), @@ -221,19 +221,19 @@ def num_channels_latents(self): ), "control_guidance_start": InputParam( "control_guidance_start", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=0.0, description="When ControlNet starts applying", ), "control_guidance_end": InputParam( "control_guidance_end", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=1.0, description="When ControlNet stops applying", ), "controlnet_conditioning_scale": InputParam( "controlnet_conditioning_scale", - type_hint=Union[float, List[float]], + type_hint=float | list[float], default=1.0, description="Scale factor for ControlNet outputs", ), @@ -244,7 +244,7 @@ def num_channels_latents(self): description="Enables ControlNet encoder to recognize input without prompts", ), "control_mode": InputParam( - "control_mode", type_hint=List[int], required=True, description="Control mode for union controlnet" + "control_mode", type_hint=list[int], required=True, description="Control mode for union controlnet" ), "prompt_embeds": InputParam( "prompt_embeds", @@ -284,18 +284,18 @@ def num_channels_latents(self): ), "timestep_cond": InputParam("timestep_cond", type_hint=torch.Tensor, description="Timestep conditioning for LCM"), "noise": InputParam("noise", type_hint=torch.Tensor, description="Noise added to image latents"), - "crops_coords": InputParam("crops_coords", type_hint=Optional[Tuple[int]], description="Crop coordinates"), + "crops_coords": InputParam("crops_coords", type_hint=Optional[tuple[int]], description="Crop coordinates"), "ip_adapter_embeds": InputParam( - "ip_adapter_embeds", type_hint=List[torch.Tensor], description="Image embeddings for IP-Adapter" + "ip_adapter_embeds", type_hint=list[torch.Tensor], description="Image embeddings for IP-Adapter" ), "negative_ip_adapter_embeds": InputParam( "negative_ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], description="Negative image embeddings for IP-Adapter", ), "images": InputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], required=True, description="Generated images", ), @@ -324,7 +324,7 @@ def num_channels_latents(self): "masked_image_latents": OutputParam( "masked_image_latents", type_hint=torch.Tensor, description="Masked image latents for inpainting" ), - "crops_coords": OutputParam("crops_coords", type_hint=Optional[Tuple[int]], description="Crop coordinates"), + "crops_coords": OutputParam("crops_coords", type_hint=Optional[tuple[int]], description="Crop coordinates"), "timesteps": OutputParam("timesteps", type_hint=torch.Tensor, description="Timesteps for inference"), "num_inference_steps": OutputParam("num_inference_steps", type_hint=int, description="Number of denoising steps"), "latent_timestep": OutputParam( @@ -338,16 +338,16 @@ def num_channels_latents(self): "latents": OutputParam("latents", type_hint=torch.Tensor, description="Denoised latents"), "noise": OutputParam("noise", type_hint=torch.Tensor, description="Noise added to image latents"), "ip_adapter_embeds": OutputParam( - "ip_adapter_embeds", type_hint=List[torch.Tensor], description="Image embeddings for IP-Adapter" + "ip_adapter_embeds", type_hint=list[torch.Tensor], description="Image embeddings for IP-Adapter" ), "negative_ip_adapter_embeds": OutputParam( "negative_ip_adapter_embeds", - type_hint=List[torch.Tensor], + type_hint=list[torch.Tensor], description="Negative image embeddings for IP-Adapter", ), "images": OutputParam( "images", - type_hint=Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]], + type_hint=list[PIL.Image.Image] | list[torch.Tensor] | list[np.array], description="Generated images", ), } @@ -356,9 +356,7 @@ def num_channels_latents(self): SDXL_OUTPUTS_SCHEMA = { "images": OutputParam( "images", - type_hint=Union[ - Tuple[Union[List[PIL.Image.Image], List[torch.Tensor], List[np.array]]], StableDiffusionXLPipelineOutput - ], + type_hint=tuple[list[PIL.Image.Image] | list[torch.Tensor] | list[np.array]] | StableDiffusionXLPipelineOutput, description="The final generated images", ) } diff --git a/src/diffusers/modular_pipelines/wan/before_denoise.py b/src/diffusers/modular_pipelines/wan/before_denoise.py index d48f678edd59..8c249f886f8d 100644 --- a/src/diffusers/modular_pipelines/wan/before_denoise.py +++ b/src/diffusers/modular_pipelines/wan/before_denoise.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Union +from typing import Optional import torch @@ -38,9 +38,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -55,15 +55,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -109,13 +109,13 @@ def description(self) -> str: ) @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_videos_per_prompt", default=1), ] @property - def intermediate_inputs(self) -> List[str]: + def intermediate_inputs(self) -> list[str]: return [ InputParam( "prompt_embeds", @@ -131,7 +131,7 @@ def intermediate_inputs(self) -> List[str]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "batch_size", @@ -198,7 +198,7 @@ class WanSetTimestepsStep(ModularPipelineBlocks): model_name = "wan" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", UniPCMultistepScheduler), ] @@ -208,7 +208,7 @@ def description(self) -> str: return "Step that sets the scheduler's timesteps for inference" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("num_inference_steps", default=50), InputParam("timesteps"), @@ -216,7 +216,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("timesteps", type_hint=torch.Tensor, description="The timesteps to use for inference"), OutputParam( @@ -247,7 +247,7 @@ class WanPrepareLatentsStep(ModularPipelineBlocks): model_name = "wan" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [] @property @@ -255,7 +255,7 @@ def description(self) -> str: return "Prepare latents step that prepares the latents for the text-to-video generation process" @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("height", type_hint=int), InputParam("width", type_hint=int), @@ -265,7 +265,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_inputs(self) -> List[InputParam]: + def intermediate_inputs(self) -> list[InputParam]: return [ InputParam("generator"), InputParam( @@ -278,7 +278,7 @@ def intermediate_inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" @@ -311,7 +311,7 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: diff --git a/src/diffusers/modular_pipelines/wan/decoders.py b/src/diffusers/modular_pipelines/wan/decoders.py index 8c751172d858..5dc316682327 100644 --- a/src/diffusers/modular_pipelines/wan/decoders.py +++ b/src/diffusers/modular_pipelines/wan/decoders.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Tuple, Union +from typing import Any import numpy as np import PIL @@ -33,7 +33,7 @@ class WanDecodeStep(ModularPipelineBlocks): model_name = "wan" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("vae", AutoencoderKLWan), ComponentSpec( @@ -49,13 +49,13 @@ def description(self) -> str: return "Step that decodes the denoised latents into images" @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("output_type", default="pil"), ] @property - def intermediate_inputs(self) -> List[str]: + def intermediate_inputs(self) -> list[str]: return [ InputParam( "latents", @@ -66,11 +66,11 @@ def intermediate_inputs(self) -> List[str]: ] @property - def intermediate_outputs(self) -> List[str]: + def intermediate_outputs(self) -> list[str]: return [ OutputParam( "videos", - type_hint=Union[List[List[PIL.Image.Image]], List[torch.Tensor], List[np.ndarray]], + type_hint=list[list[PIL.Image.Image]] | list[torch.Tensor] | list[np.ndarray], description="The generated videos, can be a PIL.Image.Image, torch.Tensor or a numpy array", ) ] diff --git a/src/diffusers/modular_pipelines/wan/denoise.py b/src/diffusers/modular_pipelines/wan/denoise.py index 66c51493bd6a..67f23164ceaf 100644 --- a/src/diffusers/modular_pipelines/wan/denoise.py +++ b/src/diffusers/modular_pipelines/wan/denoise.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Tuple +from typing import Any import torch @@ -38,7 +38,7 @@ class WanLoopDenoiser(ModularPipelineBlocks): model_name = "wan" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -58,13 +58,13 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [ InputParam("attention_kwargs"), ] @property - def intermediate_inputs(self) -> List[str]: + def intermediate_inputs(self) -> list[str]: return [ InputParam( "latents", @@ -136,7 +136,7 @@ class WanLoopAfterDenoiser(ModularPipelineBlocks): model_name = "wan" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("scheduler", UniPCMultistepScheduler), ] @@ -150,17 +150,17 @@ def description(self) -> str: ) @property - def inputs(self) -> List[Tuple[str, Any]]: + def inputs(self) -> list[tuple[str, Any]]: return [] @property - def intermediate_inputs(self) -> List[str]: + def intermediate_inputs(self) -> list[str]: return [ InputParam("generator"), ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam("latents", type_hint=torch.Tensor, description="The denoised latents")] @torch.no_grad() @@ -191,7 +191,7 @@ def description(self) -> str: ) @property - def loop_expected_components(self) -> List[ComponentSpec]: + def loop_expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "guider", @@ -204,7 +204,7 @@ def loop_expected_components(self) -> List[ComponentSpec]: ] @property - def loop_intermediate_inputs(self) -> List[InputParam]: + def loop_intermediate_inputs(self) -> list[InputParam]: return [ InputParam( "timesteps", diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index cb2fc242383c..2d772672bd14 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import List, Optional, Union +from typing import Optional import regex as re import torch @@ -59,7 +59,7 @@ def description(self) -> str: return "Text Encoder step that generate text_embeddings to guide the video generation" @property - def expected_components(self) -> List[ComponentSpec]: + def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", UMT5EncoderModel), ComponentSpec("tokenizer", AutoTokenizer), @@ -72,11 +72,11 @@ def expected_components(self) -> List[ComponentSpec]: ] @property - def expected_configs(self) -> List[ConfigSpec]: + def expected_configs(self) -> list[ConfigSpec]: return [] @property - def inputs(self) -> List[InputParam]: + def inputs(self) -> list[InputParam]: return [ InputParam("prompt"), InputParam("negative_prompt"), @@ -84,7 +84,7 @@ def inputs(self) -> List[InputParam]: ] @property - def intermediate_outputs(self) -> List[OutputParam]: + def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( "prompt_embeds", @@ -110,7 +110,7 @@ def check_inputs(block_state): @staticmethod def _get_t5_prompt_embeds( components, - prompt: Union[str, List[str]], + prompt: str | list[str], max_sequence_length: int, device: torch.device, ): @@ -154,7 +154,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -162,7 +162,7 @@ def encode_prompt( number of videos that should be generated per prompt prepare_unconditional_embeds (`bool`): whether to use prepare unconditional embeddings or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). diff --git a/src/diffusers/optimization.py b/src/diffusers/optimization.py index e0b3576e4426..fcb25317e6db 100644 --- a/src/diffusers/optimization.py +++ b/src/diffusers/optimization.py @@ -16,7 +16,7 @@ import math from enum import Enum -from typing import Optional, Union +from typing import Optional from torch.optim import Optimizer from torch.optim.lr_scheduler import LambdaLR @@ -287,7 +287,7 @@ def lr_lambda(current_step: int): def get_scheduler( - name: Union[str, SchedulerType], + name: str | SchedulerType, optimizer: Optimizer, step_rules: Optional[str] = None, num_warmup_steps: Optional[int] = None, diff --git a/src/diffusers/pipelines/allegro/pipeline_allegro.py b/src/diffusers/pipelines/allegro/pipeline_allegro.py index 3be0129088fb..5a811d0b2abf 100644 --- a/src/diffusers/pipelines/allegro/pipeline_allegro.py +++ b/src/diffusers/pipelines/allegro/pipeline_allegro.py @@ -18,7 +18,7 @@ import math import re import urllib.parse as ul -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -85,9 +85,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -102,15 +102,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -214,7 +214,7 @@ def __init__( # Copied from diffusers.pipelines.pixart_alpha.pipeline_pixart_alpha.PixArtAlphaPipeline.encode_prompt with 120->512, num_images_per_prompt->num_videos_per_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_videos_per_prompt: int = 1, @@ -231,9 +231,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -719,17 +719,17 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 100, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 7.5, num_frames: Optional[int] = None, height: Optional[int] = None, width: Optional[int] = None, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -738,27 +738,27 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], clean_caption: bool = True, max_sequence_length: int = 512, - ) -> Union[AllegroPipelineOutput, Tuple]: + ) -> AllegroPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the video generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the video generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.5): @@ -778,7 +778,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/allegro/pipeline_output.py b/src/diffusers/pipelines/allegro/pipeline_output.py index 6a721783ca86..bf85a4954ce9 100644 --- a/src/diffusers/pipelines/allegro/pipeline_output.py +++ b/src/diffusers/pipelines/allegro/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL @@ -14,10 +13,10 @@ class AllegroPipelineOutput(BaseOutput): Output class for Allegro pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] diff --git a/src/diffusers/pipelines/amused/pipeline_amused.py b/src/diffusers/pipelines/amused/pipeline_amused.py index 131e34d1a4a1..a07cbbe68e4e 100644 --- a/src/diffusers/pipelines/amused/pipeline_amused.py +++ b/src/diffusers/pipelines/amused/pipeline_amused.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPTextModelWithProjection, CLIPTokenizer @@ -84,12 +84,12 @@ def __init__( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[List[str], str]] = None, + prompt: Optional[list[str] | str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 12, guidance_scale: float = 10.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, generator: Optional[torch.Generator] = None, latents: Optional[torch.IntTensor] = None, @@ -101,16 +101,16 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, micro_conditioning_aesthetic_score: int = 6, - micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), - temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + micro_conditioning_crop_coord: tuple[int, int] = (0, 0), + temperature: int | tuple[int, int] | list[int] = (2, 0), ): """ The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.transformer.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -122,7 +122,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 10.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -162,10 +162,10 @@ def __call__( The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of https://huggingface.co/papers/2307.01952. - micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + micro_conditioning_crop_coord (`tuple[int]`, *optional*, defaults to (0, 0)): The targeted height, width crop coordinates. See the micro-conditioning section of https://huggingface.co/papers/2307.01952. - temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + temperature (`Union[int, tuple[int, int], list[int]]`, *optional*, defaults to (2, 0)): Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. Examples: diff --git a/src/diffusers/pipelines/amused/pipeline_amused_img2img.py b/src/diffusers/pipelines/amused/pipeline_amused_img2img.py index a122c12236dd..5c4e6ee2b4c9 100644 --- a/src/diffusers/pipelines/amused/pipeline_amused_img2img.py +++ b/src/diffusers/pipelines/amused/pipeline_amused_img2img.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPTextModelWithProjection, CLIPTokenizer @@ -99,12 +99,12 @@ def __init__( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[List[str], str]] = None, + prompt: Optional[list[str] | str] = None, image: PipelineImageInput = None, strength: float = 0.5, num_inference_steps: int = 12, guidance_scale: float = 10.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, generator: Optional[torch.Generator] = None, prompt_embeds: Optional[torch.Tensor] = None, @@ -115,18 +115,18 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, micro_conditioning_aesthetic_score: int = 6, - micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), - temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + micro_conditioning_crop_coord: tuple[int, int] = (0, 0), + temperature: int | tuple[int, int] | list[int] = (2, 0), ): """ The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -144,7 +144,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 10.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -181,10 +181,10 @@ def __call__( The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of https://huggingface.co/papers/2307.01952. - micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + micro_conditioning_crop_coord (`tuple[int]`, *optional*, defaults to (0, 0)): The targeted height, width crop coordinates. See the micro-conditioning section of https://huggingface.co/papers/2307.01952. - temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + temperature (`Union[int, tuple[int, int], list[int]]`, *optional*, defaults to (2, 0)): Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. Examples: diff --git a/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py b/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py index f4bd4944ff9a..1fa9168f4546 100644 --- a/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py +++ b/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPTextModelWithProjection, CLIPTokenizer @@ -115,13 +115,13 @@ def __init__( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[List[str], str]] = None, + prompt: Optional[list[str] | str] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, strength: float = 1.0, num_inference_steps: int = 12, guidance_scale: float = 10.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, generator: Optional[torch.Generator] = None, prompt_embeds: Optional[torch.Tensor] = None, @@ -132,24 +132,24 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, micro_conditioning_aesthetic_score: int = 6, - micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), - temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + micro_conditioning_crop_coord: tuple[int, int] = (0, 0), + temperature: int | tuple[int, int] | list[int] = (2, 0), ): """ The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one @@ -168,7 +168,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 10.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -205,10 +205,10 @@ def __call__( The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ and the micro-conditioning section of https://huggingface.co/papers/2307.01952. - micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + micro_conditioning_crop_coord (`tuple[int]`, *optional*, defaults to (0, 0)): The targeted height, width crop coordinates. See the micro-conditioning section of https://huggingface.co/papers/2307.01952. - temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + temperature (`Union[int, tuple[int, int], list[int]]`, *optional*, defaults to (2, 0)): Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. Examples: diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff.py index 091b6db713ba..6af1bb903898 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection @@ -122,16 +122,14 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, ): @@ -169,7 +167,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -177,7 +175,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -276,7 +274,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -574,27 +572,27 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, num_frames: Optional[int] = 16, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], decode_chunk_size: int = 16, **kwargs, ): @@ -602,7 +600,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -617,13 +615,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -639,7 +637,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -660,7 +658,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py index 70180ccf0650..48fc772e1a92 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch import torch.nn.functional as F @@ -164,9 +164,9 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, feature_extractor: Optional[CLIPImageProcessor] = None, image_encoder: Optional[CLIPVisionModelWithProjection] = None, @@ -212,7 +212,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -220,7 +220,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -319,7 +319,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -721,39 +721,39 @@ def interrupt(self): @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_frames: Optional[int] = 16, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, ip_adapter_image_embeds: Optional[PipelineImageInput] = None, - conditioning_frames: Optional[List[PipelineImageInput]] = None, + conditioning_frames: Optional[list[PipelineImageInput]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], decode_chunk_size: int = 16, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -768,13 +768,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -790,12 +790,12 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. - conditioning_frames (`List[PipelineImageInput]`, *optional*): + conditioning_frames (`list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. @@ -807,16 +807,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -826,7 +826,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py index 56d319027595..16c5ac2264fb 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -151,9 +151,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -168,15 +168,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -284,16 +284,14 @@ def __init__( text_encoder_2: CLIPTextModelWithProjection, tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, image_encoder: CLIPVisionModelWithProjection = None, feature_extractor: CLIPImageProcessor = None, force_zeros_for_empty_prompt: bool = True, @@ -346,9 +344,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -357,11 +355,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -479,7 +477,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -870,50 +868,50 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, num_frames: int = 16, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the video generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders num_frames: @@ -932,11 +930,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -953,11 +951,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower video quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the video generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the video generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -965,7 +963,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -988,7 +986,7 @@ def __call__( input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. If not provided, embeddings are computed from the `ip_adapter_image` input argument. output_type (`str`, *optional*, defaults to `"pil"`): @@ -1007,31 +1005,31 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1041,7 +1039,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py index 46d650efe8b6..025c16a113f7 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL @@ -176,7 +176,7 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, controlnet: SparseControlNetModel, scheduler: KarrasDiffusionSchedulers, @@ -221,7 +221,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -229,7 +229,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -328,7 +328,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -675,7 +675,7 @@ def prepare_sparse_control_conditioning( controlnet_frame_indices: int, device: torch.device, dtype: torch.dtype, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: assert conditioning_frames.shape[2] >= len(controlnet_frame_indices) batch_size, channels, _, height, width = conditioning_frames.shape @@ -713,37 +713,37 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: int = 16, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, - conditioning_frames: Optional[List[PipelineImageInput]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, + conditioning_frames: Optional[list[PipelineImageInput]] = None, output_type: str = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, - controlnet_frame_indices: List[int] = [0], + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, + controlnet_frame_indices: list[int] = [0], guess_mode: bool = False, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -758,13 +758,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -780,12 +780,12 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. - conditioning_frames (`List[PipelineImageInput]`, *optional*): + conditioning_frames (`list[PipelineImageInput]`, *optional*): The SparseControlNet input to provide guidance to the `unet` for generation. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. @@ -795,11 +795,11 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - controlnet_frame_indices (`List[int]`): + controlnet_frame_indices (`list[int]`): The indices where the conditioning frames must be applied for generation. Multiple frames can be provided to guide the model to generate similar structure outputs, where the `unet` can "fill-in-the-gaps" for interpolation videos, or a single frame could be provided for general expected @@ -812,7 +812,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py index 6f3a609aba4a..a6f1111e1cff 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection @@ -122,9 +122,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -139,15 +139,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -225,16 +225,14 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, ): @@ -271,7 +269,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -279,7 +277,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -378,7 +376,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -644,7 +642,7 @@ def prepare_latents( timestep: Optional[int] = None, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, decode_chunk_size: int = 16, add_noise: bool = False, @@ -746,40 +744,40 @@ def interrupt(self): @torch.no_grad() def __call__( self, - video: List[List[PipelineImageInput]] = None, - prompt: Optional[Union[str, List[str]]] = None, + video: list[list[PipelineImageInput]] = None, + prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, enforce_inference_steps: bool = False, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.5, strength: float = 0.8, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], decode_chunk_size: int = 16, ): r""" The call function to the pipeline for generation. Args: - video (`List[PipelineImageInput]`): + video (`list[PipelineImageInput]`): The input video to condition the generation on. Must be a list of images/frames of the video. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -788,11 +786,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -801,13 +799,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -823,7 +821,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -843,7 +841,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py index b00f344598ad..8f7513225ad9 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch import torch.nn.functional as F @@ -140,9 +140,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -157,15 +157,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -229,7 +229,7 @@ class AnimateDiffVideoToVideoControlNetPipeline( A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]` or `Tuple[ControlNetModel]` or `MultiControlNetModel`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]` or `tuple[ControlNetModel]` or `MultiControlNetModel`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -247,17 +247,15 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, ): @@ -302,7 +300,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -310,7 +308,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -409,7 +407,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -776,7 +774,7 @@ def prepare_latents( timestep: Optional[int] = None, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, decode_chunk_size: int = 16, add_noise: bool = False, @@ -911,45 +909,45 @@ def interrupt(self): @torch.no_grad() def __call__( self, - video: List[List[PipelineImageInput]] = None, - prompt: Optional[Union[str, List[str]]] = None, + video: list[list[PipelineImageInput]] = None, + prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, enforce_inference_steps: bool = False, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.5, strength: float = 0.8, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, - conditioning_frames: Optional[List[PipelineImageInput]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, + conditioning_frames: Optional[list[PipelineImageInput]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], decode_chunk_size: int = 16, ): r""" The call function to the pipeline for generation. Args: - video (`List[PipelineImageInput]`): + video (`list[PipelineImageInput]`): The input video to condition the generation on. Must be a list of images/frames of the video. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -958,11 +956,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -971,13 +969,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -993,12 +991,12 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. - conditioning_frames (`List[PipelineImageInput]`, *optional*): + conditioning_frames (`list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. @@ -1009,16 +1007,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -1028,7 +1026,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/animatediff/pipeline_output.py b/src/diffusers/pipelines/animatediff/pipeline_output.py index 2417223cf95e..436a20f455bf 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_output.py +++ b/src/diffusers/pipelines/animatediff/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -14,11 +13,11 @@ class AnimateDiffPipelineOutput(BaseOutput): Output class for AnimateDiff pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)` """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] diff --git a/src/diffusers/pipelines/audioldm/pipeline_audioldm.py b/src/diffusers/pipelines/audioldm/pipeline_audioldm.py index 6a70f00c76c7..6cdc1f246367 100644 --- a/src/diffusers/pipelines/audioldm/pipeline_audioldm.py +++ b/src/diffusers/pipelines/audioldm/pipeline_audioldm.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -88,7 +88,7 @@ def __init__( self, vae: AutoencoderKL, text_encoder: ClapTextModelWithProjection, - tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], + tokenizer: RobertaTokenizer | RobertaTokenizerFast, unet: UNet2DConditionModel, scheduler: KarrasDiffusionSchedulers, vocoder: SpeechT5HifiGan, @@ -119,7 +119,7 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device (`torch.device`): torch device @@ -127,7 +127,7 @@ def _encode_prompt( number of waveforms that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the audio generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -189,7 +189,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -361,28 +361,28 @@ def prepare_latents(self, batch_size, num_channels_latents, height, dtype, devic @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, audio_length_in_s: Optional[float] = None, num_inference_steps: int = 10, guidance_scale: float = 2.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_waveforms_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, output_type: Optional[str] = "np", ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. audio_length_in_s (`int`, *optional*, defaults to 5.12): The length of the generated audio sample in seconds. @@ -392,7 +392,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 2.5): A higher guidance scale value encourages the model to generate audio that is closely linked to the text `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in audio generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_waveforms_per_prompt (`int`, *optional*, defaults to 1): @@ -400,7 +400,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py b/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py index b6b40cd6e616..788217cad851 100644 --- a/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py +++ b/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -174,23 +174,23 @@ class AudioLDM2UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoad for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. flip_sin_to_cos (`bool`, *optional*, defaults to `False`): Whether to flip the sin to cos in the time embedding. freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): Block type for middle of UNet, it can only be `UNetMidBlock2DCrossAttn` for AudioLDM2. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): The tuple of upsample blocks to use. - only_cross_attention (`bool` or `Tuple[bool]`, *optional*, default to `False`): + only_cross_attention (`bool` or `tuple[bool]`, *optional*, default to `False`): Whether to include self-attention in the basic transformer blocks, see [`~models.attention.BasicTransformerBlock`]. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. @@ -199,9 +199,9 @@ class AudioLDM2UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoad norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. If `None`, normalization and activation layers is skipped in post-processing. norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. - cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + cross_attention_dim (`int` or `tuple[int]`, *optional*, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + transformer_layers_per_block (`int` or `tuple[int]`, *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. @@ -245,26 +245,26 @@ def __init__( out_channels: int = 4, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D", ), mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", - up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int] = (320, 640, 1280, 1280), - layers_per_block: Union[int, Tuple[int]] = 2, + up_block_types: tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int] = (320, 640, 1280, 1280), + layers_per_block: int | tuple[int] = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, act_fn: str = "silu", norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, - cross_attention_dim: Union[int, Tuple[int]] = 1280, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, - attention_head_dim: Union[int, Tuple[int]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + cross_attention_dim: int | tuple[int] = 1280, + transformer_layers_per_block: int | tuple[int] = 1, + attention_head_dim: int | tuple[int] = 8, + num_attention_heads: Optional[int | tuple[int]] = None, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, num_class_embeds: Optional[int] = None, @@ -532,7 +532,7 @@ def __init__( @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -541,7 +541,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -556,7 +556,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -661,7 +661,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -675,17 +675,17 @@ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[i def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, encoder_hidden_states_1: Optional[torch.Tensor] = None, encoder_attention_mask_1: Optional[torch.Tensor] = None, - ) -> Union[UNet2DConditionOutput, Tuple]: + ) -> UNet2DConditionOutput | tuple: r""" The [`AudioLDM2UNet2DConditionModel`] forward method. @@ -1090,7 +1090,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, encoder_hidden_states_1: Optional[torch.Tensor] = None, encoder_attention_mask_1: Optional[torch.Tensor] = None, @@ -1252,7 +1252,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, encoder_hidden_states_1: Optional[torch.Tensor] = None, encoder_attention_mask_1: Optional[torch.Tensor] = None, @@ -1399,10 +1399,10 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, diff --git a/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py b/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py index 452fc3c01b27..0e5971ea9acf 100644 --- a/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py +++ b/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -196,11 +196,11 @@ def __init__( self, vae: AutoencoderKL, text_encoder: ClapModel, - text_encoder_2: Union[T5EncoderModel, VitsModel], + text_encoder_2: T5EncoderModel | VitsModel, projection_model: AudioLDM2ProjectionModel, language_model: GPT2LMHeadModel, - tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], - tokenizer_2: Union[T5Tokenizer, T5TokenizerFast, VitsTokenizer], + tokenizer: RobertaTokenizer | RobertaTokenizerFast, + tokenizer_2: T5Tokenizer | T5TokenizerFast | VitsTokenizer, feature_extractor: ClapFeatureExtractor, unet: AudioLDM2UNet2DConditionModel, scheduler: KarrasDiffusionSchedulers, @@ -251,7 +251,7 @@ def disable_vae_slicing(self): ) self.vae.disable_slicing() - def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = "cuda"): + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = "cuda"): r""" Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` @@ -316,7 +316,7 @@ def generate_language_model( The sequence used as a prompt for the generation. max_new_tokens (`int`): Number of new tokens to generate. - model_kwargs (`Dict[str, Any]`, *optional*): + model_kwargs (`dict[str, Any]`, *optional*): Ad hoc parametrization of additional model-specific kwargs that will be forwarded to the `forward` function of the model. @@ -373,9 +373,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - transcription (`str` or `List[str]`): + transcription (`str` or `list[str]`): transcription of text to speech device (`torch.device`): torch device @@ -383,7 +383,7 @@ def encode_prompt( number of waveforms that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the audio generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -569,7 +569,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -862,15 +862,15 @@ def prepare_latents(self, batch_size, num_channels_latents, height, dtype, devic @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - transcription: Union[str, List[str]] = None, + prompt: str | list[str] = None, + transcription: str | list[str] = None, audio_length_in_s: Optional[float] = None, num_inference_steps: int = 200, guidance_scale: float = 3.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_waveforms_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -882,16 +882,16 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, output_type: Optional[str] = "np", ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. - transcription (`str` or `List[str]`, *optional*):\ + transcription (`str` or `list[str]`, *optional*):\ The transcript for text to speech. audio_length_in_s (`int`, *optional*, defaults to 10.24): The length of the generated audio sample in seconds. @@ -901,7 +901,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 3.5): A higher guidance scale value encourages the model to generate audio that is closely linked to the text `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in audio generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_waveforms_per_prompt (`int`, *optional*, defaults to 1): @@ -912,7 +912,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/aura_flow/pipeline_aura_flow.py b/src/diffusers/pipelines/aura_flow/pipeline_aura_flow.py index 6251ca443533..fc3c50dd2a85 100644 --- a/src/diffusers/pipelines/aura_flow/pipeline_aura_flow.py +++ b/src/diffusers/pipelines/aura_flow/pipeline_aura_flow.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import T5Tokenizer, UMT5EncoderModel @@ -64,9 +64,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -81,15 +81,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -231,8 +231,8 @@ def check_inputs( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str], + negative_prompt: str | list[str] = None, do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, @@ -247,9 +247,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): @@ -439,15 +439,15 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, num_inference_steps: int = 50, - sigmas: List[float] = None, + sigmas: list[float] = None, guidance_scale: float = 3.5, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = 1024, width: Optional[int] = 1024, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -456,20 +456,20 @@ def __call__( max_sequence_length: int = 256, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - ) -> Union[ImagePipelineOutput, Tuple]: + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + ) -> ImagePipelineOutput | tuple: r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -480,7 +480,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. guidance_scale (`float`, *optional*, defaults to 5.0): @@ -491,7 +491,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -524,7 +524,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 8a32d4c367a3..52e8e0a40111 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -353,7 +353,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -375,7 +375,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn’t need to be defined for each parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the same device. @@ -644,7 +644,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -666,7 +666,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn’t need to be defined for each parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the same device. @@ -950,7 +950,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -972,7 +972,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. - device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + device_map (`str` or `dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn’t need to be defined for each parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the same device. diff --git a/src/diffusers/pipelines/blip_diffusion/__init__.py b/src/diffusers/pipelines/blip_diffusion/__init__.py index af6c879d5ce8..e726d40a78d5 100644 --- a/src/diffusers/pipelines/blip_diffusion/__init__.py +++ b/src/diffusers/pipelines/blip_diffusion/__init__.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional, Union import numpy as np import PIL diff --git a/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py b/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py index e45f431d0b9d..3685b7cbb2d5 100644 --- a/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py +++ b/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -from typing import Dict, List, Optional, Union +from typing import Optional import numpy as np import torch @@ -69,11 +69,11 @@ class BlipImageProcessor(BaseImageProcessor): do_normalize (`bool`, *optional*, defaults to `True`): Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. - image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): + image_mean (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): + image_std (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): Standard deviation to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. Can be overridden by the `image_std` parameter in the `preprocess` method. @@ -86,13 +86,13 @@ class BlipImageProcessor(BaseImageProcessor): def __init__( self, do_resize: bool = True, - size: Dict[str, int] = None, + size: dict[str, int] = None, resample: PILImageResampling = PILImageResampling.BICUBIC, do_rescale: bool = True, - rescale_factor: Union[int, float] = 1 / 255, + rescale_factor: int | float = 1 / 255, do_normalize: bool = True, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, + image_mean: Optional[float | list[float]] = None, + image_std: Optional[float | list[float]] = None, do_convert_rgb: bool = True, do_center_crop: bool = True, **kwargs, @@ -116,10 +116,10 @@ def __init__( def resize( self, image: np.ndarray, - size: Dict[str, int], + size: dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, - data_format: Optional[Union[str, ChannelDimension]] = None, - input_data_format: Optional[Union[str, ChannelDimension]] = None, + data_format: Optional[str | ChannelDimension] = None, + input_data_format: Optional[str | ChannelDimension] = None, **kwargs, ) -> np.ndarray: """ @@ -128,7 +128,7 @@ def resize( Args: image (`np.ndarray`): Image to resize. - size (`Dict[str, int]`): + size (`dict[str, int]`): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. @@ -165,18 +165,18 @@ def preprocess( self, images: ImageInput, do_resize: Optional[bool] = None, - size: Optional[Dict[str, int]] = None, + size: Optional[dict[str, int]] = None, resample: PILImageResampling = None, do_rescale: Optional[bool] = None, do_center_crop: Optional[bool] = None, rescale_factor: Optional[float] = None, do_normalize: Optional[bool] = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - return_tensors: Optional[Union[str, TensorType]] = None, + image_mean: Optional[float | list[float]] = None, + image_std: Optional[float | list[float]] = None, + return_tensors: Optional[str | TensorType] = None, do_convert_rgb: bool = None, data_format: ChannelDimension = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[str | ChannelDimension] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -188,7 +188,7 @@ def preprocess( passing in images with pixel values between 0 and 1, set `do_rescale=False`. do_resize (`bool`, *optional*, defaults to `self.do_resize`): Whether to resize the image. - size (`Dict[str, int]`, *optional*, defaults to `self.size`): + size (`dict[str, int]`, *optional*, defaults to `self.size`): Controls the size of the image after `resize`. The shortest edge of the image is resized to `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest @@ -201,9 +201,9 @@ def preprocess( Rescale factor to rescale the image by if `do_rescale` is set to `True`. do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`): Image mean to normalize the image by if `do_normalize` is set to `True`. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`): Image standard deviation to normalize the image by if `do_normalize` is set to `True`. do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): Whether to convert the image to RGB. diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py b/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py index b061ac2636a5..9294e6a9767c 100644 --- a/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py +++ b/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import torch from torch import nn @@ -376,7 +376,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: + ) -> tuple | BaseModelOutputWithPooling: r""" Returns: @@ -464,7 +464,7 @@ class PreTrainedModel def get_extended_attention_mask( self, attention_mask: torch.Tensor, - input_shape: Tuple[int], + input_shape: tuple[int], device: torch.device, has_query: bool = False, ) -> torch.Tensor: @@ -474,7 +474,7 @@ def get_extended_attention_mask( Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. - input_shape (`Tuple[int]`): + input_shape (`tuple[int]`): The shape of the input to the model. device (`torch.device`): The device of the input to the model. diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py b/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py index 1b0342ce7a56..9bd007a53f42 100644 --- a/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +++ b/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple, Union +from typing import Optional import torch from torch import nn @@ -60,7 +60,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: + ) -> tuple | BaseModelOutputWithPooling: return self.text_model( ctx_embeddings=ctx_embeddings, ctx_begin_pos=ctx_begin_pos, @@ -92,7 +92,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: + ) -> tuple | BaseModelOutputWithPooling: r""" Returns: diff --git a/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py b/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py index 705d930b59fe..f453785874b1 100644 --- a/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +++ b/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Union +from typing import Optional import PIL.Image import torch @@ -116,8 +116,8 @@ def __init__( qformer: Blip2QFormerModel, image_processor: BlipImageProcessor, ctx_begin_pos: int = 2, - mean: List[float] = None, - std: List[float] = None, + mean: list[float] = None, + std: list[float] = None, ): super().__init__() @@ -193,16 +193,16 @@ def encode_prompt(self, query_embeds, prompt, device=None): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: List[str], + prompt: list[str], reference_image: PIL.Image.Image, - source_subject_category: List[str], - target_subject_category: List[str], + source_subject_category: list[str], + target_subject_category: list[str], latents: Optional[torch.Tensor] = None, guidance_scale: float = 7.5, height: int = 512, width: int = 512, num_inference_steps: int = 50, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, neg_prompt: Optional[str] = "", prompt_strength: float = 1.0, prompt_reps: int = 20, @@ -213,13 +213,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`List[str]`): + prompt (`list[str]`): The prompt or prompts to guide the image generation. reference_image (`PIL.Image.Image`): The reference image to condition the generation on. - source_subject_category (`List[str]`): + source_subject_category (`list[str]`): The source subject category. - target_subject_category (`List[str]`): + target_subject_category (`list[str]`): The target subject category. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image @@ -238,7 +238,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. neg_prompt (`str`, *optional*, defaults to ""): diff --git a/src/diffusers/pipelines/bria/pipeline_bria.py b/src/diffusers/pipelines/bria/pipeline_bria.py index ebddfb0c0eee..fb7590e2da34 100644 --- a/src/diffusers/pipelines/bria/pipeline_bria.py +++ b/src/diffusers/pipelines/bria/pipeline_bria.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -116,7 +116,7 @@ class BriaPipeline(DiffusionPipeline): def __init__( self, transformer: BriaTransformer2DModel, - scheduler: Union[FlowMatchEulerDiscreteScheduler, KarrasDiffusionSchedulers], + scheduler: FlowMatchEulerDiscreteScheduler | KarrasDiffusionSchedulers, vae: AutoencoderKL, text_encoder: T5EncoderModel, tokenizer: T5TokenizerFast, @@ -145,11 +145,11 @@ def __init__( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, max_sequence_length: int = 128, @@ -158,7 +158,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -166,7 +166,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -320,7 +320,7 @@ def check_inputs( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 128, device: Optional[torch.device] = None, @@ -449,32 +449,32 @@ def _prepare_latent_image_ids(batch_size, height, width, device, dtype): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 30, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 128, - clip_value: Union[None, float] = None, + clip_value: None | float = None, normalize: bool = False, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -484,7 +484,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -494,13 +494,13 @@ def __call__( Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -528,7 +528,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/bria/pipeline_output.py b/src/diffusers/pipelines/bria/pipeline_output.py index 54eed0623371..4bf2ed949c26 100644 --- a/src/diffusers/pipelines/bria/pipeline_output.py +++ b/src/diffusers/pipelines/bria/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class BriaPipelineOutput(BaseOutput): Output class for Bria pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/chroma/pipeline_chroma.py b/src/diffusers/pipelines/chroma/pipeline_chroma.py index 5482035b3afb..b3da5c2f4675 100644 --- a/src/diffusers/pipelines/chroma/pipeline_chroma.py +++ b/src/diffusers/pipelines/chroma/pipeline_chroma.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -92,9 +92,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -109,15 +109,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -208,7 +208,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -261,8 +261,8 @@ def _get_t5_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str], + negative_prompt: str | list[str] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -276,9 +276,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). device: (`torch.device`): @@ -639,39 +639,39 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 35, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 5.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is not greater than `1`). @@ -682,7 +682,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -694,7 +694,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -705,13 +705,13 @@ def __call__( Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -741,7 +741,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py b/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py index 9afd4b9e1577..cc2e58b58323 100644 --- a/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py +++ b/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -104,9 +104,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -121,15 +121,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -222,7 +222,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -290,8 +290,8 @@ def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str], + negative_prompt: str | list[str] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -305,9 +305,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). device: (`torch.device`): @@ -700,41 +700,41 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 35, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 5.0, strength: float = 0.9, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is not greater than `1`). @@ -745,7 +745,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 35): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -763,7 +763,7 @@ def __call__( A value of 1, therefore, essentially ignores image. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -774,13 +774,13 @@ def __call__( Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -810,7 +810,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/chroma/pipeline_output.py b/src/diffusers/pipelines/chroma/pipeline_output.py index 951d132dba2e..229b0fe42b90 100644 --- a/src/diffusers/pipelines/chroma/pipeline_output.py +++ b/src/diffusers/pipelines/chroma/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class ChromaPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox.py b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox.py index 4ac33b24bbe1..7897641c3c63 100644 --- a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox.py +++ b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox.py @@ -15,7 +15,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -88,9 +88,9 @@ def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -105,15 +105,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -182,7 +182,7 @@ def __init__( text_encoder: T5EncoderModel, vae: AutoencoderKLCogVideoX, transformer: CogVideoXTransformer3DModel, - scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + scheduler: CogVideoXDDIMScheduler | CogVideoXDPMScheduler, ): super().__init__() @@ -201,7 +201,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -243,8 +243,8 @@ def _get_t5_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -257,9 +257,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -444,7 +444,7 @@ def _prepare_rotary_positional_embeddings( width: int, num_frames: int, device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) @@ -506,38 +506,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 6, use_dynamic_cfg: bool = False, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 226, - ) -> Union[CogVideoXPipelineOutput, Tuple]: + ) -> CogVideoXPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -553,7 +553,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -565,7 +565,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -594,7 +594,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py index c1335839f848..94f37f200af1 100644 --- a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py +++ b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py @@ -15,7 +15,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from PIL import Image @@ -95,9 +95,9 @@ def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -112,15 +112,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -209,7 +209,7 @@ def __init__( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -252,8 +252,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -266,9 +266,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -361,7 +361,7 @@ def prepare_latents( # Adapted from https://github.com/aigc-apps/CogVideoX-Fun/blob/2a93e5c14e02b2b5921d533fd59fc8c0ed69fb24/cogvideox/pipeline/pipeline_cogvideox_control.py#L366 def prepare_control_latents( self, mask: Optional[torch.Tensor] = None, masked_image: Optional[torch.Tensor] = None - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: if mask is not None: masks = [] for i in range(mask.size(0)): @@ -490,7 +490,7 @@ def _prepare_rotary_positional_embeddings( width: int, num_frames: int, device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) @@ -552,43 +552,43 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, - control_video: Optional[List[Image.Image]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, + control_video: Optional[list[Image.Image]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 6, use_dynamic_cfg: bool = False, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, control_video_latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 226, - ) -> Union[CogVideoXPipelineOutput, Tuple]: + ) -> CogVideoXPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - control_video (`List[PIL.Image.Image]`): + control_video (`list[PIL.Image.Image]`): The control video to condition the generation on. Must be a list of images/frames of the video. If not provided, `control_video_latents` must be provided. height (`int`, *optional*, defaults to self.transformer.config.sample_height * self.vae_scale_factor_spatial): @@ -598,7 +598,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -610,7 +610,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -642,7 +642,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py index c523c9adec98..e124cec1a301 100644 --- a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py +++ b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py @@ -15,7 +15,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL import torch @@ -87,9 +87,9 @@ def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -104,15 +104,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -195,7 +195,7 @@ def __init__( text_encoder: T5EncoderModel, vae: AutoencoderKLCogVideoX, transformer: CogVideoXTransformer3DModel, - scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + scheduler: CogVideoXDDIMScheduler | CogVideoXDPMScheduler, ): super().__init__() @@ -219,7 +219,7 @@ def __init__( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -262,8 +262,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -276,9 +276,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -470,7 +470,7 @@ def check_inputs( and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -537,7 +537,7 @@ def _prepare_rotary_positional_embeddings( width: int, num_frames: int, device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) @@ -600,40 +600,40 @@ def interrupt(self): def __call__( self, image: PipelineImageInput, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: int = 49, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 6, use_dynamic_cfg: bool = False, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 226, - ) -> Union[CogVideoXPipelineOutput, Tuple]: + ) -> CogVideoXPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -649,7 +649,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -661,7 +661,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -690,7 +690,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py index 897dc6d1b70a..41e0db25714f 100644 --- a/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py +++ b/src/diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py @@ -15,7 +15,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from PIL import Image @@ -96,9 +96,9 @@ def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -113,15 +113,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -204,7 +204,7 @@ def __init__( text_encoder: T5EncoderModel, vae: AutoencoderKLCogVideoX, transformer: CogVideoXTransformer3DModel, - scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + scheduler: CogVideoXDDIMScheduler | CogVideoXDPMScheduler, ): super().__init__() @@ -225,7 +225,7 @@ def __init__( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -268,8 +268,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -282,9 +282,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -514,7 +514,7 @@ def _prepare_rotary_positional_embeddings( width: int, num_frames: int, device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) @@ -576,41 +576,41 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - video: List[Image.Image] = None, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + video: list[Image.Image] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, strength: float = 0.8, guidance_scale: float = 6, use_dynamic_cfg: bool = False, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 226, - ) -> Union[CogVideoXPipelineOutput, Tuple]: + ) -> CogVideoXPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - video (`List[PIL.Image.Image]`): + video (`list[PIL.Image.Image]`): The input video to condition the generation on. Must be a list of images/frames of the video. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -621,7 +621,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -635,7 +635,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -664,7 +664,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogvideo/pipeline_output.py b/src/diffusers/pipelines/cogvideo/pipeline_output.py index 3de030dd6928..56ff50132231 100644 --- a/src/diffusers/pipelines/cogvideo/pipeline_output.py +++ b/src/diffusers/pipelines/cogvideo/pipeline_output.py @@ -11,8 +11,8 @@ class CogVideoXPipelineOutput(BaseOutput): Output class for CogVideo pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/cogview3/pipeline_cogview3plus.py b/src/diffusers/pipelines/cogview3/pipeline_cogview3plus.py index 304a5c5ad00b..0dead09aa2d4 100644 --- a/src/diffusers/pipelines/cogview3/pipeline_cogview3plus.py +++ b/src/diffusers/pipelines/cogview3/pipeline_cogview3plus.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -59,9 +59,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -76,15 +76,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -153,7 +153,7 @@ def __init__( text_encoder: T5EncoderModel, vae: AutoencoderKL, transformer: CogView3PlusTransformer2DModel, - scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + scheduler: CogVideoXDDIMScheduler | CogVideoXDPMScheduler, ): super().__init__() @@ -167,7 +167,7 @@ def __init__( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds with num_videos_per_prompt->num_images_per_prompt def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -209,8 +209,8 @@ def _get_t5_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -223,9 +223,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -408,36 +408,36 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 5.0, num_images_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), output_type: str = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 224, - ) -> Union[CogView3PipelineOutput, Tuple]: + ) -> CogView3PipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -448,7 +448,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -460,7 +460,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to `1`): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -474,12 +474,12 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of @@ -499,7 +499,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogview3/pipeline_output.py b/src/diffusers/pipelines/cogview3/pipeline_output.py index 3891dd51e691..6c89e117b74c 100644 --- a/src/diffusers/pipelines/cogview3/pipeline_output.py +++ b/src/diffusers/pipelines/cogview3/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class CogView3PipelineOutput(BaseOutput): Output class for CogView3 pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/cogview4/pipeline_cogview4.py b/src/diffusers/pipelines/cogview4/pipeline_cogview4.py index 22510f5d9d50..b4fe82d99a20 100644 --- a/src/diffusers/pipelines/cogview4/pipeline_cogview4.py +++ b/src/diffusers/pipelines/cogview4/pipeline_cogview4.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -70,9 +70,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -87,15 +87,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) @@ -177,7 +177,7 @@ def __init__( def _get_glm_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 1024, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -220,8 +220,8 @@ def _get_glm_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -234,9 +234,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -403,37 +403,37 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 5.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 1024, - ) -> Union[CogView4PipelineOutput, Tuple]: + ) -> CogView4PipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -444,11 +444,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -460,7 +460,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to `1`): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -474,12 +474,12 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of @@ -499,7 +499,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogview4/pipeline_cogview4_control.py b/src/diffusers/pipelines/cogview4/pipeline_cogview4_control.py index e26b7ba415de..30fec730530d 100644 --- a/src/diffusers/pipelines/cogview4/pipeline_cogview4_control.py +++ b/src/diffusers/pipelines/cogview4/pipeline_cogview4_control.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -72,9 +72,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -89,15 +89,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) @@ -180,7 +180,7 @@ def __init__( # Copied from diffusers.pipelines.cogview4.pipeline_cogview4.CogView4Pipeline._get_glm_embeds def _get_glm_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 1024, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -224,8 +224,8 @@ def _get_glm_embeds( # Copied from diffusers.pipelines.cogview4.pipeline_cogview4.CogView4Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -238,9 +238,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -435,38 +435,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, control_image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 5.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 1024, - ) -> Union[CogView4PipelineOutput, Tuple]: + ) -> CogView4PipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -477,11 +477,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -493,7 +493,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to `1`): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -507,12 +507,12 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of @@ -532,7 +532,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cogview4/pipeline_output.py b/src/diffusers/pipelines/cogview4/pipeline_output.py index 4efec1310845..997444c6c009 100644 --- a/src/diffusers/pipelines/cogview4/pipeline_output.py +++ b/src/diffusers/pipelines/cogview4/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class CogView4PipelineOutput(BaseOutput): Output class for CogView3 pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/consisid/consisid_utils.py b/src/diffusers/pipelines/consisid/consisid_utils.py index 521d4d787e54..c1646e15efbc 100644 --- a/src/diffusers/pipelines/consisid/consisid_utils.py +++ b/src/diffusers/pipelines/consisid/consisid_utils.py @@ -135,7 +135,7 @@ def process_face_embeddings( is_align_face: Boolean flag indicating whether face alignment should be performed. Returns: - Tuple: + tuple: - id_cond: Concatenated tensor of Ante face embedding and CLIP vision embedding - id_vit_hidden: Hidden state of the CLIP vision model, a list of tensors. - return_face_features_image_2: Processed face features image after normalization and parsing. @@ -245,7 +245,7 @@ def process_face_embeddings_infer( is_align_face: Boolean flag indicating whether face alignment should be performed (default: True). Returns: - Tuple: + tuple: - id_cond: Concatenated tensor of Ante face embedding and CLIP vision embedding. - id_vit_hidden: Hidden state of the CLIP vision model, a list of tensors. - image: Processed face image after feature extraction and alignment. diff --git a/src/diffusers/pipelines/consisid/pipeline_consisid.py b/src/diffusers/pipelines/consisid/pipeline_consisid.py index 3e6c149d7f80..e6c7c73de378 100644 --- a/src/diffusers/pipelines/consisid/pipeline_consisid.py +++ b/src/diffusers/pipelines/consisid/pipeline_consisid.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL @@ -102,7 +102,7 @@ def draw_kps(image_pil, kps, color_list=[(255, 0, 0), (0, 255, 0), (0, 0, 255), Parameters: - image_pil (PIL.Image): Input image as a PIL object. - kps (list of tuples): A list of keypoints where each keypoint is a tuple of (x, y) coordinates. - - color_list (list of tuples, optional): List of colors (in RGB format) for each keypoint. Default is a set of five + - color_list (list of tuples, optional): list of colors (in RGB format) for each keypoint. Default is a set of five colors. Returns: @@ -177,9 +177,9 @@ def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -194,15 +194,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -311,7 +311,7 @@ def __init__( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -354,8 +354,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -368,9 +368,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -572,7 +572,7 @@ def check_inputs( and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -623,7 +623,7 @@ def _prepare_rotary_positional_embeddings( width: int, num_frames: int, device: torch.device, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) base_size_width = self.transformer.config.sample_width // self.transformer.config.patch_size @@ -663,8 +663,8 @@ def interrupt(self): def __call__( self, image: PipelineImageInput, - prompt: Optional[Union[str, List[str]]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 480, width: int = 720, num_frames: int = 49, @@ -673,32 +673,32 @@ def __call__( use_dynamic_cfg: bool = False, num_videos_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: str = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 226, id_vit_hidden: Optional[torch.Tensor] = None, id_cond: Optional[torch.Tensor] = None, kps_cond: Optional[torch.Tensor] = None, - ) -> Union[ConsisIDPipelineOutput, Tuple]: + ) -> ConsisIDPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -727,7 +727,7 @@ def __call__( more faithful image generation, while later steps reduce it for more diverse and natural results. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -756,7 +756,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/consisid/pipeline_output.py b/src/diffusers/pipelines/consisid/pipeline_output.py index dd4a63aa50b9..83a5be8d230b 100644 --- a/src/diffusers/pipelines/consisid/pipeline_output.py +++ b/src/diffusers/pipelines/consisid/pipeline_output.py @@ -11,8 +11,8 @@ class ConsisIDPipelineOutput(BaseOutput): Output class for ConsisID pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py b/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py index 1fbdeb1f2741..3f8914bfaa71 100644 --- a/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py +++ b/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch @@ -173,10 +173,10 @@ def check_inputs(self, num_inference_steps, timesteps, latents, batch_size, img_ def __call__( self, batch_size: int = 1, - class_labels: Optional[Union[torch.Tensor, List[int], int]] = None, + class_labels: Optional[torch.Tensor | list[int] | int] = None, num_inference_steps: int = 1, - timesteps: List[int] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + timesteps: list[int] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -187,13 +187,13 @@ def __call__( Args: batch_size (`int`, *optional*, defaults to 1): The number of images to generate. - class_labels (`torch.Tensor` or `List[int]` or `int`, *optional*): + class_labels (`torch.Tensor` or `list[int]` or `int`, *optional*): Optional class labels for conditioning class-conditional consistency models. Not used if the model is not class-conditional. num_inference_steps (`int`, *optional*, defaults to 1): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. generator (`torch.Generator`, *optional*): diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index fe0e69314cca..ac15cf78c5b1 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -103,9 +103,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -120,15 +120,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -189,7 +189,7 @@ class StableDiffusionControlNetPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -215,7 +215,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, @@ -311,7 +311,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -319,7 +319,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -418,7 +418,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -908,45 +908,45 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -962,18 +962,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -981,7 +981,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -995,7 +995,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1014,16 +1014,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -1033,7 +1033,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py index e0f1879405aa..52ddddfdb291 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Union +from typing import Optional import PIL.Image import torch @@ -126,8 +126,8 @@ def __init__( controlnet: ControlNetModel, image_processor: BlipImageProcessor, ctx_begin_pos: int = 2, - mean: List[float] = None, - std: List[float] = None, + mean: list[float] = None, + std: list[float] = None, ): super().__init__() @@ -241,17 +241,17 @@ def prepare_control_image( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: List[str], + prompt: list[str], reference_image: PIL.Image.Image, condtioning_image: PIL.Image.Image, - source_subject_category: List[str], - target_subject_category: List[str], + source_subject_category: list[str], + target_subject_category: list[str], latents: Optional[torch.Tensor] = None, guidance_scale: float = 7.5, height: int = 512, width: int = 512, num_inference_steps: int = 50, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, neg_prompt: Optional[str] = "", prompt_strength: float = 1.0, prompt_reps: int = 20, @@ -262,15 +262,15 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`List[str]`): + prompt (`list[str]`): The prompt or prompts to guide the image generation. reference_image (`PIL.Image.Image`): The reference image to condition the generation on. condtioning_image (`PIL.Image.Image`): The conditioning canny edge image to condition the generation on. - source_subject_category (`List[str]`): + source_subject_category (`list[str]`): The source subject category. - target_subject_category (`List[str]`): + target_subject_category (`list[str]`): The target subject category. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image @@ -291,7 +291,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. neg_prompt (`str`, *optional*, defaults to ""): diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index 12cc6f630d80..5fc035b495b1 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -167,7 +167,7 @@ class StableDiffusionControlNetImg2ImgPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -193,7 +193,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, @@ -289,7 +289,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -297,7 +297,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -396,7 +396,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -906,7 +906,7 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, height: Optional[int] = None, @@ -914,41 +914,41 @@ def __call__( strength: float = 0.8, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.8, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The initial image to be used as the starting point for the image generation process. Can also accept image latents as `image`, and if passing latents directly they are not encoded again. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -971,7 +971,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -979,7 +979,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -993,7 +993,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1006,16 +1006,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -1025,7 +1025,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index 6de8e5747b02..f10b24dd33e1 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -15,7 +15,7 @@ # This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -163,7 +163,7 @@ class StableDiffusionControlNetInpaintPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -196,7 +196,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, @@ -295,7 +295,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -303,7 +303,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -402,7 +402,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -995,7 +995,7 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, control_image: PipelineImageInput = None, @@ -1005,52 +1005,52 @@ def __call__( strength: float = 1.0, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.5, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.5, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, - `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, + `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, NumPy array or tensor representing an image batch to be used as the starting point. For both NumPy array and PyTorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a NumPy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, - `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, + `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, NumPy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a NumPy array or PyTorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for PyTorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for NumPy array, it would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - control_image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, - `List[List[torch.Tensor]]`, or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, + `list[list[torch.Tensor]]`, or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1080,7 +1080,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1088,7 +1088,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1102,7 +1102,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1115,16 +1115,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 0.5): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -1134,7 +1134,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 397ab15715c2..20aedca1821d 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -248,7 +248,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, @@ -312,9 +312,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -323,11 +323,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -445,7 +445,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1179,14 +1179,11 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, - control_image: Union[ - PipelineImageInput, - List[PipelineImageInput], - ] = None, + control_image: PipelineImageInput | list[PipelineImageInput] = None, height: Optional[int] = None, width: Optional[int] = None, padding_mask_crop: Optional[int] = None, @@ -1195,46 +1192,46 @@ def __call__( denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders image (`PIL.Image.Image`): @@ -1288,11 +1285,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -1303,7 +1300,7 @@ def __call__( weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1337,17 +1334,17 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). @@ -1367,7 +1364,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py index 8b37d38b9099..7eddb0ce7d89 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -125,9 +125,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -142,15 +142,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -216,7 +216,7 @@ class StableDiffusionXLControlNetPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -261,7 +261,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, @@ -320,9 +320,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -331,11 +331,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -453,7 +453,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1011,59 +1011,59 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1081,11 +1081,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1099,10 +1099,10 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1110,7 +1110,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1131,7 +1131,7 @@ def __call__( weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1144,42 +1144,42 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1192,7 +1192,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py index 4d4845c5a0a3..14afe1d5042d 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -204,7 +204,7 @@ class StableDiffusionXLControlNetImg2ImgPipeline( Second Tokenizer of class [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -253,7 +253,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, @@ -314,9 +314,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -325,11 +325,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -447,7 +447,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1091,8 +1091,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, height: Optional[int] = None, @@ -1100,56 +1100,56 @@ def __call__( strength: float = 0.8, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.8, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The initial image will be used as the starting point for the image generation process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height @@ -1179,11 +1179,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1191,7 +1191,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1213,7 +1213,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1228,42 +1228,42 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original unet. If multiple ControlNets are specified in init, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): In this mode, the ControlNet encoder will try best to recognize the content of the input image even if you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the controlnet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the controlnet stops applying. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1284,7 +1284,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py index fb58b222112a..937b4f845e39 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -235,9 +235,10 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ - ControlNetUnionModel, List[ControlNetUnionModel], Tuple[ControlNetUnionModel], MultiControlNetUnionModel - ], + controlnet: ControlNetUnionModel + | list[ControlNetUnionModel] + | tuple[ControlNetUnionModel] + | MultiControlNetUnionModel, scheduler: KarrasDiffusionSchedulers, requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, @@ -301,9 +302,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -312,11 +313,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -434,7 +435,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1158,11 +1159,11 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, - control_image: Union[PipelineImageInput, List[PipelineImageInput]] = None, + control_image: PipelineImageInput | list[PipelineImageInput] = None, height: Optional[int] = None, width: Optional[int] = None, padding_mask_crop: Optional[int] = None, @@ -1171,47 +1172,47 @@ def __call__( denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - control_mode: Optional[Union[int, List[int], List[List[int]]]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + control_mode: Optional[int | list[int] | list[list[int]]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders image (`PIL.Image.Image`): @@ -1222,7 +1223,7 @@ def __call__( repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`. - control_image (`PipelineImageInput` or `List[PipelineImageInput]`, *optional*): + control_image (`PipelineImageInput` or `list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1272,11 +1273,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -1287,7 +1288,7 @@ def __call__( weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1321,33 +1322,33 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_mode (`int` or `List[int]` or `List[List[int]], *optional*): + control_mode (`int` or `list[int]` or `list[list[int]], *optional*): The control condition types for the ControlNet. See the ControlNet's model card forinformation on the available control modes. If multiple ControlNets are specified in `init`, control_mode should be a list where each ControlNet should have its corresponding control mode list. Should reflect the order of conditions in control_image. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). @@ -1367,7 +1368,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py index 5961d389effb..85e52443c85d 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -119,9 +119,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -136,15 +136,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -249,9 +249,10 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ - ControlNetUnionModel, List[ControlNetUnionModel], Tuple[ControlNetUnionModel], MultiControlNetUnionModel - ], + controlnet: ControlNetUnionModel + | list[ControlNetUnionModel] + | tuple[ControlNetUnionModel] + | MultiControlNetUnionModel, scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, @@ -310,9 +311,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -321,11 +322,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -443,7 +444,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -986,58 +987,58 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - control_image: Union[PipelineImageInput, List[PipelineImageInput]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + control_image: PipelineImageInput | list[PipelineImageInput] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - control_mode: Optional[Union[int, List[int], List[List[int]]]] = None, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + control_mode: Optional[int | list[int] | list[list[int]]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders. - control_image (`PipelineImageInput` or `List[PipelineImageInput]`, *optional*): + control_image (`PipelineImageInput` or `list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1055,11 +1056,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1073,10 +1074,10 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1084,7 +1085,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1105,7 +1106,7 @@ def __call__( weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1118,47 +1119,47 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_mode (`int` or `List[int]` or `List[List[int]], *optional*): + control_mode (`int` or `list[int]` or `list[list[int]], *optional*): The control condition types for the ControlNet. See the ControlNet's model card forinformation on the available control modes. If multiple ControlNets are specified in `init`, control_mode should be a list where each ControlNet should have its corresponding control mode list. Should reflect the order of conditions in control_image. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1171,7 +1172,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py index 8fedb6d8609a..cbe6db034c07 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -267,9 +267,10 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ - ControlNetUnionModel, List[ControlNetUnionModel], Tuple[ControlNetUnionModel], MultiControlNetUnionModel - ], + controlnet: ControlNetUnionModel + | list[ControlNetUnionModel] + | tuple[ControlNetUnionModel] + | MultiControlNetUnionModel, scheduler: KarrasDiffusionSchedulers, requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, @@ -330,9 +331,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -341,11 +342,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -463,7 +464,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1079,65 +1080,65 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, - control_image: Union[PipelineImageInput, List[PipelineImageInput]] = None, + control_image: PipelineImageInput | list[PipelineImageInput] = None, height: Optional[int] = None, width: Optional[int] = None, strength: float = 0.8, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.8, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - control_mode: Optional[Union[int, List[int], List[List[int]]]] = None, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + control_mode: Optional[int | list[int] | list[list[int]]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The initial image will be used as the starting point for the image generation process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. - control_image (`PipelineImageInput` or `List[PipelineImageInput]`, *optional*): + control_image (`PipelineImageInput` or `list[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1167,11 +1168,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1179,7 +1180,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1201,7 +1202,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1216,47 +1217,47 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): In this mode, the ControlNet encoder will try best to recognize the content of the input image even if you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_mode (`int` or `List[int]` or `List[List[int]], *optional*): + control_mode (`int` or `list[int]` or `list[list[int]], *optional*): The control condition types for the ControlNet. See the ControlNet's model card forinformation on the available control modes. If multiple ControlNets are specified in `init`, control_mode should be a list where each ControlNet should have its corresponding control mode list. Should reflect the order of conditions in control_image - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1277,7 +1278,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py index d4c6f336dfef..aa901c6f725c 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py @@ -14,7 +14,7 @@ import warnings from functools import partial -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import jax import jax.numpy as jnp @@ -148,9 +148,7 @@ def __init__( tokenizer: CLIPTokenizer, unet: FlaxUNet2DConditionModel, controlnet: FlaxControlNetModel, - scheduler: Union[ - FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler - ], + scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler, safety_checker: FlaxStableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, dtype: jnp.dtype = jnp.float32, @@ -180,7 +178,7 @@ def __init__( ) self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 - def prepare_text_inputs(self, prompt: Union[str, List[str]]): + def prepare_text_inputs(self, prompt: str | list[str]): if not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") @@ -194,7 +192,7 @@ def prepare_text_inputs(self, prompt: Union[str, List[str]]): return text_input.input_ids - def prepare_image_inputs(self, image: Union[Image.Image, List[Image.Image]]): + def prepare_image_inputs(self, image: Image.Image | list[Image.Image]): if not isinstance(image, (Image.Image, list)): raise ValueError(f"image has to be of type `PIL.Image.Image` or list but is {type(image)}") @@ -243,7 +241,7 @@ def _generate( self, prompt_ids: jnp.ndarray, image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int, guidance_scale: float, @@ -353,13 +351,13 @@ def __call__( self, prompt_ids: jnp.ndarray, image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int = 50, - guidance_scale: Union[float, jnp.ndarray] = 7.5, + guidance_scale: float | jnp.ndarray = 7.5, latents: jnp.ndarray = None, neg_prompt_ids: jnp.ndarray = None, - controlnet_conditioning_scale: Union[float, jnp.ndarray] = 1.0, + controlnet_conditioning_scale: float | jnp.ndarray = 1.0, return_dict: bool = True, jit: bool = False, ): diff --git a/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py b/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py index 2b5684de9511..1e941573fec5 100644 --- a/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +++ b/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -189,7 +189,7 @@ class HunyuanDiTControlNetPipeline(DiffusionPipeline): The tokenizer for the mT5 embedder. scheduler ([`DDPMScheduler`]): A scheduler to be used in combination with HunyuanDiT to denoise the encoded image latents. - controlnet ([`HunyuanDiT2DControlNetModel`] or `List[HunyuanDiT2DControlNetModel]` or [`HunyuanDiT2DControlNetModel`]): + controlnet ([`HunyuanDiT2DControlNetModel`] or `list[HunyuanDiT2DControlNetModel]` or [`HunyuanDiT2DControlNetModel`]): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -222,12 +222,10 @@ def __init__( scheduler: DDPMScheduler, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, - controlnet: Union[ - HunyuanDiT2DControlNetModel, - List[HunyuanDiT2DControlNetModel], - Tuple[HunyuanDiT2DControlNetModel], - HunyuanDiT2DMultiControlNetModel, - ], + controlnet: HunyuanDiT2DControlNetModel + | list[HunyuanDiT2DControlNetModel] + | tuple[HunyuanDiT2DControlNetModel] + | HunyuanDiT2DMultiControlNetModel, text_encoder_2: Optional[T5EncoderModel] = None, tokenizer_2: Optional[MT5Tokenizer] = None, requires_safety_checker: bool = True, @@ -294,7 +292,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -304,7 +302,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -393,7 +391,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -635,17 +633,17 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, control_image: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_2: Optional[torch.Tensor] = None, @@ -658,20 +656,20 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = (1024, 1024), - target_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = (1024, 1024), + target_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), use_resolution_binning: bool = True, ): r""" The call function to the pipeline for generation with HunyuanDiT. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`): The height in pixels of the generated image. @@ -683,23 +681,23 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -707,7 +705,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -737,17 +735,17 @@ def __call__( plain tuple. callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A callback function or a list of callback functions to be called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): A list of tensor inputs that should be passed to the callback function. If not defined, all tensor inputs will be passed. guidance_rescale (`float`, *optional*, defaults to 0.0): Rescale the noise_cfg according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). See Section 3.4 - original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): + original_size (`tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): The original size of the image. Used to calculate the time ids. - target_size (`Tuple[int, int]`, *optional*): + target_size (`tuple[int, int]`, *optional*): The target size of the image. Used to calculate the time ids. - crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`): + crops_coords_top_left (`tuple[int, int]`, *optional*, defaults to `(0, 0)`): The top left coordinates of the crop. Used to calculate the time ids. use_resolution_binning (`bool`, *optional*, defaults to `True`): Whether to use resolution binning or not. If `True`, the input resolution will be mapped to the closest diff --git a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py index c763411ab5f7..89635de049b7 100644 --- a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +++ b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -84,9 +84,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -101,15 +101,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -174,7 +174,7 @@ class StableDiffusion3ControlNetPipeline( tokenizer_3 (`T5TokenizerFast`): Tokenizer of class [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). - controlnet ([`SD3ControlNetModel`] or `List[SD3ControlNetModel]` or [`SD3MultiControlNetModel`]): + controlnet ([`SD3ControlNetModel`] or `list[SD3ControlNetModel]` or [`SD3MultiControlNetModel`]): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -199,9 +199,10 @@ def __init__( tokenizer_2: CLIPTokenizer, text_encoder_3: T5EncoderModel, tokenizer_3: T5TokenizerFast, - controlnet: Union[ - SD3ControlNetModel, List[SD3ControlNetModel], Tuple[SD3ControlNetModel], SD3MultiControlNetModel - ], + controlnet: SD3ControlNetModel + | list[SD3ControlNetModel] + | tuple[SD3ControlNetModel] + | SD3MultiControlNetModel, image_encoder: Optional[SiglipVisionModel] = None, feature_extractor: Optional[SiglipImageProcessor] = None, ): @@ -250,7 +251,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -307,7 +308,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -363,15 +364,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -383,12 +384,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -397,14 +398,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -819,24 +820,24 @@ def enable_sequential_cpu_offload(self, *args, **kwargs): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, control_image: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + controlnet_conditioning_scale: float | list[float] = 1.0, controlnet_pooled_projections: Optional[torch.FloatTensor] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -846,23 +847,23 @@ def __call__( ip_adapter_image_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -872,7 +873,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -882,37 +883,37 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. controlnet_pooled_projections (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Embeddings projected from the embeddings of controlnet input conditions. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -954,7 +955,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py index c33cf979c6d8..6eedffba8c2e 100644 --- a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +++ b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -105,9 +105,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -122,15 +122,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -195,7 +195,7 @@ class StableDiffusion3ControlNetInpaintingPipeline( tokenizer_3 (`T5TokenizerFast`): Tokenizer of class [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). - controlnet ([`SD3ControlNetModel`] or `List[SD3ControlNetModel]` or [`SD3MultiControlNetModel`]): + controlnet ([`SD3ControlNetModel`] or `list[SD3ControlNetModel]` or [`SD3MultiControlNetModel`]): Provides additional conditioning to the `transformer` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -220,9 +220,10 @@ def __init__( tokenizer_2: CLIPTokenizer, text_encoder_3: T5EncoderModel, tokenizer_3: T5TokenizerFast, - controlnet: Union[ - SD3ControlNetModel, List[SD3ControlNetModel], Tuple[SD3ControlNetModel], SD3MultiControlNetModel - ], + controlnet: SD3ControlNetModel + | list[SD3ControlNetModel] + | tuple[SD3ControlNetModel] + | SD3MultiControlNetModel, image_encoder: SiglipModel = None, feature_extractor: Optional[SiglipImageProcessor] = None, ): @@ -268,7 +269,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -325,7 +326,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -381,15 +382,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -401,12 +402,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -415,14 +416,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -869,25 +870,25 @@ def enable_sequential_cpu_offload(self, *args, **kwargs): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, control_image: PipelineImageInput = None, control_mask: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + controlnet_conditioning_scale: float | list[float] = 1.0, controlnet_pooled_projections: Optional[torch.FloatTensor] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -897,23 +898,23 @@ def __call__( ip_adapter_image_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -923,7 +924,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -933,41 +934,41 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be inpainted (which parts of the image to be masked out with `control_mask` and repainted according to `prompt`). For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. - control_mask (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`): + control_mask (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. controlnet_pooled_projections (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Embeddings projected from the embeddings of controlnet input conditions. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -1009,7 +1010,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py b/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py index 3682ddc91156..543069144cea 100644 --- a/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py +++ b/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -150,7 +150,7 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetControlNetXSModel], + unet: UNet2DConditionModel | UNetControlNetXSModel, controlnet: ControlNetXSAdapter, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, @@ -245,7 +245,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -253,7 +253,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -352,7 +352,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -644,39 +644,39 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, control_guidance_start: float = 0.0, control_guidance_end: float = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -693,7 +693,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -701,7 +701,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -722,13 +722,13 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -738,7 +738,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py b/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py index c10931a0f44a..741c9bddab45 100644 --- a/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +++ b/src/diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -185,7 +185,7 @@ def __init__( text_encoder_2: CLIPTextModelWithProjection, tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetControlNetXSModel], + unet: UNet2DConditionModel | UNetControlNetXSModel, controlnet: ControlNetXSAdapter, scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, @@ -243,9 +243,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -254,11 +254,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -376,7 +376,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -730,18 +730,18 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -749,33 +749,33 @@ def __call__( negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, control_guidance_start: float = 0.0, control_guidance_end: float = 1.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -796,10 +796,10 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -807,7 +807,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -835,38 +835,38 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. control_guidance_start (`float`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -879,7 +879,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos2_text2image.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos2_text2image.py index 66490c2be159..c2f4fe826684 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos2_text2image.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos2_text2image.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -76,9 +76,9 @@ def __init__(self, *args, **kwargs): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -93,15 +93,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -202,7 +202,7 @@ def __init__( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 512, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -245,8 +245,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline.encode_prompt with num_videos_per_prompt->num_images_per_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -259,9 +259,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -335,7 +335,7 @@ def prepare_latents( num_frames: int = 1, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -410,30 +410,30 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 768, width: int = 1360, num_inference_steps: int = 35, guidance_scale: float = 7.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `768`): @@ -450,7 +450,7 @@ def __call__( `guidance_scale > 1`. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -472,7 +472,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos2_video2world.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos2_video2world.py index 23a74ad00f93..93ab1a915369 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos2_video2world.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos2_video2world.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -81,9 +81,9 @@ def __init__(self, *args, **kwargs): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -98,15 +98,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -221,7 +221,7 @@ def __init__( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 512, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -264,8 +264,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -278,9 +278,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -356,7 +356,7 @@ def prepare_latents( do_classifier_free_guidance: bool = True, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if isinstance(generator, list) and len(generator) != batch_size: @@ -479,9 +479,9 @@ def interrupt(self): def __call__( self, image: PipelineImageInput = None, - video: List[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + video: list[PipelineImageInput] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 704, width: int = 1280, num_frames: int = 93, @@ -489,16 +489,16 @@ def __call__( guidance_scale: float = 7.0, fps: int = 16, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, sigma_conditioning: float = 0.0001, ): @@ -508,9 +508,9 @@ def __call__( Args: image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, *optional*): The image to be used as a conditioning input for the video generation. - video (`List[PIL.Image.Image]`, `np.ndarray`, `torch.Tensor`, *optional*): + video (`list[PIL.Image.Image]`, `np.ndarray`, `torch.Tensor`, *optional*): The video to be used as a conditioning input for the video generation. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `704`): @@ -531,7 +531,7 @@ def __call__( The frames per second of the generated video. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -553,7 +553,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos_text2world.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos_text2world.py index f0aa1ecf0e0f..22f3e3863102 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos_text2world.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos_text2world.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -73,9 +73,9 @@ def __init__(self, *args, **kwargs): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -90,15 +90,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -188,7 +188,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 512, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -230,8 +230,8 @@ def _get_t5_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -244,9 +244,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -320,7 +320,7 @@ def prepare_latents( num_frames: int = 121, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -394,8 +394,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 704, width: int = 1280, num_frames: int = 121, @@ -403,23 +403,23 @@ def __call__( guidance_scale: float = 7.0, fps: int = 30, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `720`): @@ -440,7 +440,7 @@ def __call__( The frames per second of the generated video. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -462,7 +462,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos_video2world.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos_video2world.py index cd5a734cc311..a49245b79781 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos_video2world.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos_video2world.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -102,9 +102,9 @@ def __init__(self, *args, **kwargs): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -119,15 +119,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -233,7 +233,7 @@ def __init__( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 512, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -276,8 +276,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.cosmos.pipeline_cosmos_text2world.CosmosTextToWorldPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -290,9 +290,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -369,7 +369,7 @@ def prepare_latents( input_frames_guidance: bool = False, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if isinstance(generator, list) and len(generator) != batch_size: @@ -507,9 +507,9 @@ def interrupt(self): def __call__( self, image: PipelineImageInput = None, - video: List[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + video: list[PipelineImageInput] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 704, width: int = 1280, num_frames: int = 121, @@ -519,23 +519,23 @@ def __call__( augment_sigma: float = 0.001, fps: int = 30, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `720`): @@ -556,7 +556,7 @@ def __call__( The frames per second of the generated video. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -578,7 +578,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/cosmos/pipeline_output.py b/src/diffusers/pipelines/cosmos/pipeline_output.py index ec5f4826f62a..1ded292f8dfb 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_output.py +++ b/src/diffusers/pipelines/cosmos/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -17,8 +16,8 @@ class CosmosPipelineOutput(BaseOutput): Output class for Cosmos any-to-world/video pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ @@ -32,9 +31,9 @@ class CosmosImagePipelineOutput(BaseOutput): Output class for Cosmos any-to-image pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py b/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py index 5a70c4f5ff9a..e9f3ae1998d5 100644 --- a/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +++ b/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -61,10 +61,10 @@ def __call__( self, batch_size: int = 1, num_inference_steps: int = 100, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, audio_length_in_s: Optional[float] = None, return_dict: bool = True, - ) -> Union[AudioPipelineOutput, Tuple]: + ) -> AudioPipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/ddim/pipeline_ddim.py b/src/diffusers/pipelines/ddim/pipeline_ddim.py index 39587ca5221d..5cd83fff483d 100644 --- a/src/diffusers/pipelines/ddim/pipeline_ddim.py +++ b/src/diffusers/pipelines/ddim/pipeline_ddim.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -60,13 +60,13 @@ def __init__(self, unet: UNet2DModel, scheduler: DDIMScheduler): def __call__( self, batch_size: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, eta: float = 0.0, num_inference_steps: int = 50, use_clipped_model_output: Optional[bool] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py index 0d7766a8cfd0..72c1c5518af2 100644 --- a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py +++ b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -57,11 +57,11 @@ def __init__(self, unet: UNet2DModel, scheduler: DDPMScheduler): def __call__( self, batch_size: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, num_inference_steps: int = 1000, output_type: Optional[str] = "pil", return_dict: bool = True, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py index 8fa31f8504d3..ea5328e720be 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer @@ -168,11 +168,11 @@ def __init__( @torch.no_grad() def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -181,7 +181,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -189,7 +189,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -269,7 +269,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -548,16 +548,16 @@ def _clean_caption(self, caption): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 100, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = None, width: Optional[int] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", @@ -565,19 +565,19 @@ def __call__( callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, clean_caption: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.0): @@ -586,7 +586,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -599,7 +599,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py index 507927faf61b..4dfca1ab60ee 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -192,11 +192,11 @@ def __init__( @torch.no_grad() def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -205,7 +205,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -213,7 +213,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -293,7 +293,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -430,7 +430,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -662,18 +662,21 @@ def prepare_intermediate_images( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, + prompt: str | list[str] = None, + image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, strength: float = 0.7, num_inference_steps: int = 80, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 10.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", @@ -681,13 +684,13 @@ def __call__( callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, clean_caption: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. image (`torch.Tensor` or `PIL.Image.Image`): @@ -702,7 +705,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 80): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 10.0): @@ -711,7 +714,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -720,7 +723,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py index 9bc15c3c6f62..9d2ef3cbc659 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -345,11 +345,11 @@ def _clean_caption(self, caption): # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -358,7 +358,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -366,7 +366,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -446,7 +446,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -586,7 +586,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -617,7 +617,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`original_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`original_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -745,26 +745,29 @@ def prepare_intermediate_images( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], - original_image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, + image: PIL.Image.Image | np.ndarray | torch.Tensor, + original_image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, strength: float = 0.8, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 4.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, noise_level: int = 250, clean_caption: bool = True, ): @@ -783,13 +786,13 @@ def __call__( denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will be maximum and the denoising process will run for the full number of iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 4.0): @@ -798,7 +801,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -807,7 +810,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py index 9d6cf62020a9..94e4c7f72bbf 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -196,11 +196,11 @@ def __init__( # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -209,7 +209,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -217,7 +217,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -297,7 +297,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -437,7 +437,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -468,7 +468,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`mask_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`mask_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -754,21 +754,27 @@ def prepare_intermediate_images( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, - mask_image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, + prompt: str | list[str] = None, + image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, + mask_image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, strength: float = 1.0, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", @@ -776,13 +782,13 @@ def __call__( callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, clean_caption: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. image (`torch.Tensor` or `PIL.Image.Image`): @@ -802,7 +808,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.0): @@ -811,7 +817,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -820,7 +826,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py index 0122c164d8b8..c19a1cc38064 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -347,11 +347,11 @@ def _clean_caption(self, caption): # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -360,7 +360,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -368,7 +368,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -448,7 +448,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -589,7 +589,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -620,7 +620,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`original_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`original_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -653,7 +653,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`mask_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`mask_image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -833,29 +833,35 @@ def prepare_intermediate_images( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], - original_image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, - mask_image: Union[ - PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] - ] = None, + image: PIL.Image.Image | np.ndarray | torch.Tensor, + original_image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, + mask_image: PIL.Image.Image + | torch.Tensor + | np.ndarray + | list[PIL.Image.Image] + | list[torch.Tensor] + | list[np.ndarray] = None, strength: float = 0.8, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 100, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 4.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, noise_level: int = 0, clean_caption: bool = True, ): @@ -879,13 +885,13 @@ def __call__( denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will be maximum and the denoising process will run for the full number of iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 4.0): @@ -894,7 +900,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -903,7 +909,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py index ffa60575fe33..2dbc3a18d848 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py @@ -2,7 +2,7 @@ import inspect import re import urllib.parse as ul -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -303,11 +303,11 @@ def _clean_caption(self, caption): # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, clean_caption: bool = False, @@ -316,7 +316,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not @@ -324,7 +324,7 @@ def encode_prompt( number of images that should be generated per prompt device: (`torch.device`, *optional*): torch device to place the resulting embeddings on - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -404,7 +404,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -547,7 +547,7 @@ def check_inputs( and not isinstance(check_image_type, np.ndarray) ): raise ValueError( - "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + "`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, or list[...] but is" f" {type(check_image_type)}" ) @@ -615,24 +615,24 @@ def preprocess_image(self, image, num_images_per_prompt, device): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: int = None, width: int = None, - image: Union[PIL.Image.Image, np.ndarray, torch.Tensor] = None, + image: PIL.Image.Image | np.ndarray | torch.Tensor = None, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 4.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, noise_level: int = 250, clean_caption: bool = True, ): @@ -640,7 +640,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to None): @@ -652,7 +652,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*, defaults to None): + timesteps (`list[int]`, *optional*, defaults to None): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 4.0): @@ -661,7 +661,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -670,7 +670,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py b/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py index b8bae89cec03..9dc2fd360c21 100644 --- a/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py +++ b/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -13,17 +13,17 @@ class IFPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`): - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`): + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. - nsfw_detected (`List[bool]`): - List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + nsfw_detected (`list[bool]`): + list of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) content or a watermark. `None` if safety checking could not be performed. - watermark_detected (`List[bool]`): - List of flags denoting whether the corresponding generated image likely has a watermark. `None` if safety + watermark_detected (`list[bool]`): + list of flags denoting whether the corresponding generated image likely has a watermark. `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_detected: Optional[List[bool]] - watermark_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_detected: Optional[list[bool]] + watermark_detected: Optional[list[bool]] diff --git a/src/diffusers/pipelines/deepfloyd_if/watermark.py b/src/diffusers/pipelines/deepfloyd_if/watermark.py index e03e3fab026a..d5fe99f681f7 100644 --- a/src/diffusers/pipelines/deepfloyd_if/watermark.py +++ b/src/diffusers/pipelines/deepfloyd_if/watermark.py @@ -1,5 +1,3 @@ -from typing import List - import PIL.Image import torch from PIL import Image @@ -16,7 +14,7 @@ def __init__(self): self.register_buffer("watermark_image", torch.zeros((62, 62, 4))) self.watermark_image_as_pil = None - def apply_watermark(self, images: List[PIL.Image.Image], sample_size=None): + def apply_watermark(self, images: list[PIL.Image.Image], sample_size=None): # Copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287 h = images[0].height diff --git a/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py b/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py index f69f905b56c5..3423f3d6daa0 100644 --- a/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py +++ b/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional import torch from torch import nn @@ -18,12 +18,12 @@ class TransformationModelOutput(ModelOutput): last_hidden_state (`torch.Tensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.Tensor` (one for the output of the embeddings, if the model has an embedding layer, + one + tuple of `torch.Tensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + tuple of `torch.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention @@ -32,8 +32,8 @@ class TransformationModelOutput(ModelOutput): projection_state: Optional[torch.Tensor] = None last_hidden_state: torch.Tensor = None - hidden_states: Optional[Tuple[torch.Tensor]] = None - attentions: Optional[Tuple[torch.Tensor]] = None + hidden_states: Optional[tuple[torch.Tensor]] = None + attentions: Optional[tuple[torch.Tensor]] = None class RobertaSeriesConfig(XLMRobertaConfig): diff --git a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py index 6f484aa3e298..e49bda6034b0 100644 --- a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +++ b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from packaging import version @@ -94,9 +94,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -111,15 +111,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -340,7 +340,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -348,7 +348,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -447,7 +447,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -704,35 +704,35 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -741,14 +741,14 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -756,7 +756,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -790,7 +790,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py index d6bf90120755..adb2aabaea97 100644 --- a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -122,9 +122,9 @@ def preprocess(image): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -139,15 +139,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -368,7 +368,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -376,7 +376,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -475,7 +475,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -772,35 +772,35 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -815,14 +815,14 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. This parameter is modulated by `strength`. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -830,7 +830,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -856,7 +856,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py index dd174ae3c21f..3a5b032a8f7a 100644 --- a/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py +++ b/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -16,13 +16,13 @@ class AltDiffusionPipelineOutput(BaseOutput): Output class for Alt Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`) - List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + nsfw_content_detected (`list[bool]`) + list indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] diff --git a/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py b/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py index 81fa999eb1fb..f63fc8aacbc8 100644 --- a/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py +++ b/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py @@ -14,7 +14,6 @@ from math import acos, sin -from typing import List, Tuple, Union import numpy as np import torch @@ -53,7 +52,7 @@ def __init__( vqvae: AutoencoderKL, unet: UNet2DConditionModel, mel: Mel, - scheduler: Union[DDIMScheduler, DDPMScheduler], + scheduler: DDIMScheduler | DDPMScheduler, ): super().__init__() self.register_modules(unet=unet, scheduler=scheduler, mel=mel, vqvae=vqvae) @@ -84,10 +83,7 @@ def __call__( noise: torch.Tensor = None, encoding: torch.Tensor = None, return_dict=True, - ) -> Union[ - Union[AudioPipelineOutput, ImagePipelineOutput], - Tuple[List[Image.Image], Tuple[int, List[np.ndarray]]], - ]: + ) -> AudioPipelineOutput | ImagePipelineOutput | tuple[list[Image.Image], tuple[int, list[np.ndarray]]]: """ The call function to the pipeline for generation. @@ -170,8 +166,8 @@ def __call__( ``` Returns: - `List[PIL Image]`: - A list of Mel spectrograms (`float`, `List[np.ndarray]`) with the sample rate and raw audio. + `list[PIL Image]`: + A list of Mel spectrograms (`float`, `list[np.ndarray]`) with the sample rate and raw audio. """ steps = steps or self.get_default_steps() @@ -268,13 +264,13 @@ def __call__( return BaseOutput(**AudioPipelineOutput(np.array(audios)[:, np.newaxis, :]), **ImagePipelineOutput(images)) @torch.no_grad() - def encode(self, images: List[Image.Image], steps: int = 50) -> np.ndarray: + def encode(self, images: list[Image.Image], steps: int = 50) -> np.ndarray: """ Reverse the denoising step process to recover a noisy image from the generated image. Args: - images (`List[PIL Image]`): - List of images to encode. + images (`list[PIL Image]`): + list of images to encode. steps (`int`): Number of encoding steps to perform (defaults to `50`). diff --git a/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py b/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py index 0bb24ed0b1ce..734bda14dfb5 100644 --- a/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +++ b/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -47,13 +47,13 @@ def __init__(self, vqvae: VQModel, unet: UNet2DModel, scheduler: DDIMScheduler): def __call__( self, batch_size: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, eta: float = 0.0, num_inference_steps: int = 50, output_type: Optional[str] = "pil", return_dict: bool = True, **kwargs, - ) -> Union[Tuple, ImagePipelineOutput]: + ) -> tuple | ImagePipelineOutput: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py b/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py index 71e3e156e0e4..1e23cfb08144 100644 --- a/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py +++ b/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -52,11 +52,11 @@ def __call__( self, batch_size: int = 1, num_inference_steps: int = 50, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, **kwargs, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py b/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py index 56c6007ae886..5cea66b9858d 100644 --- a/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +++ b/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import PIL.Image @@ -30,7 +30,7 @@ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess -def _preprocess_image(image: Union[List, PIL.Image.Image, torch.Tensor]): +def _preprocess_image(image: list | PIL.Image.Image | torch.Tensor): deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) if isinstance(image, torch.Tensor): @@ -53,7 +53,7 @@ def _preprocess_image(image: Union[List, PIL.Image.Image, torch.Tensor]): return image -def _preprocess_mask(mask: Union[List, PIL.Image.Image, torch.Tensor]): +def _preprocess_mask(mask: list | PIL.Image.Image | torch.Tensor): if isinstance(mask, torch.Tensor): return mask elif isinstance(mask, PIL.Image.Image): @@ -98,16 +98,16 @@ def __init__(self, unet: UNet2DModel, scheduler: RePaintScheduler): @torch.no_grad() def __call__( self, - image: Union[torch.Tensor, PIL.Image.Image], - mask_image: Union[torch.Tensor, PIL.Image.Image], + image: torch.Tensor | PIL.Image.Image, + mask_image: torch.Tensor | PIL.Image.Image, num_inference_steps: int = 250, eta: float = 0.0, jump_length: int = 10, jump_n_sample: int = 10, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py b/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py index 3f04db7ad699..3f7ecba4b342 100644 --- a/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py +++ b/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -48,11 +48,11 @@ def __call__( self, batch_size: int = 1, num_inference_steps: int = 2000, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, **kwargs, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py b/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py index 8985a6f88800..8733ddbb7942 100644 --- a/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py +++ b/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py @@ -16,7 +16,7 @@ import dataclasses import math import os -from typing import Any, Callable, List, Mapping, MutableMapping, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Mapping, MutableMapping, Optional, Sequence import numpy as np import torch @@ -102,7 +102,7 @@ class NoteEncodingState: """Encoding state for note transcription, keeping track of active pitches.""" # velocity bin for active pitches and programs - active_pitches: MutableMapping[Tuple[int, int], int] = dataclasses.field(default_factory=dict) + active_pitches: MutableMapping[tuple[int, int], int] = dataclasses.field(default_factory=dict) @dataclasses.dataclass @@ -153,7 +153,7 @@ class Codec: and specified separately. """ - def __init__(self, max_shift_steps: int, steps_per_second: float, event_ranges: List[EventRange]): + def __init__(self, max_shift_steps: int, steps_per_second: float, event_ranges: list[EventRange]): """Define Codec. Args: @@ -197,7 +197,7 @@ def encode_event(self, event: Event) -> int: raise ValueError(f"Unknown event type: {event.type}") - def event_type_range(self, event_type: str) -> Tuple[int, int]: + def event_type_range(self, event_type: str) -> tuple[int, int]: """Return [min_id, max_id] for an event type.""" offset = 0 for er in self._event_ranges: @@ -280,7 +280,7 @@ def audio_to_frames( samples, hop_size: int, frame_rate: int, -) -> Tuple[Sequence[Sequence[int]], torch.Tensor]: +) -> tuple[Sequence[Sequence[int]], torch.Tensor]: """Convert audio samples to non-overlapping frames and frame times.""" frame_size = hop_size samples = np.pad(samples, [0, frame_size - len(samples) % frame_size], mode="constant") @@ -301,7 +301,7 @@ def audio_to_frames( def note_sequence_to_onsets_and_offsets_and_programs( ns: note_seq.NoteSequence, -) -> Tuple[Sequence[float], Sequence[NoteEventData]]: +) -> tuple[Sequence[float], Sequence[NoteEventData]]: """Extract onset & offset times and pitches & programs from a NoteSequence. The onset & offset times will not necessarily be in sorted order. @@ -632,7 +632,7 @@ def __init__(self): self.tokenizer = Tokenizer(self.codec.num_classes) self.note_representation_config = NoteRepresentationConfig(onsets_only=False, include_ties=True) - def __call__(self, midi: Union[bytes, os.PathLike, str]): + def __call__(self, midi: bytes | os.PathLike | str): if not isinstance(midi, bytes): with open(midi, "rb") as f: midi = f.read() diff --git a/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py b/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py index be07b1b15ea8..9b983ca1205a 100644 --- a/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +++ b/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py @@ -14,7 +14,7 @@ # limitations under the License. import math -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -129,14 +129,14 @@ def decode(self, encodings_and_masks, input_tokens, noise_time): @torch.no_grad() def __call__( self, - input_tokens: List[List[int]], + input_tokens: list[list[int]], generator: Optional[torch.Generator] = None, num_inference_steps: int = 100, return_dict: bool = True, output_type: str = "np", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - ) -> Union[AudioPipelineOutput, Tuple]: + ) -> AudioPipelineOutput | tuple: if (callback_steps is None) or ( callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) ): @@ -148,8 +148,8 @@ def __call__( The call function to the pipeline for generation. Args: - input_tokens (`List[List[int]]`): - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + input_tokens (`list[list[int]]`): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. num_inference_steps (`int`, *optional*, defaults to 100): diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py index 08f8c7e26fae..4cef499c7bfa 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -301,7 +301,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -309,7 +309,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -408,7 +408,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -633,8 +633,8 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], - source_prompt: Union[str, List[str]], + prompt: str | list[str], + source_prompt: str | list[str], image: PipelineImageInput = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, @@ -642,22 +642,22 @@ def __call__( source_guidance_scale: Optional[float] = 1, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be used as the starting point. Can also accept image latents as `image`, but if passing latents directly it is not encoded again. strength (`float`, *optional*, defaults to 0.8): @@ -680,7 +680,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py index fcd8bf317adf..435863c15eb2 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py @@ -1,5 +1,5 @@ import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -76,7 +76,7 @@ class OnnxStableDiffusionInpaintPipelineLegacy(DiffusionPipeline): text_encoder: OnnxRuntimeModel tokenizer: CLIPTokenizer unet: OnnxRuntimeModel - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler safety_checker: OnnxRuntimeModel feature_extractor: CLIPImageProcessor @@ -87,7 +87,7 @@ def __init__( text_encoder: OnnxRuntimeModel, tokenizer: CLIPTokenizer, unet: OnnxRuntimeModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: OnnxRuntimeModel, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = True, @@ -152,7 +152,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt def _encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: Optional[int], do_classifier_free_guidance: bool, negative_prompt: Optional[str], @@ -163,13 +163,13 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`np.ndarray`, *optional*): @@ -214,7 +214,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -297,13 +297,13 @@ def check_inputs( def __call__( self, - prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image] = None, - mask_image: Union[np.ndarray, PIL.Image.Image] = None, + prompt: str | list[str], + image: np.ndarray | PIL.Image.Image = None, + mask_image: np.ndarray | PIL.Image.Image = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, generator: Optional[np.random.RandomState] = None, @@ -318,7 +318,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`nd.ndarray` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the @@ -343,7 +343,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py index ba0dd66c2938..e158d1edd027 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -272,7 +272,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -280,7 +280,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -379,7 +379,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -562,31 +562,31 @@ def prepare_latents(self, image, timestep, num_images_per_prompt, dtype, device, @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, - image: Union[torch.Tensor, PIL.Image.Image] = None, - mask_image: Union[torch.Tensor, PIL.Image.Image] = None, + prompt: str | list[str] = None, + image: torch.Tensor | PIL.Image.Image = None, + mask_image: torch.Tensor | PIL.Image.Image = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, add_predicted_noise: Optional[bool] = False, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. image (`torch.Tensor` or `PIL.Image.Image`): @@ -611,7 +611,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py index b7a0be57c12b..efbf59f7a9ef 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py @@ -13,7 +13,7 @@ import copy import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer @@ -205,7 +205,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -213,7 +213,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -312,7 +312,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -614,15 +614,15 @@ def edit_model( @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -630,14 +630,14 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -649,7 +649,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -657,7 +657,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py index c236e73bf448..c5bb234f2c63 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer @@ -202,7 +202,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -210,7 +210,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -309,7 +309,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -488,17 +488,17 @@ def _cumsum(self, input, dim, debug=False): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, parallel: int = 10, tolerance: float = 0.1, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -506,7 +506,7 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, debug: bool = False, clip_skip: int = None, ): @@ -514,7 +514,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -533,7 +533,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -541,7 +541,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py index 2a461ae20cc9..964e2ede3d38 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -62,13 +62,13 @@ class Pix2PixInversionPipelineOutput(BaseOutput, TextualInversionLoaderMixin): Args: latents (`torch.Tensor`) inverted latents tensor - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ latents: torch.Tensor - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray EXAMPLE_DOC_STRING = """ @@ -328,7 +328,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDPMScheduler, DDIMScheduler, EulerAncestralDiscreteScheduler, LMSDiscreteScheduler], + scheduler: DDPMScheduler | DDIMScheduler | EulerAncestralDiscreteScheduler | LMSDiscreteScheduler, feature_extractor: CLIPImageProcessor, safety_checker: StableDiffusionSafetyChecker, inverse_scheduler: DDIMInverseScheduler, @@ -420,7 +420,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -428,7 +428,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -527,7 +527,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -709,7 +709,7 @@ def construct_direction(self, embs_source: torch.Tensor, embs_target: torch.Tens return (embs_target.mean(0) - embs_source.mean(0)).unsqueeze(0) @torch.no_grad() - def get_embeds(self, prompt: List[str], batch_size: int = 16) -> torch.Tensor: + def get_embeds(self, prompt: list[str], batch_size: int = 16) -> torch.Tensor: num_prompts = len(prompt) embeds = [] for i in range(0, num_prompts, batch_size): @@ -818,17 +818,17 @@ def kl_divergence(self, hidden_states): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, source_embeds: torch.Tensor = None, target_embeds: torch.Tensor = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -837,14 +837,14 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. source_embeds (`torch.Tensor`): @@ -866,7 +866,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -875,7 +875,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1113,7 +1113,7 @@ def invert( image: PipelineImageInput = None, num_inference_steps: int = 50, guidance_scale: float = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, cross_attention_guidance_amount: float = 0.1, @@ -1121,7 +1121,7 @@ def invert( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, lambda_auto_corr: float = 20.0, lambda_kl: float = 20.0, num_reg_steps: int = 5, @@ -1131,10 +1131,10 @@ def invert( Function used to generate inverted latents given a prompt and image. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch which will be used for conditioning. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. num_inference_steps (`int`, *optional*, defaults to 50): @@ -1146,7 +1146,7 @@ def invert( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py b/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py index 50b8b0bcbc1d..11bdd465ecfe 100644 --- a/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py +++ b/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -46,11 +46,11 @@ def __call__( self, batch_size: int = 1, num_inference_steps: int = 50, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, **kwargs, - ) -> Union[Tuple, ImagePipelineOutput]: + ) -> tuple | ImagePipelineOutput: r""" The call function to the pipeline for generation. diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py index 397fbc0d85b8..ae4af4ce5747 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -277,7 +277,7 @@ class UNetFlatConditionModel(ModelMixin, ConfigMixin): for all models (such as downloading or saving). Parameters: - sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + sample_size (`int` or `tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. @@ -285,17 +285,17 @@ class UNetFlatConditionModel(ModelMixin, ConfigMixin): flip_sin_to_cos (`bool`, *optional*, defaults to `False`): Whether to flip the sin to cos in the time embedding. freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. - down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "DownBlockFlat")`): + down_block_types (`tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "DownBlockFlat")`): The tuple of downsample blocks to use. mid_block_type (`str`, *optional*, defaults to `"UNetMidBlockFlatCrossAttn"`): Block type for middle of UNet, it can be one of `UNetMidBlockFlatCrossAttn`, `UNetMidBlockFlat`, or `UNetMidBlockFlatSimpleCrossAttn`. If `None`, the mid block layer is skipped. - up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat")`): + up_block_types (`tuple[str]`, *optional*, defaults to `("UpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat")`): The tuple of upsample blocks to use. - only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + only_cross_attention(`bool` or `tuple[bool]`, *optional*, default to `False`): Whether to include self-attention in the basic transformer blocks, see [`~models.attention.BasicTransformerBlock`]. - block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + block_out_channels (`tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. @@ -305,15 +305,15 @@ class UNetFlatConditionModel(ModelMixin, ConfigMixin): norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. If `None`, normalization and activation layers is skipped in post-processing. norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. - cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + cross_attention_dim (`int` or `tuple[int]`, *optional*, defaults to 1280): The dimension of the cross attention features. - transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + transformer_layers_per_block (`int`, `tuple[int]`, or `tuple[tuple]` , *optional*, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlockFlat`], [`~models.unet_2d_blocks.CrossAttnUpBlockFlat`], [`~models.unet_2d_blocks.UNetMidBlockFlatCrossAttn`]. - reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None): + reverse_transformer_layers_per_block : (`tuple[tuple]`, *optional*, defaults to None): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling - blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for + blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `tuple[tuple]` and for [`~models.unet_2d_blocks.CrossAttnDownBlockFlat`], [`~models.unet_2d_blocks.CrossAttnUpBlockFlat`], [`~models.unet_2d_blocks.UNetMidBlockFlatCrossAttn`]. encoder_hid_dim (`int`, *optional*, defaults to None): @@ -374,35 +374,35 @@ def __init__( center_input_sample: bool = False, flip_sin_to_cos: bool = True, freq_shift: int = 0, - down_block_types: Tuple[str] = ( + down_block_types: tuple[str] = ( "CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "DownBlockFlat", ), mid_block_type: Optional[str] = "UNetMidBlockFlatCrossAttn", - up_block_types: Tuple[str] = ( + up_block_types: tuple[str] = ( "UpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat", ), - only_cross_attention: Union[bool, Tuple[bool]] = False, - block_out_channels: Tuple[int] = (320, 640, 1280, 1280), - layers_per_block: Union[int, Tuple[int]] = 2, + only_cross_attention: bool | tuple[bool] = False, + block_out_channels: tuple[int] = (320, 640, 1280, 1280), + layers_per_block: int | tuple[int] = 2, downsample_padding: int = 1, mid_block_scale_factor: float = 1, dropout: float = 0.0, act_fn: str = "silu", norm_num_groups: Optional[int] = 32, norm_eps: float = 1e-5, - cross_attention_dim: Union[int, Tuple[int]] = 1280, - transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, - reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None, + cross_attention_dim: int | tuple[int] = 1280, + transformer_layers_per_block: int | tuple[int] | tuple[tuple] = 1, + reverse_transformer_layers_per_block: Optional[tuple[tuple[int]]] = None, encoder_hid_dim: Optional[int] = None, encoder_hid_dim_type: Optional[str] = None, - attention_head_dim: Union[int, Tuple[int]] = 8, - num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + attention_head_dim: int | tuple[int] = 8, + num_attention_heads: Optional[int | tuple[int]] = None, dual_cross_attention: bool = False, use_linear_projection: bool = False, class_embed_type: Optional[str] = None, @@ -826,7 +826,7 @@ def __init__( ) @property - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -835,7 +835,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -849,7 +849,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. @@ -952,7 +952,7 @@ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): # Recursively walk through all the children. # Any children which exposes the set_attention_slice method # gets the message - def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: list[int]): if hasattr(module, "set_attention_slice"): module.set_attention_slice(slice_size.pop()) @@ -1037,19 +1037,19 @@ def unload_lora(self): def forward( self, sample: torch.Tensor, - timestep: Union[torch.Tensor, float, int], + timestep: torch.Tensor | float | int, encoder_hidden_states: torch.Tensor, class_labels: Optional[torch.Tensor] = None, timestep_cond: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, - down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + added_cond_kwargs: Optional[dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[tuple[torch.Tensor]] = None, mid_block_additional_residual: Optional[torch.Tensor] = None, - down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + down_intrablock_additional_residuals: Optional[tuple[torch.Tensor]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[UNet2DConditionOutput, Tuple]: + ) -> UNet2DConditionOutput | tuple: r""" The [`UNetFlatConditionModel`] forward method. @@ -1580,7 +1580,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, temb: Optional[torch.Tensor] = None - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: output_states = () for resnet in self.resnets: @@ -1608,7 +1608,7 @@ def __init__( temb_channels: int, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -1698,10 +1698,10 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, additional_residuals: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, ...]]: + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: output_states = () blocks = list(zip(self.resnets, self.attentions)) @@ -1797,7 +1797,7 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, upsample_size: Optional[int] = None, *args, @@ -1856,7 +1856,7 @@ def __init__( resolution_idx: Optional[int] = None, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -1940,10 +1940,10 @@ def __init__( def forward( self, hidden_states: torch.Tensor, - res_hidden_states_tuple: Tuple[torch.Tensor, ...], + res_hidden_states_tuple: tuple[torch.Tensor, ...], temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, upsample_size: Optional[int] = None, attention_mask: Optional[torch.Tensor] = None, encoder_attention_mask: Optional[torch.Tensor] = None, @@ -2178,7 +2178,7 @@ def __init__( out_channels: Optional[int] = None, dropout: float = 0.0, num_layers: int = 1, - transformer_layers_per_block: Union[int, Tuple[int]] = 1, + transformer_layers_per_block: int | tuple[int] = 1, resnet_eps: float = 1e-6, resnet_time_scale_shift: str = "default", resnet_act_fn: str = "swish", @@ -2279,7 +2279,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: if cross_attention_kwargs is not None: @@ -2404,7 +2404,7 @@ def forward( temb: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py index 9ff8e9857791..5619cd2c83cf 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py @@ -1,5 +1,5 @@ import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import PIL.Image import torch @@ -81,15 +81,15 @@ def __init__( @torch.no_grad() def image_variation( self, - image: Union[torch.Tensor, PIL.Image.Image], + image: torch.Tensor | PIL.Image.Image, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -100,7 +100,7 @@ def image_variation( The call function to the pipeline for generation. Args: - image (`PIL.Image.Image`, `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image`, `list[PIL.Image.Image]` or `torch.Tensor`): The image prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -112,7 +112,7 @@ def image_variation( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -193,15 +193,15 @@ def image_variation( @torch.no_grad() def text_to_image( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -212,7 +212,7 @@ def text_to_image( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -224,7 +224,7 @@ def text_to_image( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -301,8 +301,8 @@ def text_to_image( @torch.no_grad() def dual_guided( self, - prompt: Union[PIL.Image.Image, List[PIL.Image.Image]], - image: Union[str, List[str]], + prompt: PIL.Image.Image | list[PIL.Image.Image], + image: str | list[str], text_to_image_strength: float = 0.5, height: Optional[int] = None, width: Optional[int] = None, @@ -310,7 +310,7 @@ def dual_guided( guidance_scale: float = 7.5, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -321,7 +321,7 @@ def dual_guided( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -333,7 +333,7 @@ def dual_guided( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -341,7 +341,7 @@ def dual_guided( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py index 0252f4f6af7f..ddf81d3d8b11 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -158,7 +158,7 @@ def _encode_text_prompt(self, prompt, device, num_images_per_prompt, do_classifi Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded device: (`torch.device`): torch device @@ -249,7 +249,7 @@ def _encode_image_prompt(self, prompt, device, num_images_per_prompt, do_classif Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded device: (`torch.device`): torch device @@ -369,7 +369,7 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype latents = latents * self.scheduler.init_noise_sigma return latents - def set_transformer_params(self, mix_ratio: float = 0.5, condition_types: Tuple = ("text", "image")): + def set_transformer_params(self, mix_ratio: float = 0.5, condition_types: tuple = ("text", "image")): for name, module in self.image_unet.named_modules(): if isinstance(module, DualTransformer2DModel): module.mix_ratio = mix_ratio @@ -385,8 +385,8 @@ def set_transformer_params(self, mix_ratio: float = 0.5, condition_types: Tuple @torch.no_grad() def __call__( self, - prompt: Union[PIL.Image.Image, List[PIL.Image.Image]], - image: Union[str, List[str]], + prompt: PIL.Image.Image | list[PIL.Image.Image], + image: str | list[str], text_to_image_strength: float = 0.5, height: Optional[int] = None, width: Optional[int] = None, @@ -394,7 +394,7 @@ def __call__( guidance_scale: float = 7.5, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -406,7 +406,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -418,7 +418,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -426,7 +426,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py index 034a0226419b..1954539ddbc4 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -84,7 +84,7 @@ def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_fr Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded device: (`torch.device`): torch device @@ -92,7 +92,7 @@ def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_fr number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). """ @@ -122,7 +122,7 @@ def normalize_embeddings(encoder_output): # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - uncond_images: List[str] + uncond_images: list[str] if negative_prompt is None: uncond_images = [np.zeros((512, 512, 3)) + 0.5] * batch_size elif type(prompt) is not type(negative_prompt): @@ -196,7 +196,7 @@ def check_inputs(self, image, height, width, callback_steps): and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -237,15 +237,15 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.Tensor], + image: PIL.Image.Image | list[PIL.Image.Image] | torch.Tensor, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -257,7 +257,7 @@ def __call__( The call function to the pipeline for generation. Args: - image (`PIL.Image.Image`, `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image`, `list[PIL.Image.Image]` or `torch.Tensor`): The image prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -269,7 +269,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py index 2f54f4fc98a4..a35e5276f0ec 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer @@ -108,7 +108,7 @@ def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_fr Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded device: (`torch.device`): torch device @@ -116,7 +116,7 @@ def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_fr number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). """ @@ -164,7 +164,7 @@ def normalize_embeddings(encoder_output): # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -323,15 +323,15 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -343,7 +343,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -355,7 +355,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py b/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py index e8617a54b691..694a9b5db2ab 100644 --- a/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +++ b/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import torch from transformers import CLIPTextModel, CLIPTokenizer @@ -163,23 +163,23 @@ def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guida @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_inference_steps: int = 100, guidance_scale: float = 5.0, truncation_rate: float = 1.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the diff --git a/src/diffusers/pipelines/dit/pipeline_dit.py b/src/diffusers/pipelines/dit/pipeline_dit.py index 68ff6c9b559a..0d0a69190c11 100644 --- a/src/diffusers/pipelines/dit/pipeline_dit.py +++ b/src/diffusers/pipelines/dit/pipeline_dit.py @@ -18,7 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Tuple, Union +from typing import Optional import torch @@ -62,7 +62,7 @@ def __init__( transformer: DiTTransformer2DModel, vae: AutoencoderKL, scheduler: KarrasDiffusionSchedulers, - id2label: Optional[Dict[int, str]] = None, + id2label: Optional[dict[int, str]] = None, ): super().__init__() self.register_modules(transformer=transformer, vae=vae, scheduler=scheduler) @@ -75,7 +75,7 @@ def __init__( self.labels[label.lstrip().rstrip()] = int(key) self.labels = dict(sorted(self.labels.items())) - def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + def get_label_ids(self, label: str | list[str]) -> list[int]: r""" Map label strings from ImageNet to corresponding class ids. @@ -103,19 +103,19 @@ def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: @torch.no_grad() def __call__( self, - class_labels: List[int], + class_labels: list[int], guidance_scale: float = 4.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, num_inference_steps: int = 50, output_type: Optional[str] = "pil", return_dict: bool = True, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: r""" The call function to the pipeline for generation. Args: - class_labels (List[int]): - List of ImageNet class labels for the images to be generated. + class_labels (list[int]): + list of ImageNet class labels for the images to be generated. guidance_scale (`float`, *optional*, defaults to 4.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. diff --git a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate.py b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate.py index 92239c0d32f0..73dad81f4d2f 100755 --- a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate.py +++ b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import torch from transformers import ( @@ -127,9 +127,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -144,15 +144,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -211,8 +211,8 @@ class EasyAnimatePipeline(DiffusionPipeline): def __init__( self, vae: AutoencoderKLMagvit, - text_encoder: Union[Qwen2VLForConditionalGeneration, BertModel], - tokenizer: Union[Qwen2Tokenizer, BertTokenizer], + text_encoder: Qwen2VLForConditionalGeneration | BertModel, + tokenizer: Qwen2Tokenizer | BertTokenizer, transformer: EasyAnimateTransformer3DModel, scheduler: FlowMatchEulerDiscreteScheduler, ): @@ -240,10 +240,10 @@ def __init__( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -256,7 +256,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -266,7 +266,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -525,35 +525,35 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_frames: Optional[int] = 49, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, ): r""" Generates images or video using the EasyAnimate pipeline based on the provided prompts. Examples: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): Text prompts to guide the image or video generation. If not provided, use `prompt_embeds` instead. num_frames (`int`, *optional*): Length of the generated video (in frames). @@ -566,13 +566,13 @@ def __call__( down inference. guidance_scale (`float`, *optional*, defaults to 5.0): Encourages the model to align outputs with prompts. A higher value may decrease image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): Prompts indicating what to exclude in generation. If not specified, use `negative_prompt_embeds`. num_images_per_prompt (`int`, *optional*, defaults to 1): Number of images to generate for each prompt. eta (`float`, *optional*, defaults to 0.0): Applies to DDIM scheduling. Controlled by the eta parameter from the related literature. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A generator to ensure reproducibility in image generation. latents (`torch.Tensor`, *optional*): Predefined latent tensors to condition generation. @@ -590,15 +590,15 @@ def __call__( If `True`, returns a structured output. Otherwise returns a simple tuple. callback_on_step_end (`Callable`, *optional*): Functions called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): Tensor names to be included in callback function calls. guidance_rescale (`float`, *optional*, defaults to 0.0): Adjusts noise levels based on guidance scale. - original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): + original_size (`tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): Original dimensions of the output. - target_size (`Tuple[int, int]`, *optional*): + target_size (`tuple[int, int]`, *optional*): Desired output dimensions for calculations. - crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`): + crops_coords_top_left (`tuple[int, int]`, *optional*, defaults to `(0, 0)`): Coordinates for cropping. Returns: diff --git a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_control.py b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_control.py index f74a11f87d75..9dc401f0cecf 100755 --- a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_control.py +++ b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_control.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -229,9 +229,9 @@ def resize_mask(mask, latent, process_first_frame_only=True): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -246,15 +246,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -313,8 +313,8 @@ class EasyAnimateControlPipeline(DiffusionPipeline): def __init__( self, vae: AutoencoderKLMagvit, - text_encoder: Union[Qwen2VLForConditionalGeneration, BertModel], - tokenizer: Union[Qwen2Tokenizer, BertTokenizer], + text_encoder: Qwen2VLForConditionalGeneration | BertModel, + tokenizer: Qwen2Tokenizer | BertTokenizer, transformer: EasyAnimateTransformer3DModel, scheduler: FlowMatchEulerDiscreteScheduler, ): @@ -351,10 +351,10 @@ def __init__( # Copied from diffusers.pipelines.easyanimate.pipeline_easyanimate.EasyAnimatePipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -367,7 +367,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -377,7 +377,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -671,19 +671,19 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_frames: Optional[int] = 49, height: Optional[int] = 512, width: Optional[int] = 512, - control_video: Union[torch.FloatTensor] = None, - control_camera_video: Union[torch.FloatTensor] = None, - ref_image: Union[torch.FloatTensor] = None, + control_video: torch.FloatTensor = None, + control_camera_video: torch.FloatTensor = None, + ref_image: torch.FloatTensor = None, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -692,17 +692,17 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, ): r""" Generates images or video using the EasyAnimate pipeline based on the provided prompts. Examples: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): Text prompts to guide the image or video generation. If not provided, use `prompt_embeds` instead. num_frames (`int`, *optional*): Length of the generated video (in frames). @@ -715,13 +715,13 @@ def __call__( down inference. guidance_scale (`float`, *optional*, defaults to 5.0): Encourages the model to align outputs with prompts. A higher value may decrease image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): Prompts indicating what to exclude in generation. If not specified, use `negative_prompt_embeds`. num_images_per_prompt (`int`, *optional*, defaults to 1): Number of images to generate for each prompt. eta (`float`, *optional*, defaults to 0.0): Applies to DDIM scheduling. Controlled by the eta parameter from the related literature. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A generator to ensure reproducibility in image generation. latents (`torch.Tensor`, *optional*): Predefined latent tensors to condition generation. @@ -739,7 +739,7 @@ def __call__( If `True`, returns a structured output. Otherwise returns a simple tuple. callback_on_step_end (`Callable`, *optional*): Functions called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): Tensor names to be included in callback function calls. guidance_rescale (`float`, *optional*, defaults to 0.0): Adjusts noise levels based on guidance scale. diff --git a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_inpaint.py b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_inpaint.py index b16ef92d8e6b..97a51d324be7 100755 --- a/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_inpaint.py +++ b/src/diffusers/pipelines/easyanimate/pipeline_easyanimate_inpaint.py @@ -14,7 +14,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -271,9 +271,9 @@ def add_noise_to_reference_video(image, ratio=None, generator=None): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -288,15 +288,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -355,8 +355,8 @@ class EasyAnimateInpaintPipeline(DiffusionPipeline): def __init__( self, vae: AutoencoderKLMagvit, - text_encoder: Union[Qwen2VLForConditionalGeneration, BertModel], - tokenizer: Union[Qwen2Tokenizer, BertTokenizer], + text_encoder: Qwen2VLForConditionalGeneration | BertModel, + tokenizer: Qwen2Tokenizer | BertTokenizer, transformer: EasyAnimateTransformer3DModel, scheduler: FlowMatchEulerDiscreteScheduler, ): @@ -393,10 +393,10 @@ def __init__( # Copied from diffusers.pipelines.easyanimate.pipeline_easyanimate.EasyAnimatePipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -409,7 +409,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -419,7 +419,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -789,19 +789,19 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_frames: Optional[int] = 49, - video: Union[torch.FloatTensor] = None, - mask_video: Union[torch.FloatTensor] = None, - masked_video_latents: Union[torch.FloatTensor] = None, + video: torch.FloatTensor = None, + mask_video: torch.FloatTensor = None, + masked_video_latents: torch.FloatTensor = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -810,19 +810,19 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, strength: float = 1.0, noise_aug_strength: float = 0.0563, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, ): r""" The call function to the pipeline for generation with HunyuanDiT. Examples: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. num_frames (`int`, *optional*): Length of the video to be generated in seconds. This parameter influences the number of frames and @@ -843,7 +843,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is effective when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to exclude in image generation. If not defined, you need to provide `negative_prompt_embeds`. This parameter is ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -852,7 +852,7 @@ def __call__( A parameter defined in the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`] and is ignored in other schedulers. It adjusts noise level during the inference process. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) for setting random seeds which helps in making generation deterministic. latents (`torch.Tensor`, *optional*): @@ -878,7 +878,7 @@ def __call__( *optional*): A callback function (or a list of them) that will be executed at the end of each denoising step, allowing for custom processing during generation. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): Specifies which tensor inputs should be included in the callback function. If not defined, all tensor inputs will be passed, facilitating enhanced logging or monitoring of the generation process. guidance_rescale (`float`, *optional*, defaults to 0.0): diff --git a/src/diffusers/pipelines/easyanimate/pipeline_output.py b/src/diffusers/pipelines/easyanimate/pipeline_output.py index c761a3b1079f..4fd904ae7dfe 100644 --- a/src/diffusers/pipelines/easyanimate/pipeline_output.py +++ b/src/diffusers/pipelines/easyanimate/pipeline_output.py @@ -11,8 +11,8 @@ class EasyAnimatePipelineOutput(BaseOutput): Output class for EasyAnimate pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/flux/pipeline_flux.py b/src/diffusers/pipelines/flux/pipeline_flux.py index 5041e352f73d..9154872a3932 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux.py +++ b/src/diffusers/pipelines/flux/pipeline_flux.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -88,9 +88,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -105,15 +105,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -217,7 +217,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -266,7 +266,7 @@ def _get_t5_prompt_embeds( def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -310,8 +310,8 @@ def _get_clip_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -322,9 +322,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -653,49 +653,49 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 3.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. true_cfg_scale (`float`, *optional*, defaults to 1.0): @@ -708,7 +708,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -720,7 +720,7 @@ def __call__( the [paper](https://huggingface.co/papers/2210.03142) to learn more. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -734,13 +734,13 @@ def __call__( Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -766,7 +766,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control.py b/src/diffusers/pipelines/flux/pipeline_flux_control.py index 848d7bd39254..081da53c0613 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -101,9 +101,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -118,15 +118,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -229,7 +229,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -279,7 +279,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -324,8 +324,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -336,9 +336,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -636,38 +636,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, control_image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 3.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -681,7 +681,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -693,7 +693,7 @@ def __call__( the [paper](https://huggingface.co/papers/2210.03142) to learn more. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -720,7 +720,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py b/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py index 262345c75afc..f5440f296467 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -119,9 +119,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -136,15 +136,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -239,7 +239,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -289,7 +289,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -334,8 +334,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -346,9 +346,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -635,46 +635,46 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -694,7 +694,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -706,7 +706,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -733,7 +733,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py index 6915a83a7ca7..cbdeb768e07d 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -146,9 +146,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -163,15 +163,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -279,7 +279,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -329,7 +329,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -374,8 +374,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -386,9 +386,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -806,8 +806,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, mask_image: PipelineImageInput = None, @@ -816,52 +816,52 @@ def __call__( width: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will be generated by `mask_image`. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -877,7 +877,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -889,7 +889,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -916,7 +916,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py index 507ec687347c..e4d58036c321 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -118,9 +118,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -135,15 +135,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -214,9 +214,10 @@ def __init__( text_encoder_2: T5EncoderModel, tokenizer_2: T5TokenizerFast, transformer: FluxTransformer2DModel, - controlnet: Union[ - FluxControlNetModel, List[FluxControlNetModel], Tuple[FluxControlNetModel], FluxMultiControlNetModel - ], + controlnet: FluxControlNetModel + | list[FluxControlNetModel] + | tuple[FluxControlNetModel] + | FluxMultiControlNetModel, image_encoder: CLIPVisionModelWithProjection = None, feature_extractor: CLIPImageProcessor = None, ): @@ -247,7 +248,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -296,7 +297,7 @@ def _get_t5_prompt_embeds( def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -340,8 +341,8 @@ def _get_clip_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -352,9 +353,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -678,47 +679,47 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, control_image: PipelineImageInput = None, - control_mode: Optional[Union[int, List[int]]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + control_mode: Optional[int | list[int]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -728,7 +729,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -738,27 +739,27 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - control_mode (`int` or `List[int]`,, *optional*, defaults to None): + control_mode (`int` or `list[int]`,, *optional*, defaults to None): The control mode when applying ControlNet-Union. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -772,13 +773,13 @@ def __call__( Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -796,7 +797,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py index 582c7bbad84e..5bb4eb7df932 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -113,9 +113,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -130,15 +130,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -209,9 +209,10 @@ def __init__( text_encoder_2: T5EncoderModel, tokenizer_2: T5TokenizerFast, transformer: FluxTransformer2DModel, - controlnet: Union[ - FluxControlNetModel, List[FluxControlNetModel], Tuple[FluxControlNetModel], FluxMultiControlNetModel - ], + controlnet: FluxControlNetModel + | list[FluxControlNetModel] + | tuple[FluxControlNetModel] + | FluxMultiControlNetModel, ): super().__init__() if isinstance(controlnet, (list, tuple)): @@ -239,7 +240,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -289,7 +290,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -334,8 +335,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -346,9 +347,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -635,43 +636,43 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - control_mode: Optional[Union[int, List[int]]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + control_mode: Optional[int | list[int]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.FloatTensor`): The image(s) to modify with the pipeline. - control_image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + control_image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.FloatTensor`): The ControlNet input condition. Image to control the generation. height (`int`, *optional*, defaults to self.default_sample_size * self.vae_scale_factor): The height in pixels of the generated image. @@ -682,21 +683,21 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 28): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.0): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). - control_mode (`int` or `List[int]`, *optional*): + control_mode (`int` or `list[int]`, *optional*): The mode for the ControlNet. If multiple ControlNets are used, this should be a list. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original transformer. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or more [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -714,7 +715,7 @@ def __call__( Additional keyword arguments to be passed to the joint attention mechanism. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising step during the inference. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. max_sequence_length (`int`, *optional*, defaults to 512): The maximum length of the sequence to be generated. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py index f7f34ef231af..03d22672afae 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL @@ -115,9 +115,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -132,15 +132,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -211,9 +211,10 @@ def __init__( text_encoder_2: T5EncoderModel, tokenizer_2: T5TokenizerFast, transformer: FluxTransformer2DModel, - controlnet: Union[ - FluxControlNetModel, List[FluxControlNetModel], Tuple[FluxControlNetModel], FluxMultiControlNetModel - ], + controlnet: FluxControlNetModel + | list[FluxControlNetModel] + | tuple[FluxControlNetModel] + | FluxMultiControlNetModel, ): super().__init__() if isinstance(controlnet, (list, tuple)): @@ -250,7 +251,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -300,7 +301,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -345,8 +346,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -357,9 +358,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -739,8 +740,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: PipelineImageInput = None, @@ -749,41 +750,41 @@ def __call__( width: Optional[int] = None, strength: float = 0.6, padding_mask_crop: Optional[int] = None, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, num_inference_steps: int = 28, guidance_scale: float = 7.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - control_mode: Optional[Union[int, List[int]]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + control_mode: Optional[int | list[int]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.FloatTensor`): The image(s) to inpaint. - mask_image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + mask_image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.FloatTensor`): The mask image(s) to use for inpainting. White pixels in the mask will be repainted, while black pixels will be preserved. masked_image_latents (`torch.FloatTensor`, *optional*): Pre-generated masked image latents. - control_image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + control_image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.FloatTensor`): The ControlNet input condition. Image to control the generation. height (`int`, *optional*, defaults to self.default_sample_size * self.vae_scale_factor): The height in pixels of the generated image. @@ -796,25 +797,25 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 28): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.0): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - control_mode (`int` or `List[int]`, *optional*): + control_mode (`int` or `list[int]`, *optional*): The mode for the ControlNet. If multiple ControlNets are used, this should be a list. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original transformer. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or more [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -832,7 +833,7 @@ def __call__( Additional keyword arguments to be passed to the joint attention mechanism. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising step during the inference. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. max_sequence_length (`int`, *optional*, defaults to 512): The maximum length of the sequence to be generated. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_fill.py b/src/diffusers/pipelines/flux/pipeline_flux_fill.py index 5cb9c82204b2..24796cf7ef06 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_fill.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_fill.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -95,9 +95,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -112,15 +112,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -244,7 +244,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -294,7 +294,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -419,8 +419,8 @@ def prepare_mask_latents( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -431,9 +431,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -753,8 +753,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: Optional[torch.FloatTensor] = None, mask_image: Optional[torch.FloatTensor] = None, masked_image_latents: Optional[torch.FloatTensor] = None, @@ -762,43 +762,43 @@ def __call__( width: Optional[int] = None, strength: float = 1.0, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 30.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will be generated by `mask_image`. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -814,7 +814,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -826,7 +826,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -853,7 +853,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_img2img.py b/src/diffusers/pipelines/flux/pipeline_flux_img2img.py index ab9140dae921..330540ad5c0a 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_img2img.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -111,9 +111,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -128,15 +128,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -238,7 +238,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -288,7 +288,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -333,8 +333,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -345,9 +345,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -735,47 +735,47 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -794,7 +794,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -806,7 +806,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -820,13 +820,13 @@ def __call__( Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -844,7 +844,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py index 3bfe82cf4382..03b224d199bd 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -107,9 +107,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -124,15 +124,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -241,7 +241,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -291,7 +291,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -336,8 +336,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -348,9 +348,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -776,10 +776,10 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, @@ -789,50 +789,50 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will be generated by `mask_image`. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -855,7 +855,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -867,7 +867,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -881,13 +881,13 @@ def __call__( Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -905,7 +905,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_kontext.py b/src/diffusers/pipelines/flux/pipeline_flux_kontext.py index 94ae460afcd0..68b32f538446 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_kontext.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_kontext.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -118,9 +118,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -135,15 +135,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -263,7 +263,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -313,7 +313,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -358,8 +358,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -370,9 +370,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -675,7 +675,7 @@ def prepare_latents( width: int, dtype: torch.dtype, device: torch.device, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ): if isinstance(generator, list) and len(generator) != batch_size: @@ -753,32 +753,32 @@ def interrupt(self): def __call__( self, image: Optional[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 3.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, max_area: int = 1024**2, _auto_resize: bool = True, @@ -787,23 +787,23 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. true_cfg_scale (`float`, *optional*, defaults to 1.0): @@ -815,7 +815,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -827,7 +827,7 @@ def __call__( the [paper](https://huggingface.co/papers/2210.03142) to learn more. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -842,13 +842,13 @@ def __call__( If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -874,7 +874,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py index b6f957981e14..2d2da96048d3 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py @@ -2,7 +2,7 @@ # author: @vuongminh1907 import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -142,9 +142,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -159,15 +159,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -296,7 +296,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -346,7 +346,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -391,8 +391,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -403,9 +403,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -750,7 +750,7 @@ def prepare_latents( width: int, dtype: torch.dtype, device: torch.device, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, image_reference: Optional[torch.Tensor] = None, ): @@ -944,34 +944,34 @@ def __call__( image: Optional[PipelineImageInput] = None, image_reference: Optional[PipelineImageInput] = None, mask_image: PipelineImageInput = None, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: Optional[str | list[str]] = None, true_cfg_scale: float = 1.0, height: Optional[int] = None, width: Optional[int] = None, strength: float = 1.0, padding_mask_crop: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 3.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_ip_adapter_image: Optional[PipelineImageInput] = None, - negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, max_area: int = 1024**2, _auto_resize: bool = True, @@ -980,37 +980,37 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be be inpainted (which parts of the image to be masked out with `mask_image` and repainted according to `prompt` and `image_reference`). For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - image_reference (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image_reference (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point for the masked area. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)` If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. true_cfg_scale (`float`, *optional*, defaults to 1.0): @@ -1036,7 +1036,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1048,7 +1048,7 @@ def __call__( the [paper](https://huggingface.co/papers/2210.03142) to learn more. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -1063,13 +1063,13 @@ def __call__( If not provided, pooled text embeddings will be generated from `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. negative_ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. @@ -1095,7 +1095,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py b/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py index e79db337b2e3..a3965460877e 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import List, Optional, Union +from typing import Optional import torch from PIL import Image @@ -196,7 +196,7 @@ def encode_image(self, image, device, num_images_per_prompt): # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -246,7 +246,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -291,8 +291,8 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -303,9 +303,9 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -373,28 +373,28 @@ def encode_prompt( def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, - prompt_embeds_scale: Optional[Union[float, List[float]]] = 1.0, - pooled_prompt_embeds_scale: Optional[Union[float, List[float]]] = 1.0, + prompt_embeds_scale: Optional[float | list[float]] = 1.0, + pooled_prompt_embeds_scale: Optional[float | list[float]] = 1.0, return_dict: bool = True, ): r""" Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. **experimental feature**: to use this feature, make sure to explicitly load text encoders to the pipeline. Prompts will be ignored if text encoders are not loaded. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. prompt_embeds (`torch.FloatTensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. diff --git a/src/diffusers/pipelines/flux/pipeline_output.py b/src/diffusers/pipelines/flux/pipeline_output.py index 69e742d3e026..7f24bdbf5d60 100644 --- a/src/diffusers/pipelines/flux/pipeline_output.py +++ b/src/diffusers/pipelines/flux/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -14,14 +13,14 @@ class FluxPipelineOutput(BaseOutput): Output class for Flux image generation pipelines. Args: - images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + images (`list[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be passed to the decoder. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray @dataclass @@ -30,8 +29,8 @@ class FluxPriorReduxPipelineOutput(BaseOutput): Output class for Flux Prior Redux pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ diff --git a/src/diffusers/pipelines/free_init_utils.py b/src/diffusers/pipelines/free_init_utils.py index 4495c5ea2683..04a385edd819 100644 --- a/src/diffusers/pipelines/free_init_utils.py +++ b/src/diffusers/pipelines/free_init_utils.py @@ -13,7 +13,6 @@ # limitations under the License. import math -from typing import Tuple, Union import torch import torch.fft as fft @@ -73,8 +72,8 @@ def free_init_enabled(self): def _get_free_init_freq_filter( self, - shape: Tuple[int, ...], - device: Union[str, torch.dtype], + shape: tuple[int, ...], + device: str | torch.dtype, filter_type: str, order: float, spatial_stop_frequency: float, diff --git a/src/diffusers/pipelines/free_noise_utils.py b/src/diffusers/pipelines/free_noise_utils.py index 2910afaf237b..357f9f3cf426 100644 --- a/src/diffusers/pipelines/free_noise_utils.py +++ b/src/diffusers/pipelines/free_noise_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Optional import torch import torch.nn as nn @@ -48,7 +48,7 @@ class SplitInferenceModule(nn.Module): The size of each chunk after splitting the input tensor. split_dim (`int`, defaults to `0`): The dimension along which the input tensors are split. - input_kwargs_to_split (`List[str]`, defaults to `["hidden_states"]`): + input_kwargs_to_split (`list[str]`, defaults to `["hidden_states"]`): A list of keyword arguments (strings) that represent the input tensors to be split. Workflow: @@ -80,7 +80,7 @@ def __init__( module: nn.Module, split_size: int = 1, split_dim: int = 0, - input_kwargs_to_split: List[str] = ["hidden_states"], + input_kwargs_to_split: list[str] = ["hidden_states"], ) -> None: super().__init__() @@ -89,7 +89,7 @@ def __init__( self.split_dim = split_dim self.input_kwargs_to_split = set(input_kwargs_to_split) - def forward(self, *args, **kwargs) -> Union[torch.Tensor, Tuple[torch.Tensor]]: + def forward(self, *args, **kwargs) -> torch.Tensor | tuple[torch.Tensor]: r"""Forward method for the `SplitInferenceModule`. This method processes the input by splitting specified keyword arguments along a given dimension, running the @@ -99,13 +99,13 @@ def forward(self, *args, **kwargs) -> Union[torch.Tensor, Tuple[torch.Tensor]]: Args: *args (`Any`): Positional arguments that are passed directly to the `module` without modification. - **kwargs (`Dict[str, torch.Tensor]`): + **kwargs (`dict[str, torch.Tensor]`): Keyword arguments passed to the underlying `module`. Only keyword arguments whose names match the entries in `input_kwargs_to_split` and are of type `torch.Tensor` will be split. The remaining keyword arguments are passed unchanged. Returns: - `Union[torch.Tensor, Tuple[torch.Tensor]]`: + `Union[torch.Tensor, tuple[torch.Tensor]]`: The outputs obtained from `SplitInferenceModule` are the same as if the underlying module was inferred without it. - If the underlying module returns a single tensor, the result will be a single concatenated tensor @@ -145,7 +145,7 @@ def forward(self, *args, **kwargs) -> Union[torch.Tensor, Tuple[torch.Tensor]]: class AnimateDiffFreeNoiseMixin: r"""Mixin class for [FreeNoise](https://huggingface.co/papers/2310.15169).""" - def _enable_free_noise_in_block(self, block: Union[CrossAttnDownBlockMotion, DownBlockMotion, UpBlockMotion]): + def _enable_free_noise_in_block(self, block: CrossAttnDownBlockMotion | DownBlockMotion | UpBlockMotion): r"""Helper function to enable FreeNoise in transformer blocks.""" for motion_module in block.motion_modules: @@ -186,7 +186,7 @@ def _enable_free_noise_in_block(self, block: Union[CrossAttnDownBlockMotion, Dow basic_transfomer_block._chunk_size, basic_transfomer_block._chunk_dim ) - def _disable_free_noise_in_block(self, block: Union[CrossAttnDownBlockMotion, DownBlockMotion, UpBlockMotion]): + def _disable_free_noise_in_block(self, block: CrossAttnDownBlockMotion | DownBlockMotion | UpBlockMotion): r"""Helper function to disable FreeNoise in transformer blocks.""" for motion_module in block.motion_modules: @@ -255,12 +255,12 @@ def _check_inputs_free_noise( def _encode_prompt_free_noise( self, - prompt: Union[str, Dict[int, str]], + prompt: str | dict[int, str], num_frames: int, device: torch.device, num_videos_per_prompt: int, do_classifier_free_guidance: bool, - negative_prompt: Optional[Union[str, Dict[int, str]]] = None, + negative_prompt: Optional[str | dict[int, str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, lora_scale: Optional[float] = None, @@ -529,7 +529,7 @@ def disable_free_noise(self) -> None: self._disable_free_noise_in_block(block) def _enable_split_inference_motion_modules_( - self, motion_modules: List[AnimateDiffTransformer3D], spatial_split_size: int + self, motion_modules: list[AnimateDiffTransformer3D], spatial_split_size: int ) -> None: for motion_module in motion_modules: motion_module.proj_in = SplitInferenceModule(motion_module.proj_in, spatial_split_size, 0, ["input"]) @@ -545,19 +545,19 @@ def _enable_split_inference_motion_modules_( motion_module.proj_out = SplitInferenceModule(motion_module.proj_out, spatial_split_size, 0, ["input"]) def _enable_split_inference_attentions_( - self, attentions: List[Transformer2DModel], temporal_split_size: int + self, attentions: list[Transformer2DModel], temporal_split_size: int ) -> None: for i in range(len(attentions)): attentions[i] = SplitInferenceModule( attentions[i], temporal_split_size, 0, ["hidden_states", "encoder_hidden_states"] ) - def _enable_split_inference_resnets_(self, resnets: List[ResnetBlock2D], temporal_split_size: int) -> None: + def _enable_split_inference_resnets_(self, resnets: list[ResnetBlock2D], temporal_split_size: int) -> None: for i in range(len(resnets)): resnets[i] = SplitInferenceModule(resnets[i], temporal_split_size, 0, ["input_tensor", "temb"]) def _enable_split_inference_samplers_( - self, samplers: Union[List[Downsample2D], List[Upsample2D]], temporal_split_size: int + self, samplers: list[Downsample2D] | list[Upsample2D], temporal_split_size: int ) -> None: for i in range(len(samplers)): samplers[i] = SplitInferenceModule(samplers[i], temporal_split_size, 0, ["hidden_states"]) diff --git a/src/diffusers/pipelines/hidream_image/pipeline_hidream_image.py b/src/diffusers/pipelines/hidream_image/pipeline_hidream_image.py index b6af23bca8fd..b781414ed327 100644 --- a/src/diffusers/pipelines/hidream_image/pipeline_hidream_image.py +++ b/src/diffusers/pipelines/hidream_image/pipeline_hidream_image.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -101,9 +101,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -118,15 +118,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -202,7 +202,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 128, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -241,7 +241,7 @@ def _get_clip_prompt_embeds( self, tokenizer, text_encoder, - prompt: Union[str, List[str]], + prompt: str | list[str], max_sequence_length: int = 128, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -276,7 +276,7 @@ def _get_clip_prompt_embeds( def _get_llama3_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, max_sequence_length: int = 128, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -320,22 +320,22 @@ def _get_llama3_prompt_embeds( def encode_prompt( self, - prompt: Optional[Union[str, List[str]]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, - prompt_4: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, + prompt_4: Optional[str | list[str]] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, - negative_prompt_4: Optional[Union[str, List[str]]] = None, - prompt_embeds_t5: Optional[List[torch.FloatTensor]] = None, - prompt_embeds_llama3: Optional[List[torch.FloatTensor]] = None, - negative_prompt_embeds_t5: Optional[List[torch.FloatTensor]] = None, - negative_prompt_embeds_llama3: Optional[List[torch.FloatTensor]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, + negative_prompt_4: Optional[str | list[str]] = None, + prompt_embeds_t5: Optional[list[torch.FloatTensor]] = None, + prompt_embeds_llama3: Optional[list[torch.FloatTensor]] = None, + negative_prompt_embeds_t5: Optional[list[torch.FloatTensor]] = None, + negative_prompt_embeds_llama3: Optional[list[torch.FloatTensor]] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, max_sequence_length: int = 128, @@ -729,21 +729,21 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, - prompt_4: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, + prompt_4: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, - negative_prompt_4: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, + negative_prompt_4: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds_t5: Optional[torch.FloatTensor] = None, prompt_embeds_llama3: Optional[torch.FloatTensor] = None, @@ -753,9 +753,9 @@ def __call__( negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 128, **kwargs, ): @@ -763,16 +763,16 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead. - prompt_4 (`str` or `List[str]`, *optional*): + prompt_4 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_4` and `text_encoder_4`. If not defined, `prompt` is will be used instead. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -782,7 +782,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -792,22 +792,22 @@ def __call__( Guidance-distilled models approximates true classifer-free guidance for `guidance_scale` > 1. Refer to the [paper](https://huggingface.co/papers/2210.03142) to learn more. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_4 (`str` or `List[str]`, *optional*): + negative_prompt_4 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_4` and `text_encoder_4`. If not defined, `negative_prompt` is used in all the text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -842,7 +842,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/hidream_image/pipeline_output.py b/src/diffusers/pipelines/hidream_image/pipeline_output.py index 66f0f1260d18..1802c7220691 100644 --- a/src/diffusers/pipelines/hidream_image/pipeline_output.py +++ b/src/diffusers/pipelines/hidream_image/pipeline_output.py @@ -13,7 +13,6 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -27,9 +26,9 @@ class HiDreamImagePipelineOutput(BaseOutput): Output class for HiDreamImage pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py index b50a6ae3ed27..4cbdfa6fc9d1 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -96,9 +96,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -113,15 +113,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -225,14 +225,14 @@ def __init__( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline._get_llama_prompt_embeds def _get_llama_prompt_embeds( self, - prompt: Union[str, List[str]], - prompt_template: Dict[str, Any], + prompt: str | list[str], + prompt_template: dict[str, Any], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, max_sequence_length: int = 256, num_hidden_layers_to_skip: int = 2, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: device = device or self._execution_device dtype = dtype or self.text_encoder.dtype @@ -292,7 +292,7 @@ def _get_llama_prompt_embeds( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -332,9 +332,9 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]] = None, - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + prompt: str | list[str], + prompt_2: str | list[str] = None, + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -424,7 +424,7 @@ def prepare_latents( num_frames: int = 97, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if isinstance(generator, list) and len(generator) != batch_size: @@ -536,19 +536,19 @@ def interrupt(self): def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, - prompt_2: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Union[str, List[str]] = None, + prompt: str | list[str] = None, + prompt_2: str | list[str] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 97, num_inference_steps: int = 50, - sigmas: List[float] = None, + sigmas: list[float] = None, true_cfg_scale: float = 6.0, guidance_scale: float = 1.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -558,29 +558,29 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, max_sequence_length: int = 256, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. height (`int`, defaults to `720`): @@ -592,7 +592,7 @@ def __call__( num_inference_steps (`int`, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -608,7 +608,7 @@ def __call__( conditional latent is not applied. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -645,7 +645,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py index 5c8e295eaf4c..2233659eb1b6 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -85,9 +85,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -102,15 +102,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -198,14 +198,14 @@ def __init__( def _get_llama_prompt_embeds( self, - prompt: Union[str, List[str]], - prompt_template: Dict[str, Any], + prompt: str | list[str], + prompt_template: dict[str, Any], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, max_sequence_length: int = 256, num_hidden_layers_to_skip: int = 2, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: device = device or self._execution_device dtype = dtype or self.text_encoder.dtype @@ -264,7 +264,7 @@ def _get_llama_prompt_embeds( def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -303,9 +303,9 @@ def _get_clip_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]] = None, - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + prompt: str | list[str], + prompt_2: str | list[str] = None, + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -393,7 +393,7 @@ def prepare_latents( num_frames: int = 129, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -492,19 +492,19 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Union[str, List[str]] = None, + prompt: str | list[str] = None, + prompt_2: str | list[str] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: str | list[str] = None, height: int = 720, width: int = 1280, num_frames: int = 129, num_inference_steps: int = 50, - sigmas: List[float] = None, + sigmas: list[float] = None, true_cfg_scale: float = 1.0, guidance_scale: float = 6.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -514,29 +514,29 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, max_sequence_length: int = 256, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. height (`int`, defaults to `720`): @@ -548,7 +548,7 @@ def __call__( num_inference_steps (`int`, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -563,7 +563,7 @@ def __call__( the [paper](https://huggingface.co/papers/2210.03142) to learn more. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -600,7 +600,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_framepack.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_framepack.py index 8006514f47ea..2f1c4a386f14 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_framepack.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_framepack.py @@ -15,7 +15,7 @@ import inspect import math from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -179,9 +179,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -196,15 +196,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -302,14 +302,14 @@ def __init__( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline._get_llama_prompt_embeds def _get_llama_prompt_embeds( self, - prompt: Union[str, List[str]], - prompt_template: Dict[str, Any], + prompt: str | list[str], + prompt_template: dict[str, Any], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, max_sequence_length: int = 256, num_hidden_layers_to_skip: int = 2, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: device = device or self._execution_device dtype = dtype or self.text_encoder.dtype @@ -369,7 +369,7 @@ def _get_llama_prompt_embeds( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -409,9 +409,9 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.HunyuanVideoPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]] = None, - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + prompt: str | list[str], + prompt_2: str | list[str] = None, + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -530,7 +530,7 @@ def prepare_latents( num_frames: int = 129, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -555,7 +555,7 @@ def prepare_image_latents( image: torch.Tensor, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: device = device or self._execution_device @@ -644,20 +644,20 @@ def __call__( self, image: PipelineImageInput, last_image: Optional[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - prompt_2: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Union[str, List[str]] = None, + prompt: str | list[str] = None, + prompt_2: str | list[str] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: str | list[str] = None, height: int = 720, width: int = 1280, num_frames: int = 129, latent_window_size: int = 9, num_inference_steps: int = 50, - sigmas: List[float] = None, + sigmas: list[float] = None, true_cfg_scale: float = 1.0, guidance_scale: float = 6.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, image_latents: Optional[torch.Tensor] = None, last_image_latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, @@ -668,12 +668,12 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, max_sequence_length: int = 256, sampling_type: FramepackSamplingType = FramepackSamplingType.INVERTED_ANTI_DRIFTING, ): @@ -686,17 +686,17 @@ def __call__( last_image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`, *optional*): The optional last image to be used as the ending point for the video generation. This is useful for generating transitions between two images. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. height (`int`, defaults to `720`): @@ -708,7 +708,7 @@ def __call__( num_inference_steps (`int`, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -724,7 +724,7 @@ def __call__( conditional latent is not applied. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. image_latents (`torch.Tensor`, *optional*): @@ -761,7 +761,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py index aa04e6509730..e0bd05146c10 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -148,9 +148,9 @@ def _expand_input_ids_with_image_tokens( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -165,15 +165,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -279,15 +279,15 @@ def __init__( def _get_llama_prompt_embeds( self, image: torch.Tensor, - prompt: Union[str, List[str]], - prompt_template: Dict[str, Any], + prompt: str | list[str], + prompt_template: dict[str, Any], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, max_sequence_length: int = 256, num_hidden_layers_to_skip: int = 2, image_embed_interleave: int = 2, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: device = device or self._execution_device dtype = dtype or self.text_encoder.dtype @@ -417,7 +417,7 @@ def _get_llama_prompt_embeds( def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_videos_per_prompt: int = 1, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -451,9 +451,9 @@ def _get_clip_prompt_embeds( def encode_prompt( self, image: torch.Tensor, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]] = None, - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + prompt: str | list[str], + prompt_2: str | list[str] = None, + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -462,7 +462,7 @@ def encode_prompt( dtype: Optional[torch.dtype] = None, max_sequence_length: int = 256, image_embed_interleave: int = 2, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if prompt_embeds is None: prompt_embeds, prompt_attention_mask = self._get_llama_prompt_embeds( image, @@ -554,7 +554,7 @@ def prepare_latents( num_frames: int = 129, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, image_condition_type: str = "latent_concat", ) -> torch.Tensor: @@ -671,19 +671,19 @@ def interrupt(self): def __call__( self, image: PIL.Image.Image, - prompt: Union[str, List[str]] = None, - prompt_2: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, - negative_prompt_2: Union[str, List[str]] = None, + prompt: str | list[str] = None, + prompt_2: str | list[str] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: str | list[str] = None, height: int = 720, width: int = 1280, num_frames: int = 129, num_inference_steps: int = 50, - sigmas: List[float] = None, + sigmas: list[float] = None, true_cfg_scale: float = 1.0, guidance_scale: float = 1.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -693,12 +693,12 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + prompt_template: dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, max_sequence_length: int = 256, image_embed_interleave: Optional[int] = None, ): @@ -706,17 +706,17 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. height (`int`, defaults to `720`): @@ -728,7 +728,7 @@ def __call__( num_inference_steps (`int`, defaults to `50`): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -744,7 +744,7 @@ def __call__( conditional latent is not applied. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -781,7 +781,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_output.py b/src/diffusers/pipelines/hunyuan_video/pipeline_output.py index fae0370a53b7..4cf966262770 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_output.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -14,8 +13,8 @@ class HunyuanVideoPipelineOutput(BaseOutput): Output class for HunyuanVideo pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ @@ -29,11 +28,11 @@ class HunyuanVideoFramepackPipelineOutput(BaseOutput): Output class for HunyuanVideo pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. Or, a list of torch tensors where each tensor corresponds to a latent that decodes to multiple frames. """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]], List[torch.Tensor]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] | list[torch.Tensor] diff --git a/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py b/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py index e2f935aaf4b9..70a11f5e0599 100644 --- a/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +++ b/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -264,7 +264,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -274,7 +274,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -363,7 +363,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -569,15 +569,15 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_2: Optional[torch.Tensor] = None, @@ -590,20 +590,20 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = (1024, 1024), - target_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = (1024, 1024), + target_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), use_resolution_binning: bool = True, ): r""" The call function to the pipeline for generation with HunyuanDiT. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`): The height in pixels of the generated image. @@ -615,7 +615,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -623,7 +623,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -653,17 +653,17 @@ def __call__( plain tuple. callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A callback function or a list of callback functions to be called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): A list of tensor inputs that should be passed to the callback function. If not defined, all tensor inputs will be passed. guidance_rescale (`float`, *optional*, defaults to 0.0): Rescale the noise_cfg according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). See Section 3.4 - original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): + original_size (`tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): The original size of the image. Used to calculate the time ids. - target_size (`Tuple[int, int]`, *optional*): + target_size (`tuple[int, int]`, *optional*): The target size of the image. Used to calculate the time ids. - crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`): + crops_coords_top_left (`tuple[int, int]`, *optional*, defaults to `(0, 0)`): The top left coordinates of the crop. Used to calculate the time ids. use_resolution_binning (`bool`, *optional*, defaults to `True`): Whether to use resolution binning or not. If `True`, the input resolution will be mapped to the closest diff --git a/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py b/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py index c6cc724a71f0..ab240af63a31 100644 --- a/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +++ b/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import PIL @@ -86,14 +86,14 @@ class I2VGenXLPipelineOutput(BaseOutput): Output class for image-to-video pipeline. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)` """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] class I2VGenXLPipeline( @@ -173,7 +173,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -181,7 +181,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -263,7 +263,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if self.do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -445,7 +445,7 @@ def check_inputs( and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -511,7 +511,7 @@ def prepare_latents( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, height: Optional[int] = 704, width: Optional[int] = 1280, @@ -519,26 +519,26 @@ def __call__( num_frames: int = 16, num_inference_steps: int = 50, guidance_scale: float = 9.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, eta: float = 0.0, num_videos_per_prompt: Optional[int] = 1, decode_chunk_size: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = 1, ): r""" The call function to the pipeline for image-to-video generation with [`I2VGenXLPipeline`]. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.Tensor`): Image or images to guide image generation. If you provide a tensor, it needs to be compatible with [`CLIPImageProcessor`](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): @@ -555,7 +555,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*): @@ -567,7 +567,7 @@ def __call__( The number of frames to decode at a time. The higher the chunk size, the higher the temporal consistency between frames, but also the higher the memory consumption. By default, the decoder will decode all frames at once for maximal quality. Reduce `decode_chunk_size` to reduce memory usage. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -745,7 +745,7 @@ def __call__( # https://github.com/ali-vilab/i2vgen-xl/blob/main/utils/transforms.py. -def _convert_pt_to_pil(image: Union[torch.Tensor, List[torch.Tensor]]): +def _convert_pt_to_pil(image: torch.Tensor | list[torch.Tensor]): if isinstance(image, list) and isinstance(image[0], torch.Tensor): image = torch.cat(image, 0) @@ -761,7 +761,7 @@ def _convert_pt_to_pil(image: Union[torch.Tensor, List[torch.Tensor]]): def _resize_bilinear( - image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], resolution: Tuple[int, int] + image: torch.Tensor | list[torch.Tensor] | PIL.Image.Image | list[PIL.Image.Image], resolution: tuple[int, int] ): # First convert the images to PIL in case they are float tensors (only relevant for tests now). image = _convert_pt_to_pil(image) @@ -774,7 +774,7 @@ def _resize_bilinear( def _center_crop_wide( - image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], resolution: Tuple[int, int] + image: torch.Tensor | list[torch.Tensor] | PIL.Image.Image | list[PIL.Image.Image], resolution: tuple[int, int] ): # First convert the images to PIL in case they are float tensors (only relevant for tests now). image = _convert_pt_to_pil(image) diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py index 33529f5d0954..28cce62da6c6 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch from transformers import ( @@ -105,7 +105,7 @@ def __init__( text_encoder: MultilingualCLIP, tokenizer: XLMRobertaTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, DDPMScheduler], + scheduler: DDIMScheduler | DDPMScheduler, movq: VQModel, ): super().__init__() @@ -173,7 +173,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -237,16 +237,16 @@ def _encode_prompt( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + image_embeds: torch.Tensor | list[torch.Tensor], + negative_image_embeds: torch.Tensor | list[torch.Tensor], + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -257,13 +257,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). height (`int`, *optional*, defaults to 512): @@ -281,7 +281,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py index 7286bcbee17b..8cfb0ed6b810 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import PIL.Image import torch @@ -150,7 +150,7 @@ def __init__( text_encoder: MultilingualCLIP, tokenizer: XLMRobertaTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, DDPMScheduler], + scheduler: DDIMScheduler | DDPMScheduler, movq: VQModel, prior_prior: PriorTransformer, prior_image_encoder: CLIPVisionModelWithProjection, @@ -193,7 +193,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a @@ -216,8 +216,8 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, @@ -225,7 +225,7 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -236,9 +236,9 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -265,7 +265,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -368,7 +368,7 @@ def __init__( text_encoder: MultilingualCLIP, tokenizer: XLMRobertaTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, DDPMScheduler], + scheduler: DDIMScheduler | DDPMScheduler, movq: VQModel, prior_prior: PriorTransformer, prior_image_encoder: CLIPVisionModelWithProjection, @@ -411,7 +411,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a @@ -435,9 +435,9 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(IMAGE2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, @@ -446,7 +446,7 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -457,13 +457,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -496,7 +496,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -609,7 +609,7 @@ def __init__( text_encoder: MultilingualCLIP, tokenizer: XLMRobertaTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, DDPMScheduler], + scheduler: DDIMScheduler | DDPMScheduler, movq: VQModel, prior_prior: PriorTransformer, prior_image_encoder: CLIPVisionModelWithProjection, @@ -652,7 +652,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a @@ -676,10 +676,10 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(INPAINT_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - mask_image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + mask_image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, @@ -687,7 +687,7 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -698,9 +698,9 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. @@ -709,7 +709,7 @@ def __call__( black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -736,7 +736,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py index f5e41d499dc3..1758c7bd585c 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import PIL.Image import torch @@ -207,7 +207,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -298,18 +298,18 @@ def add_noise( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], image_embeds: torch.Tensor, negative_image_embeds: torch.Tensor, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 512, num_inference_steps: int = 100, strength: float = 0.3, guidance_scale: float = 7.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, @@ -319,16 +319,16 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`torch.Tensor`, `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). height (`int`, *optional*, defaults to 512): @@ -352,7 +352,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py index 731fce499859..60910f745caf 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. from copy import deepcopy -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -338,7 +338,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -402,18 +402,18 @@ def _encode_prompt( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image], - mask_image: Union[torch.Tensor, PIL.Image.Image, np.ndarray], + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image, + mask_image: torch.Tensor | PIL.Image.Image | np.ndarray, image_embeds: torch.Tensor, negative_image_embeds: torch.Tensor, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -424,7 +424,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`torch.Tensor`, `PIL.Image.Image` or `np.ndarray`): `Image`, or tensor representing an image batch, that will be used as the starting point for the @@ -437,11 +437,11 @@ def __call__( image or numpy array, mask should also be a either PIL image or numpy array. If it is a PIL image, it will be converted to a single channel (luminance) before use. If it is a nummpy array, the expected shape is `(H, W)`. - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). height (`int`, *optional*, defaults to 512): @@ -459,7 +459,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py index 10ea8005c90d..cb0522a225c3 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -126,12 +126,12 @@ class KandinskyPriorPipelineOutput(BaseOutput): Args: image_embeds (`torch.Tensor`) clip image embeddings for text prompt - negative_image_embeds (`List[PIL.Image.Image]` or `np.ndarray`) + negative_image_embeds (`list[PIL.Image.Image]` or `np.ndarray`) clip image embeddings for unconditional tokens """ - image_embeds: Union[torch.Tensor, np.ndarray] - negative_image_embeds: Union[torch.Tensor, np.ndarray] + image_embeds: torch.Tensor | np.ndarray + negative_image_embeds: torch.Tensor | np.ndarray class KandinskyPriorPipeline(DiffusionPipeline): @@ -182,11 +182,11 @@ def __init__( @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) def interpolate( self, - images_and_prompts: List[Union[str, PIL.Image.Image, torch.Tensor]], - weights: List[float], + images_and_prompts: list[str | PIL.Image.Image | torch.Tensor], + weights: list[float], num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, negative_prior_prompt: Optional[str] = None, negative_prompt: str = "", @@ -197,16 +197,16 @@ def interpolate( Function invoked when using the prior pipeline for interpolation. Args: - images_and_prompts (`List[Union[str, PIL.Image.Image, torch.Tensor]]`): + images_and_prompts (`list[Union[str, PIL.Image.Image, torch.Tensor]]`): list of prompts and images to guide the image generation. - weights: (`List[float]`): + weights: (`list[float]`): list of weights for each condition in `images_and_prompts` num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -216,7 +216,7 @@ def interpolate( negative_prior_prompt (`str`, *optional*): The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). guidance_scale (`float`, *optional*, defaults to 4.0): @@ -346,7 +346,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -407,11 +407,11 @@ def _encode_prompt( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, guidance_scale: float = 4.0, output_type: Optional[str] = "pt", @@ -421,9 +421,9 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -431,7 +431,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py index 429253e99898..928c03262e6c 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import torch @@ -131,28 +131,28 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], + image_embeds: torch.Tensor | list[torch.Tensor], + negative_image_embeds: torch.Tensor | list[torch.Tensor], height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -169,7 +169,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -186,7 +186,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py index fc2083247bb0..c0a9e1437a83 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import PIL.Image import torch @@ -179,7 +179,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a @@ -203,8 +203,8 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, @@ -212,24 +212,24 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, return_dict: bool = True, prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prior_callback_on_step_end_tensor_inputs: list[str] = ["latents"], callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -256,7 +256,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -272,7 +272,7 @@ def __call__( A function that calls at the end of each denoising steps during the inference of the prior pipeline. The function is called with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. - prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + prior_callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your prior pipeline class. @@ -281,7 +281,7 @@ def __call__( The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -407,7 +407,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` @@ -417,7 +417,7 @@ def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[t self.prior_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) self.decoder_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a @@ -441,9 +441,9 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(IMAGE2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, strength: float = 0.3, @@ -452,28 +452,28 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, return_dict: bool = True, prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prior_callback_on_step_end_tensor_inputs: list[str] = ["latents"], callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -506,7 +506,7 @@ def __call__( prior_num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -656,7 +656,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a @@ -680,10 +680,10 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(INPAINT_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - mask_image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + mask_image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_prompt: Optional[str | list[str]] = None, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, @@ -691,23 +691,23 @@ def __call__( width: int = 512, prior_guidance_scale: float = 4.0, prior_num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prior_callback_on_step_end_tensor_inputs: list[str] = ["latents"], callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. @@ -716,7 +716,7 @@ def __call__( black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -743,7 +743,7 @@ def __call__( prior_num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -759,7 +759,7 @@ def __call__( A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. - prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + prior_callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -768,7 +768,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py index c5faae82796b..9155d4788281 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch @@ -160,15 +160,15 @@ def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): @torch.no_grad() def __call__( self, - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], + image_embeds: torch.Tensor | list[torch.Tensor], + negative_image_embeds: torch.Tensor | list[torch.Tensor], hint: torch.Tensor, height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, @@ -179,15 +179,15 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. hint (`torch.Tensor`): The controlnet condition. - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). height (`int`, *optional*, defaults to 512): @@ -205,7 +205,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py index 54154c6ec1f2..ab11e07f7f9e 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import PIL.Image import torch @@ -200,9 +200,9 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt @torch.no_grad() def __call__( self, - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], + image_embeds: torch.Tensor | list[torch.Tensor], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_image_embeds: torch.Tensor | list[torch.Tensor], hint: torch.Tensor, height: int = 512, width: int = 512, @@ -210,7 +210,7 @@ def __call__( guidance_scale: float = 4.0, strength: float = 0.3, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, @@ -220,9 +220,9 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. @@ -234,7 +234,7 @@ def __call__( `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. hint (`torch.Tensor`): The controlnet condition. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -251,7 +251,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py index 3b2509098fd1..509ecc5bf7f1 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import PIL.Image import torch @@ -183,29 +183,29 @@ def num_timesteps(self): @torch.no_grad() def __call__( self, - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], + image_embeds: torch.Tensor | list[torch.Tensor], + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image], + negative_image_embeds: torch.Tensor | list[torch.Tensor], height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, strength: float = 0.3, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. @@ -215,7 +215,7 @@ def __call__( denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will be maximum and the denoising process will run for the full number of iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -232,7 +232,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): @@ -245,7 +245,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py index a61673293e1f..4fbf7b563785 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py @@ -13,7 +13,7 @@ # limitations under the License. from copy import deepcopy -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import PIL.Image @@ -302,28 +302,28 @@ def num_timesteps(self): @torch.no_grad() def __call__( self, - image_embeds: Union[torch.Tensor, List[torch.Tensor]], - image: Union[torch.Tensor, PIL.Image.Image], - mask_image: Union[torch.Tensor, PIL.Image.Image, np.ndarray], - negative_image_embeds: Union[torch.Tensor, List[torch.Tensor]], + image_embeds: torch.Tensor | list[torch.Tensor], + image: torch.Tensor | PIL.Image.Image, + mask_image: torch.Tensor | PIL.Image.Image | np.ndarray, + negative_image_embeds: torch.Tensor | list[torch.Tensor], height: int = 512, width: int = 512, num_inference_steps: int = 100, guidance_scale: float = 4.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. image (`PIL.Image.Image`): `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will @@ -333,7 +333,7 @@ def __call__( black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`. - negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): + negative_image_embeds (`torch.Tensor` or `list[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -350,7 +350,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -367,7 +367,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py index bc67847831a5..4a01df6bad14 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import PIL.Image import torch @@ -137,11 +137,11 @@ def __init__( @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) def interpolate( self, - images_and_prompts: List[Union[str, PIL.Image.Image, torch.Tensor]], - weights: List[float], + images_and_prompts: list[str | PIL.Image.Image | torch.Tensor], + weights: list[float], num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, negative_prior_prompt: Optional[str] = None, negative_prompt: str = "", @@ -152,16 +152,16 @@ def interpolate( Function invoked when using the prior pipeline for interpolation. Args: - images_and_prompts (`List[Union[str, PIL.Image.Image, torch.Tensor]]`): + images_and_prompts (`list[Union[str, PIL.Image.Image, torch.Tensor]]`): list of prompts and images to guide the image generation. - weights: (`List[float]`): + weights: (`list[float]`): list of weights for each condition in `images_and_prompts` num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -171,7 +171,7 @@ def interpolate( negative_prior_prompt (`str`, *optional*): The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). guidance_scale (`float`, *optional*, defaults to 4.0): @@ -303,7 +303,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -376,25 +376,25 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, guidance_scale: float = 4.0, output_type: Optional[str] = "pt", # pt only return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -402,7 +402,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -425,7 +425,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py index b586d166118b..ef159a304669 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union +from typing import Optional import PIL.Image import torch @@ -161,11 +161,11 @@ def get_timesteps(self, num_inference_steps, strength, device): @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) def interpolate( self, - images_and_prompts: List[Union[str, PIL.Image.Image, torch.Tensor]], - weights: List[float], + images_and_prompts: list[str | PIL.Image.Image | torch.Tensor], + weights: list[float], num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, negative_prior_prompt: Optional[str] = None, negative_prompt: str = "", @@ -176,16 +176,16 @@ def interpolate( Function invoked when using the prior pipeline for interpolation. Args: - images_and_prompts (`List[Union[str, PIL.Image.Image, torch.Tensor]]`): + images_and_prompts (`list[Union[str, PIL.Image.Image, torch.Tensor]]`): list of prompts and images to guide the image generation. - weights: (`List[float]`): + weights: (`list[float]`): list of weights for each condition in `images_and_prompts` num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -195,7 +195,7 @@ def interpolate( negative_prior_prompt (`str`, *optional*): The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). guidance_scale (`float`, *optional*, defaults to 4.0): @@ -249,7 +249,7 @@ def interpolate( def _encode_image( self, - image: Union[torch.Tensor, List[PIL.Image.Image]], + image: torch.Tensor | list[PIL.Image.Image], device, num_images_per_prompt, ): @@ -341,7 +341,7 @@ def _encode_prompt( text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -402,13 +402,13 @@ def _encode_prompt( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], + prompt: str | list[str], + image: torch.Tensor | list[torch.Tensor] | PIL.Image.Image | list[PIL.Image.Image], strength: float = 0.3, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, guidance_scale: float = 4.0, output_type: Optional[str] = "pt", # pt only return_dict: bool = True, @@ -417,7 +417,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. strength (`float`, *optional*, defaults to 0.8): Conceptually, indicates how much to transform the reference `emb`. Must be between 0 and 1. `image` @@ -425,7 +425,7 @@ def __call__( denoising steps depends on the amount of noise initially added. emb (`torch.Tensor`): The image embedding. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -433,7 +433,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. guidance_scale (`float`, *optional*, defaults to 4.0): @@ -480,7 +480,7 @@ def __call__( prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt ) - if not isinstance(image, List): + if not isinstance(image, list): image = [image] if isinstance(image[0], torch.Tensor): diff --git a/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py b/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py index 57cc0270442d..84fdacfc06db 100644 --- a/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +++ b/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -106,7 +106,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -114,7 +114,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -181,7 +181,7 @@ def encode_prompt( attention_mask = attention_mask.repeat(num_images_per_prompt, 1) # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size @@ -335,14 +335,14 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 25, guidance_scale: float = 3.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = 1024, width: Optional[int] = 1024, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, @@ -351,20 +351,20 @@ def __call__( return_dict: bool = True, latents=None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 3.0): @@ -373,7 +373,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -386,7 +386,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py b/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py index c7b8022c22b1..b5bdd24289c7 100644 --- a/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +++ b/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py @@ -1,5 +1,5 @@ import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import PIL import PIL.Image @@ -121,7 +121,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -129,7 +129,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -196,7 +196,7 @@ def encode_prompt( attention_mask = attention_mask.repeat(num_images_per_prompt, 1) # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size @@ -401,14 +401,14 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - image: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None, + prompt: str | list[str] = None, + image: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image] = None, strength: float = 0.3, num_inference_steps: int = 25, guidance_scale: float = 3.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, @@ -416,17 +416,17 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. strength (`float`, *optional*, defaults to 0.8): @@ -444,13 +444,13 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -474,7 +474,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kandinsky5/pipeline_kandinsky.py b/src/diffusers/pipelines/kandinsky5/pipeline_kandinsky.py index a1122a82565e..d121fc598795 100644 --- a/src/diffusers/pipelines/kandinsky5/pipeline_kandinsky.py +++ b/src/diffusers/pipelines/kandinsky5/pipeline_kandinsky.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import regex as re import torch @@ -263,7 +263,7 @@ def get_sparse_params(self, sample, device): def _encode_prompt_qwen( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, max_sequence_length: int = 256, dtype: Optional[torch.dtype] = None, @@ -275,14 +275,14 @@ def _encode_prompt_qwen( video generation. Args: - prompt (Union[str, List[str]]): Input prompt or list of prompts + prompt (Union[str, list[str]]): Input prompt or list of prompts device (torch.device): Device to run encoding on num_videos_per_prompt (int): Number of videos to generate per prompt max_sequence_length (int): Maximum sequence length for tokenization dtype (torch.dtype): Data type for embeddings Returns: - Tuple[torch.Tensor, torch.Tensor]: Text embeddings and cumulative sequence lengths + tuple[torch.Tensor, torch.Tensor]: Text embeddings and cumulative sequence lengths """ device = device or self._execution_device dtype = dtype or self.text_encoder.dtype @@ -313,7 +313,7 @@ def _encode_prompt_qwen( def _encode_prompt_clip( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -324,7 +324,7 @@ def _encode_prompt_clip( semantic information. Args: - prompt (Union[str, List[str]]): Input prompt or list of prompts + prompt (Union[str, list[str]]): Input prompt or list of prompts device (torch.device): Device to run encoding on num_videos_per_prompt (int): Number of videos to generate per prompt dtype (torch.dtype): Data type for embeddings @@ -350,7 +350,7 @@ def _encode_prompt_clip( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_videos_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -363,7 +363,7 @@ def encode_prompt( representations for video generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): Prompt to be encoded. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. @@ -375,7 +375,7 @@ def encode_prompt( Torch dtype. Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - Qwen text embeddings of shape (batch_size * num_videos_per_prompt, sequence_length, embedding_dim) - CLIP pooled embeddings of shape (batch_size * num_videos_per_prompt, clip_embedding_dim) - Cumulative sequence lengths (`cu_seqlens`) for Qwen embeddings of shape (batch_size * @@ -528,7 +528,7 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: """ @@ -611,15 +611,15 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 768, num_frames: int = 121, num_inference_steps: int = 50, guidance_scale: float = 5.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds_qwen: Optional[torch.Tensor] = None, prompt_embeds_clip: Optional[torch.Tensor] = None, @@ -630,9 +630,9 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, **kwargs, ): @@ -640,9 +640,9 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the video generation. If not defined, pass `prompt_embeds` instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to avoid during video generation. If not defined, pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale` < `1`). height (`int`, defaults to `512`): @@ -657,7 +657,7 @@ def __call__( Guidance scale as defined in classifier-free guidance. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A torch generator to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents. @@ -671,7 +671,7 @@ def __call__( Whether or not to return a [`KandinskyPipelineOutput`]. callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A function that is called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. max_sequence_length (`int`, defaults to `512`): The maximum sequence length for text encoding. diff --git a/src/diffusers/pipelines/kandinsky5/pipeline_output.py b/src/diffusers/pipelines/kandinsky5/pipeline_output.py index ed77d42a9a83..3f5022b65914 100644 --- a/src/diffusers/pipelines/kandinsky5/pipeline_output.py +++ b/src/diffusers/pipelines/kandinsky5/pipeline_output.py @@ -11,8 +11,8 @@ class KandinskyPipelineOutput(BaseOutput): Output class for Wan pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/kolors/pipeline_kolors.py b/src/diffusers/pipelines/kolors/pipeline_kolors.py index 948f73ed91eb..e93d1ee3872e 100644 --- a/src/diffusers/pipelines/kolors/pipeline_kolors.py +++ b/src/diffusers/pipelines/kolors/pipeline_kolors.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection @@ -65,9 +65,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -82,15 +82,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -214,7 +214,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -222,7 +222,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -292,7 +292,7 @@ def encode_prompt( if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: negative_prompt_embeds = torch.zeros_like(prompt_embeds) elif do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -659,45 +659,45 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -713,11 +713,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -734,7 +734,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -743,7 +743,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -765,7 +765,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -779,31 +779,31 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -813,7 +813,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py b/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py index 67d49b9a8c5e..a520189f701c 100644 --- a/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py +++ b/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -84,9 +84,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -101,15 +101,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -234,7 +234,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -242,7 +242,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -312,7 +312,7 @@ def encode_prompt( if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: negative_prompt_embeds = torch.zeros_like(prompt_embeds) elif do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -791,51 +791,51 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, strength: float = 0.3, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`): The image(s) to modify with the pipeline. strength (`float`, *optional*, defaults to 0.3): Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` @@ -857,11 +857,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -885,7 +885,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -894,7 +894,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -916,7 +916,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -930,31 +930,31 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -964,7 +964,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/kolors/pipeline_output.py b/src/diffusers/pipelines/kolors/pipeline_output.py index 310ee7e8a89b..0418191cc6e5 100644 --- a/src/diffusers/pipelines/kolors/pipeline_output.py +++ b/src/diffusers/pipelines/kolors/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class KolorsPipelineOutput(BaseOutput): Output class for Kolors pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/kolors/text_encoder.py b/src/diffusers/pipelines/kolors/text_encoder.py index 6fd17156a116..0f619a581408 100644 --- a/src/diffusers/pipelines/kolors/text_encoder.py +++ b/src/diffusers/pipelines/kolors/text_encoder.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import List, Optional, Tuple +from typing import Optional import torch import torch.nn.functional as F @@ -234,7 +234,7 @@ def split_tensor_along_last_dim( tensor: torch.Tensor, num_partitions: int, contiguous_split_chunks: bool = False, -) -> List[torch.Tensor]: +) -> list[torch.Tensor]: """Split a tensor along its last dimension. Arguments: @@ -802,7 +802,7 @@ def forward( position_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.BoolTensor] = None, full_attention_mask: Optional[torch.BoolTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + past_key_values: Optional[tuple[tuple[torch.Tensor, torch.Tensor], ...]] = None, inputs_embeds: Optional[torch.Tensor] = None, use_cache: Optional[bool] = None, output_hidden_states: Optional[bool] = None, diff --git a/src/diffusers/pipelines/kolors/tokenizer.py b/src/diffusers/pipelines/kolors/tokenizer.py index b824ba12e079..2f2fad8bcd39 100644 --- a/src/diffusers/pipelines/kolors/tokenizer.py +++ b/src/diffusers/pipelines/kolors/tokenizer.py @@ -15,7 +15,7 @@ import json import os import re -from typing import Dict, List, Optional, Union +from typing import Optional from sentencepiece import SentencePieceProcessor from transformers import PreTrainedTokenizer @@ -61,7 +61,7 @@ def tokenize(self, s: str, encode_special_tokens=False): else: return self.sp_model.EncodeAsPieces(s) - def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: + def encode(self, s: str, bos: bool = False, eos: bool = False) -> list[int]: assert isinstance(s, str) t = self.sp_model.encode(s) if bos: @@ -70,7 +70,7 @@ def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: t = t + [self.eos_id] return t - def decode(self, t: List[int]) -> str: + def decode(self, t: list[int]) -> str: text, buffer = "", [] for token in t: if token in self.index_special_tokens: @@ -84,7 +84,7 @@ def decode(self, t: List[int]) -> str: text += self.sp_model.decode(buffer) return text - def decode_tokens(self, tokens: List[str]) -> str: + def decode_tokens(self, tokens: list[str]) -> str: text = self.sp_model.DecodePieces(tokens) return text @@ -192,7 +192,7 @@ def _convert_id_to_token(self, index): """Converts an index (integer) in a token (str) using the vocab.""" return self.tokenizer.convert_id_to_token(index) - def convert_tokens_to_string(self, tokens: List[str]) -> str: + def convert_tokens_to_string(self, tokens: list[str]) -> str: return self.tokenizer.decode_tokens(tokens) def save_vocabulary(self, save_directory, filename_prefix=None): @@ -206,7 +206,7 @@ def save_vocabulary(self, save_directory, filename_prefix=None): An optional prefix to add to the named of the saved files. Returns: - `Tuple(str)`: Paths to the files saved. + `tuple(str)`: Paths to the files saved. """ if os.path.isdir(save_directory): vocab_file = os.path.join(save_directory, self.vocab_files_names["vocab_file"]) @@ -246,8 +246,8 @@ def build_chat_input(self, query, history=None, role="user"): return self.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True) def build_inputs_with_special_tokens( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: + self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None + ) -> list[int]: """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format: @@ -256,13 +256,13 @@ def build_inputs_with_special_tokens( - pair of sequences: `[CLS] A [SEP] B [SEP]` Args: - token_ids_0 (`List[int]`): - List of IDs to which the special tokens will be added. - token_ids_1 (`List[int]`, *optional*): + token_ids_0 (`list[int]`): + list of IDs to which the special tokens will be added. + token_ids_1 (`list[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + `list[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens. """ prefix_tokens = self.get_prefix_tokens() token_ids_0 = prefix_tokens + token_ids_0 @@ -272,7 +272,7 @@ def build_inputs_with_special_tokens( def _pad( self, - encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + encoded_inputs: dict[str, EncodedInput] | BatchEncoding, max_length: Optional[int] = None, padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, pad_to_multiple_of: Optional[int] = None, @@ -284,7 +284,7 @@ def _pad( Args: encoded_inputs: - Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + Dictionary of tokenized inputs (`list[int]`) or batch of tokenized inputs (`list[list[int]]`). max_length: maximum length of the returned list and optionally padding length (see below). Will truncate by taking into account the special tokens. padding_strategy: PaddingStrategy to use for padding. diff --git a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py index 59f733a498ed..c6a387b69ace 100644 --- a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +++ b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py @@ -16,7 +16,7 @@ # and https://github.com/hojonathanho/diffusion import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -69,9 +69,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -86,15 +86,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -254,7 +254,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -262,7 +262,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -361,7 +361,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -635,7 +635,7 @@ def get_timesteps(self, num_inference_steps, strength, device): def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], strength: float, callback_steps: int, prompt_embeds: Optional[torch.Tensor] = None, @@ -710,32 +710,32 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, num_inference_steps: int = 4, strength: float = 0.8, original_inference_steps: int = None, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 8.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -749,7 +749,7 @@ def __call__( we will draw `num_inference_steps` evenly spaced timesteps from as our final timestep schedule, following the Skipping-Step method in the paper (see Section 4.3). If not set this will default to the scheduler's `original_inference_steps` attribute. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps on the original LCM training/distillation timestep schedule are used. Must be in descending order. @@ -761,7 +761,7 @@ def __call__( 0`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -773,7 +773,7 @@ def __call__( provided, text embeddings are generated from the `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -794,7 +794,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py index e463884618f5..02b367753d02 100644 --- a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +++ b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py @@ -16,7 +16,7 @@ # and https://github.com/hojonathanho/diffusion import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection @@ -74,9 +74,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -91,15 +91,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -239,7 +239,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -247,7 +247,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -346,7 +346,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -565,7 +565,7 @@ def prepare_extra_step_kwargs(self, generator, eta): # Currently StableDiffusionPipeline.check_inputs with negative prompt stuff removed def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: int, width: int, callback_steps: int, @@ -641,32 +641,32 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 4, original_inference_steps: int = None, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 8.5, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -680,7 +680,7 @@ def __call__( we will draw `num_inference_steps` evenly spaced timesteps from as our final timestep schedule, following the Skipping-Step method in the paper (see Section 4.3). If not set this will default to the scheduler's `original_inference_steps` attribute. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps on the original LCM training/distillation timestep schedule are used. Must be in descending order. @@ -692,7 +692,7 @@ def __call__( 0`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -704,7 +704,7 @@ def __call__( provided, text embeddings are generated from the `prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -725,7 +725,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py index f1bf4701e31f..8feb03234254 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import torch import torch.nn as nn @@ -62,11 +62,11 @@ class LDMTextToImagePipeline(DiffusionPipeline): def __init__( self, - vqvae: Union[VQModel, AutoencoderKL], + vqvae: VQModel | AutoencoderKL, bert: PreTrainedModel, tokenizer: PreTrainedTokenizer, - unet: Union[UNet2DModel, UNet2DConditionModel], - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + unet: UNet2DModel | UNet2DConditionModel, + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, ): super().__init__() self.register_modules(vqvae=vqvae, bert=bert, tokenizer=tokenizer, unet=unet, scheduler=scheduler) @@ -75,23 +75,23 @@ def __init__( @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 1.0, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, **kwargs, - ) -> Union[Tuple, ImagePipelineOutput]: + ) -> tuple | ImagePipelineOutput: r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -236,7 +236,7 @@ def __call__( logger = logging.get_logger(__name__) -LDMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ +LDMBERT_PRETRAINED_MODEL_ARCHIVE_list = [ "ldm-bert", # See all LDMBert models at https://huggingface.co/models?filter=ldmbert ] @@ -346,11 +346,11 @@ def forward( self, hidden_states: torch.Tensor, key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, + past_key_value: Optional[tuple[torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, layer_head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" # if key_value_states are provided this layer is used as a cross-attention layer @@ -382,10 +382,10 @@ def forward( value_states = self._shape(self.v_proj(hidden_states), -1, bsz) if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # if cross_attention save tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. # Further calls to cross_attention layer can then reuse all cross-attention # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # if uni-directional self-attention (decoder) save tuple(torch.Tensor, torch.Tensor) of # all previous decoder key/value_states. Further calls to uni-directional self-attention # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) # if encoder bi-directional self-attention `past_key_value` is always `None` @@ -480,7 +480,7 @@ def forward( attention_mask: torch.Tensor, layer_head_mask: torch.Tensor, output_attentions: Optional[bool] = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """ Args: hidden_states (`torch.Tensor`): input to the layer of shape `(seq_len, batch, embed_dim)` @@ -598,7 +598,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutput]: + ) -> tuple | BaseModelOutput: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py index 631539e5c667..b32bd3675a76 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py @@ -1,5 +1,5 @@ import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import PIL.Image @@ -59,14 +59,12 @@ def __init__( self, vqvae: VQModel, unet: UNet2DModel, - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, ): super().__init__() self.register_modules(vqvae=vqvae, unet=unet, scheduler=scheduler) @@ -74,14 +72,14 @@ def __init__( @torch.no_grad() def __call__( self, - image: Union[torch.Tensor, PIL.Image.Image] = None, + image: torch.Tensor | PIL.Image.Image = None, batch_size: Optional[int] = 1, num_inference_steps: Optional[int] = 100, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - ) -> Union[Tuple, ImagePipelineOutput]: + ) -> tuple | ImagePipelineOutput: r""" The call function to the pipeline for generation. @@ -96,7 +94,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): diff --git a/src/diffusers/pipelines/latte/pipeline_latte.py b/src/diffusers/pipelines/latte/pipeline_latte.py index 4d42a7049ec9..16d9ac5eed0a 100644 --- a/src/diffusers/pipelines/latte/pipeline_latte.py +++ b/src/diffusers/pipelines/latte/pipeline_latte.py @@ -18,7 +18,7 @@ import re import urllib.parse as ul from dataclasses import dataclass -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -81,9 +81,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -98,15 +98,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -205,7 +205,7 @@ def mask_text_embeddings(self, emb, mask): # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -220,9 +220,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the video generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For Latte, this should be "". @@ -614,46 +614,46 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 7.5, num_images_per_prompt: int = 1, video_length: int = 16, height: int = 512, width: int = 512, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: str = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], clean_caption: bool = True, mask_feature: bool = True, enable_temporal_attentions: bool = True, decode_chunk_size: int = 14, - ) -> Union[LattePipelineOutput, Tuple]: + ) -> LattePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the video generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the video generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 7.0): @@ -673,7 +673,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -693,7 +693,7 @@ def __call__( Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A callback function or a list of callback functions to be called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): A list of tensor inputs that should be passed to the callback function. If not defined, all tensor inputs will be passed. clean_caption (`bool`, *optional*, defaults to `True`): diff --git a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py index fbf4dc23d043..8fa224c9a330 100644 --- a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +++ b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py @@ -1,7 +1,7 @@ import inspect import math from itertools import repeat -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch import torch.nn.functional as F @@ -115,7 +115,7 @@ def get_attention(self, step: int): return attention def aggregate_attention( - self, attention_maps, prompts, res: Union[int, Tuple[int]], from_where: List[str], is_cross: bool, select: int + self, attention_maps, prompts, res: int | tuple[int], from_where: list[str], is_cross: bool, select: int ): out = [[] for x in range(self.batch_size)] if isinstance(res, int): @@ -309,7 +309,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, DPMSolverMultistepScheduler], + scheduler: DDIMScheduler | DPMSolverMultistepScheduler, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = True, @@ -540,11 +540,11 @@ def encode_prompt( number of images that should be generated per prompt enable_edit_guidance (`bool`): whether to perform any editing or reconstruct the input image instead - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - editing_prompt (`str` or `List[str]`, *optional*): + editing_prompt (`str` or `list[str]`, *optional*): Editing prompt(s) to be encoded. If not defined, one has to pass `editing_prompt_embeds` instead. editing_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not @@ -574,7 +574,7 @@ def encode_prompt( num_edit_tokens = None if negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif isinstance(negative_prompt, str): @@ -774,29 +774,29 @@ def disable_vae_tiling(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - negative_prompt: Optional[Union[str, List[str]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + negative_prompt: Optional[str | list[str]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - editing_prompt: Optional[Union[str, List[str]]] = None, + editing_prompt: Optional[str | list[str]] = None, editing_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, - reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, - edit_guidance_scale: Optional[Union[float, List[float]]] = 5, - edit_warmup_steps: Optional[Union[int, List[int]]] = 0, - edit_cooldown_steps: Optional[Union[int, List[int]]] = None, - edit_threshold: Optional[Union[float, List[float]]] = 0.9, + reverse_editing_direction: Optional[bool | list[bool]] = False, + edit_guidance_scale: Optional[float | list[float]] = 5, + edit_warmup_steps: Optional[int | list[int]] = 0, + edit_cooldown_steps: Optional[int | list[int]] = None, + edit_threshold: Optional[float | list[float]] = 0.9, user_mask: Optional[torch.Tensor] = None, - sem_guidance: Optional[List[torch.Tensor]] = None, + sem_guidance: Optional[list[torch.Tensor]] = None, use_cross_attn_mask: bool = False, use_intersect_mask: bool = True, - attn_store_steps: Optional[List[int]] = [], + attn_store_steps: Optional[list[int]] = [], store_averaged_over_steps: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" @@ -805,7 +805,7 @@ def __call__( always be performed for the last inverted image(s). Args: - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). generator (`torch.Generator`, *optional*): @@ -817,7 +817,7 @@ def __call__( return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] instead of a plain tuple. - editing_prompt (`str` or `List[str]`, *optional*): + editing_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. The image is reconstructed by setting `editing_prompt = None`. Guidance direction of prompt should be specified via `reverse_editing_direction`. @@ -827,25 +827,25 @@ def __call__( negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. - reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + reverse_editing_direction (`bool` or `list[bool]`, *optional*, defaults to `False`): Whether the corresponding prompt in `editing_prompt` should be increased or decreased. - edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + edit_guidance_scale (`float` or `list[float]`, *optional*, defaults to 5): Guidance scale for guiding the image generation. If provided as list values should correspond to `editing_prompt`. `edit_guidance_scale` is defined as `s_e` of equation 12 of [LEDITS++ Paper](https://huggingface.co/papers/2301.12247). - edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + edit_warmup_steps (`float` or `list[float]`, *optional*, defaults to 10): Number of diffusion steps (for each prompt) for which guidance will not be applied. - edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + edit_cooldown_steps (`float` or `list[float]`, *optional*, defaults to `None`): Number of diffusion steps (for each prompt) after which guidance will no longer be applied. - edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + edit_threshold (`float` or `list[float]`, *optional*, defaults to 0.9): Masking threshold of guidance. Threshold should be proportional to the image region that is modified. 'edit_threshold' is defined as 'λ' of equation 12 of [LEDITS++ Paper](https://huggingface.co/papers/2301.12247). user_mask (`torch.Tensor`, *optional*): User-provided mask for even better control over the editing process. This is helpful when LEDITS++'s implicit masks do not meet user preferences. - sem_guidance (`List[torch.Tensor]`, *optional*): - List of pre-generated guidance vectors to be applied at generation. Length of the list has to + sem_guidance (`list[torch.Tensor]`, *optional*): + list of pre-generated guidance vectors to be applied at generation. Length of the list has to correspond to `num_inference_steps`. use_cross_attn_mask (`bool`, defaults to `False`): Whether cross-attention masks are used. Cross-attention masks are always used when use_intersect_mask @@ -855,7 +855,7 @@ def __call__( Whether the masking term is calculated as intersection of cross-attention masks and masks derived from the noise estimate. Cross-attention mask are defined as 'M^1' and masks derived from the noise estimate are defined as 'M^2' of equation 12 of [LEDITS++ paper](https://huggingface.co/papers/2311.16711). - attn_store_steps (`List[int]`, *optional*): + attn_store_steps (`list[int]`, *optional*): Steps for which the attention maps are stored in the AttentionStore. Just for visualization purposes. store_averaged_over_steps (`bool`, defaults to `True`): Whether the attention maps for the 'attn_store_steps' are stored averaged over the diffusion steps. If @@ -875,7 +875,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -1283,12 +1283,12 @@ def invert( num_inversion_steps: int = 30, skip: float = 0.15, generator: Optional[torch.Generator] = None, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, height: Optional[int] = None, width: Optional[int] = None, resize_mode: Optional[str] = "default", - crops_coords: Optional[Tuple[int, int, int, int]] = None, + crops_coords: Optional[tuple[int, int, int, int]] = None, ): r""" The function to the pipeline for image inversion as described by the [LEDITS++ @@ -1331,7 +1331,7 @@ def invert( image to fit within the specified width and height, maintaining the aspect ratio, and then center the image within the dimensions, cropping the excess. Note that resize_mode `fill` and `crop` are only supported for PIL image input. - crops_coords (`List[Tuple[int, int, int, int]]`, *optional*, defaults to `None`): + crops_coords (`list[tuple[int, int, int, int]]`, *optional*, defaults to `None`): The crop coordinates for each image in the batch. If `None`, will not crop the image. Returns: diff --git a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py index c1f9a98f0632..d10e1ea4d702 100644 --- a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +++ b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch import torch.nn.functional as F @@ -147,7 +147,7 @@ def get_attention(self, step: int): return attention def aggregate_attention( - self, attention_maps, prompts, res: Union[int, Tuple[int]], from_where: List[str], is_cross: bool, select: int + self, attention_maps, prompts, res: int | tuple[int], from_where: list[str], is_cross: bool, select: int ): out = [[] for x in range(self.batch_size)] if isinstance(res, int): @@ -353,7 +353,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - scheduler: Union[DPMSolverMultistepScheduler, DDIMScheduler], + scheduler: DPMSolverMultistepScheduler | DDIMScheduler, image_encoder: CLIPVisionModelWithProjection = None, feature_extractor: CLIPImageProcessor = None, force_zeros_for_empty_prompt: bool = True, @@ -422,10 +422,10 @@ def encode_prompt( torch device num_images_per_prompt (`int`): number of images that should be generated per prompt - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders negative_prompt_embeds (`torch.Tensor`, *optional*): @@ -443,7 +443,7 @@ def encode_prompt( the output of the pre-final layer will be used for computing the prompt embeddings. enable_edit_guidance (`bool`): Whether to guide towards an editing prompt or not. - editing_prompt (`str` or `List[str]`, *optional*): + editing_prompt (`str` or `list[str]`, *optional*): Editing prompt(s) to be encoded. If not defined and 'enable_edit_guidance' is True, one has to pass `editing_prompt_embeds` instead. editing_prompt_embeds (`torch.Tensor`, *optional*): @@ -497,7 +497,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if batch_size != len(negative_prompt): raise ValueError( @@ -849,34 +849,34 @@ def prepare_unet(self, attention_store, PnP: bool = False): def __call__( self, denoising_end: Optional[float] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - editing_prompt: Optional[Union[str, List[str]]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + editing_prompt: Optional[str | list[str]] = None, editing_prompt_embeddings: Optional[torch.Tensor] = None, editing_pooled_prompt_embeds: Optional[torch.Tensor] = None, - reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, - edit_guidance_scale: Optional[Union[float, List[float]]] = 5, - edit_warmup_steps: Optional[Union[int, List[int]]] = 0, - edit_cooldown_steps: Optional[Union[int, List[int]]] = None, - edit_threshold: Optional[Union[float, List[float]]] = 0.9, - sem_guidance: Optional[List[torch.Tensor]] = None, + reverse_editing_direction: Optional[bool | list[bool]] = False, + edit_guidance_scale: Optional[float | list[float]] = 5, + edit_warmup_steps: Optional[int | list[int]] = 0, + edit_cooldown_steps: Optional[int | list[int]] = None, + edit_threshold: Optional[float | list[float]] = 0.9, + sem_guidance: Optional[list[torch.Tensor]] = None, use_cross_attn_mask: bool = False, use_intersect_mask: bool = False, user_mask: Optional[torch.Tensor] = None, - attn_store_steps: Optional[List[int]] = [], + attn_store_steps: Optional[list[int]] = [], store_averaged_over_steps: bool = True, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" @@ -891,11 +891,11 @@ def __call__( still retain a substantial amount of noise as determined by the discrete timesteps selected by the scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders negative_prompt_embeds (`torch.Tensor`, *optional*): @@ -930,16 +930,16 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - editing_prompt (`str` or `List[str]`, *optional*): + editing_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. The image is reconstructed by setting `editing_prompt = None`. Guidance direction of prompt should be specified via `reverse_editing_direction`. @@ -950,22 +950,22 @@ def __call__( Pre-generated pooled edit text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, editing_prompt_embeddings will be generated from `editing_prompt` input argument. - reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + reverse_editing_direction (`bool` or `list[bool]`, *optional*, defaults to `False`): Whether the corresponding prompt in `editing_prompt` should be increased or decreased. - edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + edit_guidance_scale (`float` or `list[float]`, *optional*, defaults to 5): Guidance scale for guiding the image generation. If provided as list values should correspond to `editing_prompt`. `edit_guidance_scale` is defined as `s_e` of equation 12 of [LEDITS++ Paper](https://huggingface.co/papers/2301.12247). - edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + edit_warmup_steps (`float` or `list[float]`, *optional*, defaults to 10): Number of diffusion steps (for each prompt) for which guidance is not applied. - edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + edit_cooldown_steps (`float` or `list[float]`, *optional*, defaults to `None`): Number of diffusion steps (for each prompt) after which guidance is no longer applied. - edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + edit_threshold (`float` or `list[float]`, *optional*, defaults to 0.9): Masking threshold of guidance. Threshold should be proportional to the image region that is modified. 'edit_threshold' is defined as 'λ' of equation 12 of [LEDITS++ Paper](https://huggingface.co/papers/2301.12247). - sem_guidance (`List[torch.Tensor]`, *optional*): - List of pre-generated guidance vectors to be applied at generation. Length of the list has to + sem_guidance (`list[torch.Tensor]`, *optional*): + list of pre-generated guidance vectors to be applied at generation. Length of the list has to correspond to `num_inference_steps`. use_cross_attn_mask: Whether cross-attention masks are used. Cross-attention masks are always used when use_intersect_mask @@ -991,7 +991,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -1494,13 +1494,13 @@ def invert( num_inversion_steps: int = 50, skip: float = 0.15, generator: Optional[torch.Generator] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + crops_coords_top_left: tuple[int, int] = (0, 0), num_zero_noise_steps: int = 3, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, height: Optional[int] = None, width: Optional[int] = None, resize_mode: Optional[str] = "default", - crops_coords: Optional[Tuple[int, int, int, int]] = None, + crops_coords: Optional[tuple[int, int, int, int]] = None, ): r""" The function to the pipeline for image inversion as described by the [LEDITS++ @@ -1516,11 +1516,11 @@ def invert( if the `source_prompt` is `""`. source_guidance_scale (`float`, defaults to `3.5`): Strength of guidance during inversion. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_inversion_steps (`int`, defaults to `50`): @@ -1531,7 +1531,7 @@ def invert( generator (`torch.Generator`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make inversion deterministic. - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of diff --git a/src/diffusers/pipelines/ledits_pp/pipeline_output.py b/src/diffusers/pipelines/ledits_pp/pipeline_output.py index 756be82b0069..9f88d12e9517 100644 --- a/src/diffusers/pipelines/ledits_pp/pipeline_output.py +++ b/src/diffusers/pipelines/ledits_pp/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -13,16 +13,16 @@ class LEditsPPDiffusionPipelineOutput(BaseOutput): Output class for LEdits++ Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`) - List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or + nsfw_content_detected (`list[bool]`) + list indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] @dataclass @@ -31,13 +31,13 @@ class LEditsPPInversionPipelineOutput(BaseOutput): Output class for LEdits++ Diffusion pipelines. Args: - input_images (`List[PIL.Image.Image]` or `np.ndarray`) - List of the cropped and resized input images as PIL images of length `batch_size` or NumPy array of shape ` + input_images (`list[PIL.Image.Image]` or `np.ndarray`) + list of the cropped and resized input images as PIL images of length `batch_size` or NumPy array of shape ` (batch_size, height, width, num_channels)`. - vae_reconstruction_images (`List[PIL.Image.Image]` or `np.ndarray`) - List of VAE reconstruction of all input images as PIL images of length `batch_size` or NumPy array of shape + vae_reconstruction_images (`list[PIL.Image.Image]` or `np.ndarray`) + list of VAE reconstruction of all input images as PIL images of length `batch_size` or NumPy array of shape ` (batch_size, height, width, num_channels)`. """ - images: Union[List[PIL.Image.Image], np.ndarray] - vae_reconstruction_images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray + vae_reconstruction_images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/ltx/pipeline_ltx.py b/src/diffusers/pipelines/ltx/pipeline_ltx.py index bd23e657c408..ee4f628ac4d7 100644 --- a/src/diffusers/pipelines/ltx/pipeline_ltx.py +++ b/src/diffusers/pipelines/ltx/pipeline_ltx.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -84,9 +84,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -101,15 +101,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -233,7 +233,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 128, device: Optional[torch.device] = None, @@ -282,8 +282,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.mochi.pipeline_mochi.MochiPipeline.encode_prompt with 256->128 def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -298,9 +298,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -536,37 +536,37 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 704, num_frames: int = 161, frame_rate: int = 25, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 3, guidance_rescale: float = 0.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.Tensor] = None, - decode_timestep: Union[float, List[float]] = 0.0, - decode_noise_scale: Optional[Union[float, List[float]]] = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: Optional[float | list[float]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 128, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `512`): @@ -578,7 +578,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -595,7 +595,7 @@ def __call__( Guidance rescale factor should fix overexposure when using zero terminal SNR. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -630,7 +630,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/ltx/pipeline_ltx_condition.py b/src/diffusers/pipelines/ltx/pipeline_ltx_condition.py index 537588f67c95..bcb1e91e074f 100644 --- a/src/diffusers/pipelines/ltx/pipeline_ltx_condition.py +++ b/src/diffusers/pipelines/ltx/pipeline_ltx_condition.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -100,7 +100,7 @@ class LTXVideoCondition: Attributes: image (`PIL.Image.Image`): The image to condition the video on. - video (`List[PIL.Image.Image]`): + video (`list[PIL.Image.Image]`): The video to condition the video on. frame_index (`int`): The frame index at which the image or video will conditionally effect the video generation. @@ -109,7 +109,7 @@ class LTXVideoCondition: """ image: Optional[PIL.Image.Image] = None - video: Optional[List[PIL.Image.Image]] = None + video: Optional[list[PIL.Image.Image]] = None frame_index: int = 0 strength: float = 1.0 @@ -152,9 +152,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -169,15 +169,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -319,7 +319,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -368,8 +368,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.mochi.pipeline_mochi.MochiPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -384,9 +384,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -671,9 +671,9 @@ def add_noise_to_image_conditioning_latents( def prepare_latents( self, - conditions: Optional[List[torch.Tensor]] = None, - condition_strength: Optional[List[float]] = None, - condition_frame_index: Optional[List[int]] = None, + conditions: Optional[list[torch.Tensor]] = None, + condition_strength: Optional[list[float]] = None, + condition_frame_index: Optional[list[int]] = None, batch_size: int = 1, num_channels_latents: int = 128, height: int = 512, @@ -685,7 +685,7 @@ def prepare_latents( generator: Optional[torch.Generator] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: num_latent_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 latent_height = height // self.vae_spatial_compression_ratio latent_width = width // self.vae_spatial_compression_ratio @@ -849,61 +849,61 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - conditions: Union[LTXVideoCondition, List[LTXVideoCondition]] = None, - image: Union[PipelineImageInput, List[PipelineImageInput]] = None, - video: List[PipelineImageInput] = None, - frame_index: Union[int, List[int]] = 0, - strength: Union[float, List[float]] = 1.0, + conditions: LTXVideoCondition | list[LTXVideoCondition] = None, + image: PipelineImageInput | list[PipelineImageInput] = None, + video: list[PipelineImageInput] = None, + frame_index: int | list[int] = 0, + strength: float | list[float] = 1.0, denoise_strength: float = 1.0, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 704, num_frames: int = 161, frame_rate: int = 25, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 3, guidance_rescale: float = 0.0, image_cond_noise_scale: float = 0.15, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.Tensor] = None, - decode_timestep: Union[float, List[float]] = 0.0, - decode_noise_scale: Optional[Union[float, List[float]]] = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: Optional[float | list[float]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - conditions (`List[LTXVideoCondition], *optional*`): + conditions (`list[LTXVideoCondition], *optional*`): The list of frame-conditioning items for the video generation.If not provided, conditions will be created using `image`, `video`, `frame_index` and `strength`. - image (`PipelineImageInput` or `List[PipelineImageInput]`, *optional*): + image (`PipelineImageInput` or `list[PipelineImageInput]`, *optional*): The image or images to condition the video generation. If not provided, one has to pass `video` or `conditions`. - video (`List[PipelineImageInput]`, *optional*): + video (`list[PipelineImageInput]`, *optional*): The video to condition the video generation. If not provided, one has to pass `image` or `conditions`. - frame_index (`int` or `List[int]`, *optional*): + frame_index (`int` or `list[int]`, *optional*): The frame index or frame indices at which the image or video will conditionally effect the video generation. If not provided, one has to pass `conditions`. - strength (`float` or `List[float]`, *optional*): + strength (`float` or `list[float]`, *optional*): The strength or strengths of the conditioning effect. If not provided, one has to pass `conditions`. denoise_strength (`float`, defaults to `1.0`): The strength of the noise added to the latents for editing. Higher strength leads to more noise added to the latents, therefore leading to more differences between original video and generated video. This is useful for video-to-video editing. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `512`): @@ -915,7 +915,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -932,7 +932,7 @@ def __call__( Guidance rescale factor should fix overexposure when using zero terminal SNR. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -967,7 +967,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/ltx/pipeline_ltx_image2video.py b/src/diffusers/pipelines/ltx/pipeline_ltx_image2video.py index 694378b4f040..da33cdc6dab1 100644 --- a/src/diffusers/pipelines/ltx/pipeline_ltx_image2video.py +++ b/src/diffusers/pipelines/ltx/pipeline_ltx_image2video.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -89,9 +89,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -106,15 +106,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -256,7 +256,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 128, device: Optional[torch.device] = None, @@ -305,8 +305,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.mochi.pipeline_mochi.MochiPipeline.encode_prompt with 256->128 def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -321,9 +321,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -598,30 +598,30 @@ def interrupt(self): def __call__( self, image: PipelineImageInput = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 704, num_frames: int = 161, frame_rate: int = 25, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 3, guidance_rescale: float = 0.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_attention_mask: Optional[torch.Tensor] = None, - decode_timestep: Union[float, List[float]] = 0.0, - decode_noise_scale: Optional[Union[float, List[float]]] = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: Optional[float | list[float]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 128, ): r""" @@ -630,7 +630,7 @@ def __call__( Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `512`): @@ -642,7 +642,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -659,7 +659,7 @@ def __call__( Guidance rescale factor should fix overexposure when using zero terminal SNR. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -694,7 +694,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/ltx/pipeline_ltx_latent_upsample.py b/src/diffusers/pipelines/ltx/pipeline_ltx_latent_upsample.py index 9acff105e56d..fb163d1b9472 100644 --- a/src/diffusers/pipelines/ltx/pipeline_ltx_latent_upsample.py +++ b/src/diffusers/pipelines/ltx/pipeline_ltx_latent_upsample.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Union +from typing import Optional import torch @@ -243,15 +243,15 @@ def check_inputs(self, video, height, width, latents, tone_map_compression_ratio @torch.no_grad() def __call__( self, - video: Optional[List[PipelineImageInput]] = None, + video: Optional[list[PipelineImageInput]] = None, height: int = 512, width: int = 704, latents: Optional[torch.Tensor] = None, - decode_timestep: Union[float, List[float]] = 0.0, - decode_noise_scale: Optional[Union[float, List[float]]] = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: Optional[float | list[float]] = None, adain_factor: float = 0.0, tone_map_compression_ratio: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, ): diff --git a/src/diffusers/pipelines/ltx/pipeline_output.py b/src/diffusers/pipelines/ltx/pipeline_output.py index 36ec3ea884a2..f5cb34aa508d 100644 --- a/src/diffusers/pipelines/ltx/pipeline_output.py +++ b/src/diffusers/pipelines/ltx/pipeline_output.py @@ -11,8 +11,8 @@ class LTXPipelineOutput(BaseOutput): Output class for LTX pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py b/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py index 69f69d5768a8..6019e8f27970 100644 --- a/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py +++ b/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py @@ -17,7 +17,7 @@ # - Based on pipeline_wan.py, but with supports recieving a condition video appended to the channel dimension. import html -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import regex as re import torch @@ -51,7 +51,7 @@ EXAMPLE_DOC_STRING = """ Examples: ```python - >>> from typing import List + >>> from typing import list >>> import torch >>> from PIL import Image @@ -69,7 +69,7 @@ >>> # Load video - >>> def convert_video(video: List[Image.Image]) -> List[Image.Image]: + >>> def convert_video(video: list[Image.Image]) -> list[Image.Image]: ... video = load_video(url)[:num_frames] ... video = [video[i].resize((width, height)) for i in range(num_frames)] ... return video @@ -196,7 +196,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -238,8 +238,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -252,9 +252,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -377,7 +377,7 @@ def prepare_latents( device: Optional[torch.device] = None, generator: Optional[torch.Generator] = None, latents: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: if isinstance(generator, list) and len(generator) != batch_size: raise ValueError( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" @@ -451,9 +451,9 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - video: List[Image.Image], - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + video: list[Image.Image], + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 480, width: int = 832, num_frames: int = 81, @@ -461,28 +461,28 @@ def __call__( guidance_scale: float = 5.0, guidance_scale_2: Optional[float] = None, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - video (`List[Image.Image]`): + video (`list[Image.Image]`): The video to use as the condition for the video generation. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, pass `prompt_embeds` instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to avoid during image generation. If not defined, pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale` < `1`). height (`int`, defaults to `480`): @@ -506,7 +506,7 @@ def __call__( and the pipeline's `boundary_ratio` are not None. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -529,7 +529,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/lucy/pipeline_output.py b/src/diffusers/pipelines/lucy/pipeline_output.py index cf9ea91fd106..197ce194f475 100644 --- a/src/diffusers/pipelines/lucy/pipeline_output.py +++ b/src/diffusers/pipelines/lucy/pipeline_output.py @@ -11,8 +11,8 @@ class LucyPipelineOutput(BaseOutput): Output class for Lucy pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/lumina/pipeline_lumina.py b/src/diffusers/pipelines/lumina/pipeline_lumina.py index b59c265646cd..53780027e049 100644 --- a/src/diffusers/pipelines/lumina/pipeline_lumina.py +++ b/src/diffusers/pipelines/lumina/pipeline_lumina.py @@ -17,7 +17,7 @@ import math import re import urllib.parse as ul -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import torch from transformers import GemmaPreTrainedModel, GemmaTokenizer, GemmaTokenizerFast @@ -77,9 +77,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -94,15 +94,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -182,7 +182,7 @@ def __init__( scheduler: FlowMatchEulerDiscreteScheduler, vae: AutoencoderKL, text_encoder: GemmaPreTrainedModel, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, ): super().__init__() @@ -205,7 +205,7 @@ def __init__( def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clean_caption: Optional[bool] = False, @@ -261,9 +261,9 @@ def _get_gemma_prompt_embeds( # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, - negative_prompt: Union[str, List[str]] = None, + negative_prompt: str | list[str] = None, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, prompt_embeds: Optional[torch.Tensor] = None, @@ -277,9 +277,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For Lumina-T2I, this should be "". @@ -633,15 +633,15 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, width: Optional[int] = None, height: Optional[int] = None, num_inference_steps: int = 30, guidance_scale: float = 4.0, - negative_prompt: Union[str, List[str]] = None, - sigmas: List[float] = None, + negative_prompt: str | list[str] = None, + sigmas: list[float] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -654,25 +654,25 @@ def __call__( scaling_watershed: Optional[float] = 1.0, proportional_attn: Optional[bool] = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - ) -> Union[ImagePipelineOutput, Tuple]: + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -691,7 +691,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -723,7 +723,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -940,7 +940,7 @@ def __init__( scheduler: FlowMatchEulerDiscreteScheduler, vae: AutoencoderKL, text_encoder: GemmaPreTrainedModel, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, ): deprecation_message = "`LuminaText2ImgPipeline` has been renamed to `LuminaPipeline` and will be removed in a future version. Please use `LuminaPipeline` instead." deprecate("diffusers.pipelines.lumina.pipeline_lumina.LuminaText2ImgPipeline", "0.34", deprecation_message) diff --git a/src/diffusers/pipelines/lumina2/pipeline_lumina2.py b/src/diffusers/pipelines/lumina2/pipeline_lumina2.py index 937803edbcbc..97a24eb6770c 100644 --- a/src/diffusers/pipelines/lumina2/pipeline_lumina2.py +++ b/src/diffusers/pipelines/lumina2/pipeline_lumina2.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -78,9 +78,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -95,15 +95,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -164,7 +164,7 @@ def __init__( scheduler: FlowMatchEulerDiscreteScheduler, vae: AutoencoderKL, text_encoder: Gemma2PreTrainedModel, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, ): super().__init__() @@ -191,10 +191,10 @@ def __init__( def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, max_sequence_length: int = 256, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: device = device or self._execution_device prompt = [prompt] if isinstance(prompt, str) else prompt text_inputs = self.tokenizer( @@ -237,9 +237,9 @@ def _get_gemma_prompt_embeds( # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, - negative_prompt: Union[str, List[str]] = None, + negative_prompt: str | list[str] = None, num_images_per_prompt: int = 1, device: Optional[torch.device] = None, prompt_embeds: Optional[torch.Tensor] = None, @@ -248,14 +248,14 @@ def encode_prompt( negative_prompt_attention_mask: Optional[torch.Tensor] = None, system_prompt: Optional[str] = None, max_sequence_length: int = 256, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For Lumina-T2I, this should be "". @@ -525,15 +525,15 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, width: Optional[int] = None, height: Optional[int] = None, num_inference_steps: int = 30, guidance_scale: float = 4.0, - negative_prompt: Union[str, List[str]] = None, - sigmas: List[float] = None, + negative_prompt: str | list[str] = None, + sigmas: list[float] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -541,29 +541,29 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], system_prompt: Optional[str] = None, cfg_trunc_ratio: float = 1.0, cfg_normalization: bool = True, max_sequence_length: int = 256, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -582,7 +582,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -612,7 +612,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -801,7 +801,7 @@ def __init__( scheduler: FlowMatchEulerDiscreteScheduler, vae: AutoencoderKL, text_encoder: Gemma2PreTrainedModel, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, ): deprecation_message = "`Lumina2Text2ImgPipeline` has been renamed to `Lumina2Pipeline` and will be removed in a future version. Please use `Lumina2Pipeline` instead." deprecate("diffusers.pipelines.lumina2.pipeline_lumina2.Lumina2Text2ImgPipeline", "0.34", deprecation_message) diff --git a/src/diffusers/pipelines/marigold/marigold_image_processing.py b/src/diffusers/pipelines/marigold/marigold_image_processing.py index 5130a876606a..f1ef52f07ef2 100644 --- a/src/diffusers/pipelines/marigold/marigold_image_processing.py +++ b/src/diffusers/pipelines/marigold/marigold_image_processing.py @@ -16,7 +16,7 @@ # More information and citation instructions are available on the # Marigold project website: https://marigoldcomputervision.github.io # -------------------------------------------------------------------------- -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import PIL @@ -47,7 +47,7 @@ def __init__( super().__init__() @staticmethod - def expand_tensor_or_array(images: Union[torch.Tensor, np.ndarray]) -> Union[torch.Tensor, np.ndarray]: + def expand_tensor_or_array(images: torch.Tensor | np.ndarray) -> torch.Tensor | np.ndarray: """ Expand a tensor or array to a specified number of images. """ @@ -90,7 +90,7 @@ def numpy_to_pt(images: np.ndarray) -> torch.Tensor: @staticmethod def resize_antialias( - image: torch.Tensor, size: Tuple[int, int], mode: str, is_aa: Optional[bool] = None + image: torch.Tensor, size: tuple[int, int], mode: str, is_aa: Optional[bool] = None ) -> torch.Tensor: if not torch.is_tensor(image): raise ValueError(f"Invalid input type={type(image)}.") @@ -126,7 +126,7 @@ def resize_to_max_edge(image: torch.Tensor, max_edge_sz: int, mode: str) -> torc return image @staticmethod - def pad_image(image: torch.Tensor, align: int) -> Tuple[torch.Tensor, Tuple[int, int]]: + def pad_image(image: torch.Tensor, align: int) -> tuple[torch.Tensor, tuple[int, int]]: if not torch.is_tensor(image): raise ValueError(f"Invalid input type={type(image)}.") if not torch.is_floating_point(image): @@ -142,7 +142,7 @@ def pad_image(image: torch.Tensor, align: int) -> Tuple[torch.Tensor, Tuple[int, return image, (ph, pw) @staticmethod - def unpad_image(image: torch.Tensor, padding: Tuple[int, int]) -> torch.Tensor: + def unpad_image(image: torch.Tensor, padding: tuple[int, int]) -> torch.Tensor: if not torch.is_tensor(image): raise ValueError(f"Invalid input type={type(image)}.") if not torch.is_floating_point(image): @@ -160,10 +160,10 @@ def unpad_image(image: torch.Tensor, padding: Tuple[int, int]) -> torch.Tensor: @staticmethod def load_image_canonical( - image: Union[torch.Tensor, np.ndarray, Image.Image], + image: torch.Tensor | np.ndarray | Image.Image, device: torch.device = torch.device("cpu"), dtype: torch.dtype = torch.float32, - ) -> Tuple[torch.Tensor, int]: + ) -> tuple[torch.Tensor, int]: if isinstance(image, Image.Image): image = np.array(image) @@ -256,11 +256,11 @@ def preprocess( @staticmethod def colormap( - image: Union[np.ndarray, torch.Tensor], + image: np.ndarray | torch.Tensor, cmap: str = "Spectral", bytes: bool = False, _force_method: Optional[str] = None, - ) -> Union[np.ndarray, torch.Tensor]: + ) -> np.ndarray | torch.Tensor: """ Converts a monochrome image into an RGB image by applying the specified colormap. This function mimics the behavior of matplotlib.colormaps, but allows the user to use the most discriminative color maps ("Spectral", @@ -386,30 +386,28 @@ def method_custom(image, cmap, bytes=False): @staticmethod def visualize_depth( - depth: Union[ - PIL.Image.Image, - np.ndarray, - torch.Tensor, - List[PIL.Image.Image], - List[np.ndarray], - List[torch.Tensor], - ], + depth: PIL.Image.Image + | np.ndarray + | torch.Tensor + | list[PIL.Image.Image] + | list[np.ndarray] + | list[torch.Tensor], val_min: float = 0.0, val_max: float = 1.0, color_map: str = "Spectral", - ) -> List[PIL.Image.Image]: + ) -> list[PIL.Image.Image]: """ Visualizes depth maps, such as predictions of the `MarigoldDepthPipeline`. Args: - depth (`Union[PIL.Image.Image, np.ndarray, torch.Tensor, List[PIL.Image.Image], List[np.ndarray], - List[torch.Tensor]]`): Depth maps. + depth (`Union[PIL.Image.Image, np.ndarray, torch.Tensor, list[PIL.Image.Image], list[np.ndarray], + list[torch.Tensor]]`): Depth maps. val_min (`float`, *optional*, defaults to `0.0`): Minimum value of the visualized depth range. val_max (`float`, *optional*, defaults to `1.0`): Maximum value of the visualized depth range. color_map (`str`, *optional*, defaults to `"Spectral"`): Color map used to convert a single-channel depth prediction into colored representation. - Returns: `List[PIL.Image.Image]` with depth maps visualization. + Returns: `list[PIL.Image.Image]` with depth maps visualization. """ if val_max <= val_min: raise ValueError(f"Invalid values range: [{val_min}, {val_max}].") @@ -451,10 +449,10 @@ def visualize_depth_one(img, idx=None): @staticmethod def export_depth_to_16bit_png( - depth: Union[np.ndarray, torch.Tensor, List[np.ndarray], List[torch.Tensor]], + depth: np.ndarray | torch.Tensor | list[np.ndarray] | list[torch.Tensor], val_min: float = 0.0, val_max: float = 1.0, - ) -> List[PIL.Image.Image]: + ) -> list[PIL.Image.Image]: def export_depth_to_16bit_png_one(img, idx=None): prefix = "Depth" + (f"[{idx}]" if idx else "") if not isinstance(img, np.ndarray) and not torch.is_tensor(img): @@ -487,21 +485,16 @@ def export_depth_to_16bit_png_one(img, idx=None): @staticmethod def visualize_normals( - normals: Union[ - np.ndarray, - torch.Tensor, - List[np.ndarray], - List[torch.Tensor], - ], + normals: np.ndarray | torch.Tensor | list[np.ndarray] | list[torch.Tensor], flip_x: bool = False, flip_y: bool = False, flip_z: bool = False, - ) -> List[PIL.Image.Image]: + ) -> list[PIL.Image.Image]: """ Visualizes surface normals, such as predictions of the `MarigoldNormalsPipeline`. Args: - normals (`Union[np.ndarray, torch.Tensor, List[np.ndarray], List[torch.Tensor]]`): + normals (`Union[np.ndarray, torch.Tensor, list[np.ndarray], list[torch.Tensor]]`): Surface normals. flip_x (`bool`, *optional*, defaults to `False`): Flips the X axis of the normals frame of reference. Default direction is right. @@ -510,7 +503,7 @@ def visualize_normals( flip_z (`bool`, *optional*, defaults to `False`): Flips the Z axis of the normals frame of reference. Default direction is facing the observer. - Returns: `List[PIL.Image.Image]` with surface normals visualization. + Returns: `list[PIL.Image.Image]` with surface normals visualization. """ flip_vec = None if any((flip_x, flip_y, flip_z)): @@ -548,30 +541,25 @@ def visualize_normals_one(img, idx=None): @staticmethod def visualize_intrinsics( - prediction: Union[ - np.ndarray, - torch.Tensor, - List[np.ndarray], - List[torch.Tensor], - ], - target_properties: Dict[str, Any], - color_map: Union[str, Dict[str, str]] = "binary", - ) -> List[Dict[str, PIL.Image.Image]]: + prediction: np.ndarray | torch.Tensor | list[np.ndarray] | list[torch.Tensor], + target_properties: dict[str, Any], + color_map: str | dict[str, str] = "binary", + ) -> list[dict[str, PIL.Image.Image]]: """ Visualizes intrinsic image decomposition, such as predictions of the `MarigoldIntrinsicsPipeline`. Args: - prediction (`Union[np.ndarray, torch.Tensor, List[np.ndarray], List[torch.Tensor]]`): + prediction (`Union[np.ndarray, torch.Tensor, list[np.ndarray], list[torch.Tensor]]`): Intrinsic image decomposition. - target_properties (`Dict[str, Any]`): - Decomposition properties. Expected entries: `target_names: List[str]` and a dictionary with keys - `prediction_space: str`, `sub_target_names: List[Union[str, Null]]` (must have 3 entries, null for + target_properties (`dict[str, Any]`): + Decomposition properties. Expected entries: `target_names: list[str]` and a dictionary with keys + `prediction_space: str`, `sub_target_names: list[Union[str, Null]]` (must have 3 entries, null for missing modalities), `up_to_scale: bool`, one for each target and sub-target. - color_map (`Union[str, Dict[str, str]]`, *optional*, defaults to `"Spectral"`): + color_map (`Union[str, dict[str, str]]`, *optional*, defaults to `"Spectral"`): Color map used to convert a single-channel predictions into colored representations. When a dictionary is passed, each modality can be colored with its own color map. - Returns: `List[Dict[str, PIL.Image.Image]]` with intrinsic image decomposition visualization. + Returns: `list[dict[str, PIL.Image.Image]]` with intrinsic image decomposition visualization. """ if "target_names" not in target_properties: raise ValueError("Missing `target_names` in target_properties") @@ -641,25 +629,20 @@ def visualize_targets_one(images, idx=None): @staticmethod def visualize_uncertainty( - uncertainty: Union[ - np.ndarray, - torch.Tensor, - List[np.ndarray], - List[torch.Tensor], - ], + uncertainty: np.ndarray | torch.Tensor | list[np.ndarray] | list[torch.Tensor], saturation_percentile=95, - ) -> List[PIL.Image.Image]: + ) -> list[PIL.Image.Image]: """ Visualizes dense uncertainties, such as produced by `MarigoldDepthPipeline`, `MarigoldNormalsPipeline`, or `MarigoldIntrinsicsPipeline`. Args: - uncertainty (`Union[np.ndarray, torch.Tensor, List[np.ndarray], List[torch.Tensor]]`): + uncertainty (`Union[np.ndarray, torch.Tensor, list[np.ndarray], list[torch.Tensor]]`): Uncertainty maps. saturation_percentile (`int`, *optional*, defaults to `95`): Specifies the percentile uncertainty value visualized with maximum intensity. - Returns: `List[PIL.Image.Image]` with uncertainty visualization. + Returns: `list[PIL.Image.Image]` with uncertainty visualization. """ def visualize_uncertainty_one(img, idx=None): diff --git a/src/diffusers/pipelines/marigold/pipeline_marigold_depth.py b/src/diffusers/pipelines/marigold/pipeline_marigold_depth.py index 92ec16fd455b..971ad5781d6e 100644 --- a/src/diffusers/pipelines/marigold/pipeline_marigold_depth.py +++ b/src/diffusers/pipelines/marigold/pipeline_marigold_depth.py @@ -18,7 +18,7 @@ # -------------------------------------------------------------------------- from dataclasses import dataclass from functools import partial -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -96,9 +96,9 @@ class MarigoldDepthOutput(BaseOutput): The shape is `numimages * numensemble × 4 × latentheight × latentwidth`. """ - prediction: Union[np.ndarray, torch.Tensor] - uncertainty: Union[None, np.ndarray, torch.Tensor] - latent: Union[None, torch.Tensor] + prediction: np.ndarray | torch.Tensor + uncertainty: None | np.ndarray | torch.Tensor + latent: None | torch.Tensor class MarigoldDepthPipeline(DiffusionPipeline): @@ -150,7 +150,7 @@ def __init__( self, unet: UNet2DConditionModel, vae: AutoencoderKL, - scheduler: Union[DDIMScheduler, LCMScheduler], + scheduler: DDIMScheduler | LCMScheduler, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, prediction_type: Optional[str] = None, @@ -202,9 +202,9 @@ def check_inputs( resample_method_input: str, resample_method_output: str, batch_size: int, - ensembling_kwargs: Optional[Dict[str, Any]], + ensembling_kwargs: Optional[dict[str, Any]], latents: Optional[torch.Tensor], - generator: Optional[Union[torch.Generator, List[torch.Generator]]], + generator: Optional[torch.Generator | list[torch.Generator]], output_type: str, output_uncertainty: bool, ) -> int: @@ -356,9 +356,9 @@ def __call__( resample_method_input: str = "bilinear", resample_method_output: str = "bilinear", batch_size: int = 1, - ensembling_kwargs: Optional[Dict[str, Any]] = None, - latents: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ensembling_kwargs: Optional[dict[str, Any]] = None, + latents: Optional[torch.Tensor | list[torch.Tensor]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: str = "np", output_uncertainty: bool = False, output_latent: bool = False, @@ -368,8 +368,8 @@ def __call__( Function invoked when calling the pipeline. Args: - image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`), - `List[torch.Tensor]`: An input image or images used as an input for the depth estimation task. For + image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`), + `list[torch.Tensor]`: An input image or images used as an input for the depth estimation task. For arrays and tensors, the expected value range is between `[0, 1]`. Passing a batch of images is possible by providing a four-dimensional array or a tensor. Additionally, a list of images of two- or three-dimensional arrays or tensors can be passed. In the latter case, all list elements must have the @@ -406,10 +406,10 @@ def __call__( tolerance is reached. - max_res (`int`, *optional*, defaults to `None`): Resolution at which the alignment is performed; `None` matches the `processing_resolution`. - latents (`torch.Tensor`, or `List[torch.Tensor]`, *optional*, defaults to `None`): + latents (`torch.Tensor`, or `list[torch.Tensor]`, *optional*, defaults to `None`): Latent noise tensors to replace the random initialization. These can be taken from the previous function call's output. - generator (`torch.Generator`, or `List[torch.Generator]`, *optional*, defaults to `None`): + generator (`torch.Generator`, or `list[torch.Generator]`, *optional*, defaults to `None`): Random number generator object to ensure reproducibility. output_type (`str`, *optional*, defaults to `"np"`): Preferred format of the output's `prediction` and the optional `uncertainty` fields. The accepted @@ -625,7 +625,7 @@ def prepare_latents( generator: Optional[torch.Generator], ensemble_size: int, batch_size: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: def retrieve_latents(encoder_output): if hasattr(encoder_output, "latent_dist"): return encoder_output.latent_dist.mode() @@ -680,7 +680,7 @@ def ensemble_depth( max_iter: int = 2, tol: float = 1e-3, max_res: int = 1024, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """ Ensembles the depth maps represented by the `depth` tensor with expected shape `(B, 1, H, W)`, where B is the number of ensemble members for a given prediction of size `(H x W)`. Even though the function is designed for @@ -754,7 +754,7 @@ def align(depth: torch.Tensor, param: np.ndarray) -> torch.Tensor: def ensemble( depth_aligned: torch.Tensor, return_uncertainty: bool = False - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: uncertainty = None if reduction == "mean": prediction = torch.mean(depth_aligned, dim=0, keepdim=True) diff --git a/src/diffusers/pipelines/marigold/pipeline_marigold_intrinsics.py b/src/diffusers/pipelines/marigold/pipeline_marigold_intrinsics.py index bef9ca77c708..b892564dfca9 100644 --- a/src/diffusers/pipelines/marigold/pipeline_marigold_intrinsics.py +++ b/src/diffusers/pipelines/marigold/pipeline_marigold_intrinsics.py @@ -17,7 +17,7 @@ # Marigold project website: https://marigoldcomputervision.github.io # -------------------------------------------------------------------------- from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -112,9 +112,9 @@ class MarigoldIntrinsicsOutput(BaseOutput): The shape is `(numimages * numensemble) × (numtargets * 4) × latentheight × latentwidth`. """ - prediction: Union[np.ndarray, torch.Tensor] - uncertainty: Union[None, np.ndarray, torch.Tensor] - latent: Union[None, torch.Tensor] + prediction: np.ndarray | torch.Tensor + uncertainty: None | np.ndarray | torch.Tensor + latent: None | torch.Tensor class MarigoldIntrinsicsPipeline(DiffusionPipeline): @@ -139,8 +139,8 @@ class MarigoldIntrinsicsPipeline(DiffusionPipeline): CLIP tokenizer. prediction_type (`str`, *optional*): Type of predictions made by the model. - target_properties (`Dict[str, Any]`, *optional*): - Properties of the predicted modalities, such as `target_names`, a `List[str]` used to define the number, + target_properties (`dict[str, Any]`, *optional*): + Properties of the predicted modalities, such as `target_names`, a `list[str]` used to define the number, order and names of the predicted modalities, and any other metadata that may be required to interpret the predictions. default_denoising_steps (`int`, *optional*): @@ -163,11 +163,11 @@ def __init__( self, unet: UNet2DConditionModel, vae: AutoencoderKL, - scheduler: Union[DDIMScheduler, LCMScheduler], + scheduler: DDIMScheduler | LCMScheduler, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, prediction_type: Optional[str] = None, - target_properties: Optional[Dict[str, Any]] = None, + target_properties: Optional[dict[str, Any]] = None, default_denoising_steps: Optional[int] = None, default_processing_resolution: Optional[int] = None, ): @@ -216,9 +216,9 @@ def check_inputs( resample_method_input: str, resample_method_output: str, batch_size: int, - ensembling_kwargs: Optional[Dict[str, Any]], + ensembling_kwargs: Optional[dict[str, Any]], latents: Optional[torch.Tensor], - generator: Optional[Union[torch.Generator, List[torch.Generator]]], + generator: Optional[torch.Generator | list[torch.Generator]], output_type: str, output_uncertainty: bool, ) -> int: @@ -368,9 +368,9 @@ def __call__( resample_method_input: str = "bilinear", resample_method_output: str = "bilinear", batch_size: int = 1, - ensembling_kwargs: Optional[Dict[str, Any]] = None, - latents: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ensembling_kwargs: Optional[dict[str, Any]] = None, + latents: Optional[torch.Tensor | list[torch.Tensor]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: str = "np", output_uncertainty: bool = False, output_latent: bool = False, @@ -380,8 +380,8 @@ def __call__( Function invoked when calling the pipeline. Args: - image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`), - `List[torch.Tensor]`: An input image or images used as an input for the intrinsic decomposition task. + image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`), + `list[torch.Tensor]`: An input image or images used as an input for the intrinsic decomposition task. For arrays and tensors, the expected value range is between `[0, 1]`. Passing a batch of images is possible by providing a four-dimensional array or a tensor. Additionally, a list of images of two- or three-dimensional arrays or tensors can be passed. In the latter case, all list elements must have the @@ -413,7 +413,7 @@ def __call__( latents (`torch.Tensor`, *optional*, defaults to `None`): Latent noise tensors to replace the random initialization. These can be taken from the previous function call's output. - generator (`torch.Generator`, or `List[torch.Generator]`, *optional*, defaults to `None`): + generator (`torch.Generator`, or `list[torch.Generator]`, *optional*, defaults to `None`): Random number generator object to ensure reproducibility. output_type (`str`, *optional*, defaults to `"np"`): Preferred format of the output's `prediction` and the optional `uncertainty` fields. The accepted @@ -631,7 +631,7 @@ def prepare_latents( generator: Optional[torch.Generator], ensemble_size: int, batch_size: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: def retrieve_latents(encoder_output): if hasattr(encoder_output, "latent_dist"): return encoder_output.latent_dist.mode() @@ -680,7 +680,7 @@ def ensemble_intrinsics( targets: torch.Tensor, output_uncertainty: bool = False, reduction: str = "median", - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """ Ensembles the intrinsic decomposition represented by the `targets` tensor with expected shape `(B, T, 3, H, W)`, where B is the number of ensemble members for a given prediction of size `(H x W)`, and T is the number of diff --git a/src/diffusers/pipelines/marigold/pipeline_marigold_normals.py b/src/diffusers/pipelines/marigold/pipeline_marigold_normals.py index 485a39c995ec..c95ff10d1e7f 100644 --- a/src/diffusers/pipelines/marigold/pipeline_marigold_normals.py +++ b/src/diffusers/pipelines/marigold/pipeline_marigold_normals.py @@ -17,7 +17,7 @@ # Marigold project website: https://marigoldcomputervision.github.io # -------------------------------------------------------------------------- from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional import numpy as np import torch @@ -91,9 +91,9 @@ class MarigoldNormalsOutput(BaseOutput): The shape is `numimages * numensemble × 4 × latentheight × latentwidth`. """ - prediction: Union[np.ndarray, torch.Tensor] - uncertainty: Union[None, np.ndarray, torch.Tensor] - latent: Union[None, torch.Tensor] + prediction: np.ndarray | torch.Tensor + uncertainty: None | np.ndarray | torch.Tensor + latent: None | torch.Tensor class MarigoldNormalsPipeline(DiffusionPipeline): @@ -140,7 +140,7 @@ def __init__( self, unet: UNet2DConditionModel, vae: AutoencoderKL, - scheduler: Union[DDIMScheduler, LCMScheduler], + scheduler: DDIMScheduler | LCMScheduler, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, prediction_type: Optional[str] = None, @@ -189,9 +189,9 @@ def check_inputs( resample_method_input: str, resample_method_output: str, batch_size: int, - ensembling_kwargs: Optional[Dict[str, Any]], + ensembling_kwargs: Optional[dict[str, Any]], latents: Optional[torch.Tensor], - generator: Optional[Union[torch.Generator, List[torch.Generator]]], + generator: Optional[torch.Generator | list[torch.Generator]], output_type: str, output_uncertainty: bool, ) -> int: @@ -341,9 +341,9 @@ def __call__( resample_method_input: str = "bilinear", resample_method_output: str = "bilinear", batch_size: int = 1, - ensembling_kwargs: Optional[Dict[str, Any]] = None, - latents: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ensembling_kwargs: Optional[dict[str, Any]] = None, + latents: Optional[torch.Tensor | list[torch.Tensor]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: str = "np", output_uncertainty: bool = False, output_latent: bool = False, @@ -353,8 +353,8 @@ def __call__( Function invoked when calling the pipeline. Args: - image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`), - `List[torch.Tensor]`: An input image or images used as an input for the normals estimation task. For + image (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`), + `list[torch.Tensor]`: An input image or images used as an input for the normals estimation task. For arrays and tensors, the expected value range is between `[0, 1]`. Passing a batch of images is possible by providing a four-dimensional array or a tensor. Additionally, a list of images of two- or three-dimensional arrays or tensors can be passed. In the latter case, all list elements must have the @@ -386,7 +386,7 @@ def __call__( latents (`torch.Tensor`, *optional*, defaults to `None`): Latent noise tensors to replace the random initialization. These can be taken from the previous function call's output. - generator (`torch.Generator`, or `List[torch.Generator]`, *optional*, defaults to `None`): + generator (`torch.Generator`, or `list[torch.Generator]`, *optional*, defaults to `None`): Random number generator object to ensure reproducibility. output_type (`str`, *optional*, defaults to `"np"`): Preferred format of the output's `prediction` and the optional `uncertainty` fields. The accepted @@ -599,7 +599,7 @@ def prepare_latents( generator: Optional[torch.Generator], ensemble_size: int, batch_size: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: def retrieve_latents(encoder_output): if hasattr(encoder_output, "latent_dist"): return encoder_output.latent_dist.mode() @@ -660,7 +660,7 @@ def normalize_normals(normals: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: @staticmethod def ensemble_normals( normals: torch.Tensor, output_uncertainty: bool, reduction: str = "closest" - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """ Ensembles the normals maps represented by the `normals` tensor with expected shape `(B, 3, H, W)`, where B is the number of ensemble members for a given prediction of size `(H x W)`. diff --git a/src/diffusers/pipelines/mochi/pipeline_mochi.py b/src/diffusers/pipelines/mochi/pipeline_mochi.py index 5874a92c6f2f..87d097c8d86f 100644 --- a/src/diffusers/pipelines/mochi/pipeline_mochi.py +++ b/src/diffusers/pipelines/mochi/pipeline_mochi.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -79,9 +79,9 @@ def linear_quadratic_schedule(num_steps, threshold_noise, linear_steps=None): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -96,15 +96,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -196,7 +196,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -253,8 +253,8 @@ def _get_t5_prompt_embeds( # Adapted from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -269,9 +269,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -498,16 +498,16 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + negative_prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: int = 19, num_inference_steps: int = 64, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 4.5, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -515,16 +515,16 @@ def __call__( negative_prompt_attention_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to `self.default_height`): @@ -536,7 +536,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -548,7 +548,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -579,7 +579,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/mochi/pipeline_output.py b/src/diffusers/pipelines/mochi/pipeline_output.py index d15827bc0084..5068cf930aaa 100644 --- a/src/diffusers/pipelines/mochi/pipeline_output.py +++ b/src/diffusers/pipelines/mochi/pipeline_output.py @@ -11,8 +11,8 @@ class MochiPipelineOutput(BaseOutput): Output class for Mochi pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/musicldm/pipeline_musicldm.py b/src/diffusers/pipelines/musicldm/pipeline_musicldm.py index c909e5eb0d26..26837e952d5e 100644 --- a/src/diffusers/pipelines/musicldm/pipeline_musicldm.py +++ b/src/diffusers/pipelines/musicldm/pipeline_musicldm.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -106,8 +106,8 @@ class MusicLDMPipeline(DeprecatedPipelineMixin, DiffusionPipeline, StableDiffusi def __init__( self, vae: AutoencoderKL, - text_encoder: Union[ClapTextModelWithProjection, ClapModel], - tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], + text_encoder: ClapTextModelWithProjection | ClapModel, + tokenizer: RobertaTokenizer | RobertaTokenizerFast, feature_extractor: Optional[ClapFeatureExtractor], unet: UNet2DConditionModel, scheduler: KarrasDiffusionSchedulers, @@ -140,7 +140,7 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device (`torch.device`): torch device @@ -148,7 +148,7 @@ def _encode_prompt( number of waveforms that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the audio generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -207,7 +207,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -434,28 +434,28 @@ def enable_model_cpu_offload(self, gpu_id=0): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, audio_length_in_s: Optional[float] = None, num_inference_steps: int = 200, guidance_scale: float = 2.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_waveforms_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, output_type: Optional[str] = "np", ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. audio_length_in_s (`int`, *optional*, defaults to 10.24): The length of the generated audio sample in seconds. @@ -465,7 +465,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 2.0): A higher guidance scale value encourages the model to generate audio that is closely linked to the text `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in audio generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_waveforms_per_prompt (`int`, *optional*, defaults to 1): @@ -477,7 +477,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/omnigen/pipeline_omnigen.py b/src/diffusers/pipelines/omnigen/pipeline_omnigen.py index 090cb46aace4..67ea1cc4fe13 100644 --- a/src/diffusers/pipelines/omnigen/pipeline_omnigen.py +++ b/src/diffusers/pipelines/omnigen/pipeline_omnigen.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -60,9 +60,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -77,15 +77,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -170,7 +170,7 @@ def __init__( def encode_input_images( self, - input_pixel_values: List[torch.Tensor], + input_pixel_values: list[torch.Tensor], device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -331,32 +331,32 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - input_images: Union[PipelineImageInput, List[PipelineImageInput]] = None, + prompt: str | list[str], + input_images: PipelineImageInput | list[PipelineImageInput] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, max_input_image_size: int = 1024, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 2.5, img_guidance_scale: float = 1.6, use_input_image_size_as_output: bool = False, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If the input includes images, need to add placeholders `<|image_i|>` in the prompt to indicate the position of the i-th images. - input_images (`PipelineImageInput` or `List[PipelineImageInput]`, *optional*): + input_images (`PipelineImageInput` or `list[PipelineImageInput]`, *optional*): The list of input images. We will replace the "<|image_i|>" in prompt with the i-th image in list. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): The height in pixels of the generated image. This is set to 1024 by default for the best results. @@ -367,7 +367,7 @@ def __call__( expense of slower inference. max_input_image_size (`int`, *optional*, defaults to 1024): the maximum size of input image, which will be used to crop the input image to the maximum size - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -384,7 +384,7 @@ def __call__( e.g., image editing task num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -401,7 +401,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/omnigen/processor_omnigen.py b/src/diffusers/pipelines/omnigen/processor_omnigen.py index 7ed11871bb2a..b58d79dcff45 100644 --- a/src/diffusers/pipelines/omnigen/processor_omnigen.py +++ b/src/diffusers/pipelines/omnigen/processor_omnigen.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Dict, List +from typing import Dict import numpy as np import torch @@ -132,8 +132,8 @@ def add_prefix_instruction(self, prompt): def __call__( self, - instructions: List[str], - input_images: List[List[str]] = None, + instructions: list[str], + input_images: list[list[str]] = None, height: int = 1024, width: int = 1024, negative_prompt: str = "low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers.", diff --git a/src/diffusers/pipelines/onnx_utils.py b/src/diffusers/pipelines/onnx_utils.py index 74e9f0b97800..8143a16f35c7 100644 --- a/src/diffusers/pipelines/onnx_utils.py +++ b/src/diffusers/pipelines/onnx_utils.py @@ -18,7 +18,7 @@ import os import shutil from pathlib import Path -from typing import Optional, Union +from typing import Optional import numpy as np from huggingface_hub import hf_hub_download @@ -61,7 +61,7 @@ def __call__(self, **kwargs): return self.model.run(None, inputs) @staticmethod - def load_model(path: Union[str, Path], provider=None, sess_options=None, provider_options=None): + def load_model(path: str | Path, provider=None, sess_options=None, provider_options=None): """ Loads an ONNX Inference session with an ExecutionProvider. Default provider is `CPUExecutionProvider` @@ -84,7 +84,7 @@ def load_model(path: Union[str, Path], provider=None, sess_options=None, provide path, providers=[provider], sess_options=sess_options, provider_options=provider_options ) - def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): + def _save_pretrained(self, save_directory: str | Path, file_name: Optional[str] = None, **kwargs): """ Save a model and its configuration file to a directory, so that it can be re-loaded using the [`~optimum.onnxruntime.modeling_ort.ORTModel.from_pretrained`] class method. It will always save the @@ -117,7 +117,7 @@ def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, **kwargs, ): """ @@ -141,9 +141,9 @@ def save_pretrained( @validate_hf_hub_args def _from_pretrained( cls, - model_id: Union[str, Path], - token: Optional[Union[bool, str, None]] = None, - revision: Optional[Union[str, None]] = None, + model_id: str | Path, + token: Optional[bool | str | None] = None, + revision: Optional[str | None] = None, force_download: bool = False, cache_dir: Optional[str] = None, file_name: Optional[str] = None, @@ -210,7 +210,7 @@ def _from_pretrained( @validate_hf_hub_args def from_pretrained( cls, - model_id: Union[str, Path], + model_id: str | Path, force_download: bool = True, token: Optional[str] = None, cache_dir: Optional[str] = None, diff --git a/src/diffusers/pipelines/pag/pag_utils.py b/src/diffusers/pipelines/pag/pag_utils.py index 8a56961f321c..d4404a547569 100644 --- a/src/diffusers/pipelines/pag/pag_utils.py +++ b/src/diffusers/pipelines/pag/pag_utils.py @@ -13,7 +13,6 @@ # limitations under the License. import re -from typing import Dict, List, Tuple, Union import torch import torch.nn as nn @@ -112,7 +111,7 @@ def _apply_perturbed_attention_guidance( return_pred_text (bool): Whether to return the text noise prediction. Returns: - Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: The updated noise prediction tensor after applying + Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: The updated noise prediction tensor after applying perturbed attention guidance and the text noise prediction. """ pag_scale = self._get_pag_scale(t) @@ -151,8 +150,8 @@ def _prepare_perturbed_attention_guidance(self, cond, uncond, do_classifier_free def set_pag_applied_layers( self, - pag_applied_layers: Union[str, List[str]], - pag_attn_processors: Tuple[AttentionProcessor, AttentionProcessor] = ( + pag_applied_layers: str | list[str], + pag_attn_processors: tuple[AttentionProcessor, AttentionProcessor] = ( PAGCFGIdentitySelfAttnProcessor2_0(), PAGIdentitySelfAttnProcessor2_0(), ), @@ -161,7 +160,7 @@ def set_pag_applied_layers( Set the self-attention layers to apply PAG. Raise ValueError if the input is invalid. Args: - pag_applied_layers (`str` or `List[str]`): + pag_applied_layers (`str` or `list[str]`): One or more strings identifying the layer names, or a simple regex for matching multiple layers, where PAG is to be applied. A few ways of expected usage are as follows: - Single layers specified as - "blocks.{layer_index}" @@ -169,7 +168,7 @@ def set_pag_applied_layers( - Multiple layers as a block name - "mid" - Multiple layers as regex - "blocks.({layer_index_1}|{layer_index_2})" pag_attn_processors: - (`Tuple[AttentionProcessor, AttentionProcessor]`, defaults to `(PAGCFGIdentitySelfAttnProcessor2_0(), + (`tuple[AttentionProcessor, AttentionProcessor]`, defaults to `(PAGCFGIdentitySelfAttnProcessor2_0(), PAGIdentitySelfAttnProcessor2_0())`): A tuple of two attention processors. The first attention processor is for PAG with Classifier-free guidance enabled (conditional and unconditional). The second attention processor is for PAG with CFG disabled (unconditional only). @@ -214,7 +213,7 @@ def do_perturbed_attention_guidance(self) -> bool: return self._pag_scale > 0 and len(self.pag_applied_layers) > 0 @property - def pag_attn_processors(self) -> Dict[str, AttentionProcessor]: + def pag_attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of PAG attention processors: A dictionary contains all PAG attention processors used in the model diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py index 1abef014301a..8469f2a57e2e 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -109,9 +109,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -126,15 +126,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -196,7 +196,7 @@ class StableDiffusionControlNetPAGPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -222,13 +222,13 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, image_encoder: CLIPVisionModelWithProjection = None, requires_safety_checker: bool = True, - pag_applied_layers: Union[str, List[str]] = "mid", + pag_applied_layers: str | list[str] = "mid", ): super().__init__() @@ -288,7 +288,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -296,7 +296,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -395,7 +395,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -864,35 +864,35 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -900,10 +900,10 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -919,18 +919,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -938,7 +938,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -952,7 +952,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -965,16 +965,16 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -984,7 +984,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py index 2781af789018..b3fbc9570773 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py @@ -15,7 +15,7 @@ # This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -170,7 +170,7 @@ class StableDiffusionControlNetPAGInpaintPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -196,13 +196,13 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, image_encoder: CLIPVisionModelWithProjection = None, requires_safety_checker: bool = True, - pag_applied_layers: Union[str, List[str]] = "mid", + pag_applied_layers: str | list[str] = "mid", ): super().__init__() @@ -264,7 +264,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -272,7 +272,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -371,7 +371,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -973,7 +973,7 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, control_image: PipelineImageInput = None, @@ -983,26 +983,26 @@ def __call__( strength: float = 1.0, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.5, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.5, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -1010,25 +1010,25 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, - `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, + `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, NumPy array or tensor representing an image batch to be used as the starting point. For both NumPy array and PyTorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a NumPy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, - `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, + `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, NumPy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a NumPy array or PyTorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for PyTorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for NumPy array, it would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - control_image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, - `List[List[torch.Tensor]]`, or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, + `list[list[torch.Tensor]]`, or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1058,7 +1058,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1066,7 +1066,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1080,7 +1080,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1093,13 +1093,13 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 0.5): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that @@ -1109,7 +1109,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py index 24cbab43c966..37af4f5f7d79 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -129,9 +129,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -146,15 +146,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -221,7 +221,7 @@ class StableDiffusionXLControlNetPAGPipeline( A `CLIPTokenizer` to tokenize text. unet ([`UNet2DConditionModel`]): A `UNet2DConditionModel` to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -265,13 +265,13 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, - pag_applied_layers: Union[str, List[str]] = "mid", # ["down.block_2", "up.block_1.attentions_0"], "mid" + pag_applied_layers: str | list[str] = "mid", # ["down.block_2", "up.block_1.attentions_0"], "mid" ): super().__init__() @@ -326,9 +326,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -337,11 +337,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -459,7 +459,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1014,45 +1014,45 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -1060,13 +1060,13 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or @@ -1084,11 +1084,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1102,10 +1102,10 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1113,7 +1113,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1134,7 +1134,7 @@ def __call__( weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1147,39 +1147,39 @@ def __call__( cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1192,7 +1192,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py index a6df1b22c8b9..90882fbbc607 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py @@ -14,7 +14,7 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -206,7 +206,7 @@ class StableDiffusionXLControlNetPAGImg2ImgPipeline( Second Tokenizer of class [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. - controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + controlnet ([`ControlNetModel`] or `list[ControlNetModel]`): Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. @@ -254,14 +254,14 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + controlnet: ControlNetModel | list[ControlNetModel] | tuple[ControlNetModel] | MultiControlNetModel, scheduler: KarrasDiffusionSchedulers, requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, - pag_applied_layers: Union[str, List[str]] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] + pag_applied_layers: str | list[str] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] ): super().__init__() @@ -318,9 +318,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -329,11 +329,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -451,7 +451,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1092,8 +1092,8 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, control_image: PipelineImageInput = None, height: Optional[int] = None, @@ -1101,38 +1101,38 @@ def __call__( strength: float = 0.8, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + controlnet_conditioning_scale: float | list[float] = 0.8, guess_mode: bool = False, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -1140,18 +1140,18 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The initial image will be used as the starting point for the image generation process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded again. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height @@ -1181,11 +1181,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1193,7 +1193,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1215,7 +1215,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1230,42 +1230,42 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original unet. If multiple ControlNets are specified in init, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): In this mode, the ControlNet encoder will try best to recognize the content of the input image even if you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. - control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + control_guidance_start (`float` or `list[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the controlnet starts applying. - control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + control_guidance_end (`float` or `list[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the controlnet stops applying. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1286,7 +1286,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py b/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py index d156eac8f3f7..8c06d44dbedf 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -209,7 +209,7 @@ def __init__( requires_safety_checker: bool = True, text_encoder_2: Optional[T5EncoderModel] = None, tokenizer_2: Optional[MT5Tokenizer] = None, - pag_applied_layers: Union[str, List[str]] = "blocks.1", # "blocks.16.attn1", "blocks.16", "16", 16 + pag_applied_layers: str | list[str] = "blocks.1", # "blocks.16.attn1", "blocks.16", "16", 16 ): super().__init__() @@ -274,7 +274,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -284,7 +284,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -373,7 +373,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -580,15 +580,15 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_2: Optional[torch.Tensor] = None, @@ -601,13 +601,13 @@ def __call__( output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = (1024, 1024), - target_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), + original_size: Optional[tuple[int, int]] = (1024, 1024), + target_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), use_resolution_binning: bool = True, pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, @@ -616,7 +616,7 @@ def __call__( The call function to the pipeline for generation with HunyuanDiT. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`): The height in pixels of the generated image. @@ -628,7 +628,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -636,7 +636,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -666,17 +666,17 @@ def __call__( plain tuple. callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A callback function or a list of callback functions to be called at the end of each denoising step. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): A list of tensor inputs that should be passed to the callback function. If not defined, all tensor inputs will be passed. guidance_rescale (`float`, *optional*, defaults to 0.0): Rescale the noise_cfg according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). See Section 3.4 - original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): + original_size (`tuple[int, int]`, *optional*, defaults to `(1024, 1024)`): The original size of the image. Used to calculate the time ids. - target_size (`Tuple[int, int]`, *optional*): + target_size (`tuple[int, int]`, *optional*): The target size of the image. Used to calculate the time ids. - crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`): + crops_coords_top_left (`tuple[int, int]`, *optional*, defaults to `(0, 0)`): The top left coordinates of the crop. Used to calculate the time ids. use_resolution_binning (`bool`, *optional*, defaults to `True`): Whether to use resolution binning or not. If `True`, the input resolution will be mapped to the closest diff --git a/src/diffusers/pipelines/pag/pipeline_pag_kolors.py b/src/diffusers/pipelines/pag/pipeline_pag_kolors.py index 1368358db6ba..6304c0803d6f 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_kolors.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_kolors.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection @@ -70,9 +70,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -87,15 +87,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -155,7 +155,7 @@ class KolorsPAGPipeline( force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"False"`): Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of `Kwai-Kolors/Kolors-diffusers`. - pag_applied_layers (`str` or `List[str]``, *optional*, defaults to `"mid"`): + pag_applied_layers (`str` or `list[str]``, *optional*, defaults to `"mid"`): Set the transformer attention layers where to apply the perturbed attention guidance. Can be a string or a list of strings with "down", "mid", "up", a whole transformer block or specific transformer block attention layers, e.g.: @@ -188,7 +188,7 @@ def __init__( image_encoder: CLIPVisionModelWithProjection = None, feature_extractor: CLIPImageProcessor = None, force_zeros_for_empty_prompt: bool = False, - pag_applied_layers: Union[str, List[str]] = "mid", + pag_applied_layers: str | list[str] = "mid", ): super().__init__() @@ -231,7 +231,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -239,7 +239,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -309,7 +309,7 @@ def encode_prompt( if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: negative_prompt_embeds = torch.zeros_like(prompt_embeds) elif do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -677,38 +677,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, max_sequence_length: int = 256, @@ -717,7 +717,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -733,11 +733,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -754,7 +754,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -763,7 +763,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -785,7 +785,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -799,31 +799,31 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -833,7 +833,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py b/src/diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py index 9031877b5b8d..2537f8470beb 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py @@ -16,7 +16,7 @@ import inspect import re import urllib.parse as ul -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -85,9 +85,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -102,15 +102,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -173,7 +173,7 @@ def __init__( vae: AutoencoderKL, transformer: PixArtTransformer2DModel, scheduler: KarrasDiffusionSchedulers, - pag_applied_layers: Union[str, List[str]] = "blocks.1", # 1st transformer block + pag_applied_layers: str | list[str] = "blocks.1", # 1st transformer block ): super().__init__() @@ -189,7 +189,7 @@ def __init__( # Copied from diffusers.pipelines.pixart_alpha.pipeline_pixart_alpha.PixArtAlphaPipeline.encode_prompt with 120->300 def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -206,9 +206,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -575,17 +575,17 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = None, width: Optional[int] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -600,26 +600,26 @@ def __call__( max_sequence_length: int = 300, pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -638,7 +638,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sana.py b/src/diffusers/pipelines/pag/pipeline_pag_sana.py index 9e91ccbe8006..062c17e4652b 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sana.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sana.py @@ -17,7 +17,7 @@ import re import urllib.parse as ul import warnings -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, Optional import torch from transformers import Gemma2PreTrainedModel, GemmaTokenizer, GemmaTokenizerFast @@ -89,9 +89,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -106,15 +106,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -161,12 +161,12 @@ class SanaPAGPipeline(DiffusionPipeline, PAGMixin): def __init__( self, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, text_encoder: Gemma2PreTrainedModel, vae: AutoencoderDC, transformer: SanaTransformer2DModel, scheduler: FlowMatchEulerDiscreteScheduler, - pag_applied_layers: Union[str, List[str]] = "transformer_blocks.0", + pag_applied_layers: str | list[str] = "transformer_blocks.0", ): super().__init__() @@ -241,7 +241,7 @@ def disable_vae_tiling(self): def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -252,15 +252,15 @@ def encode_prompt( negative_prompt_attention_mask: Optional[torch.Tensor] = None, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -649,17 +649,17 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, height: int = 1024, width: int = 1024, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -670,9 +670,9 @@ def __call__( clean_caption: bool = False, use_resolution_binning: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 300, - complex_human_instruction: List[str] = [ + complex_human_instruction: list[str] = [ "Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:", "- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.", "- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.", @@ -684,26 +684,26 @@ def __call__( ], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 20): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -722,7 +722,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -756,12 +756,12 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to 300): Maximum sequence length to use with the `prompt`. - complex_human_instruction (`List[str]`, *optional*): + complex_human_instruction (`list[str]`, *optional*): Instructions for complex human attention: https://github.com/NVlabs/Sana/blob/main/configs/sana_app_config/Sana_1600M_app.yaml#L55. pag_scale (`float`, *optional*, defaults to 3.0): diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd.py b/src/diffusers/pipelines/pag/pipeline_pag_sd.py index ea64f8be2c50..c605eec0aeb1 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from packaging import version @@ -98,9 +98,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -115,15 +115,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -212,7 +212,7 @@ def __init__( feature_extractor: CLIPImageProcessor, image_encoder: CLIPVisionModelWithProjection = None, requires_safety_checker: bool = True, - pag_applied_layers: Union[str, List[str]] = "mid", + pag_applied_layers: str | list[str] = "mid", ): super().__init__() @@ -317,7 +317,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -325,7 +325,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -424,7 +424,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -746,29 +746,29 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -776,7 +776,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -785,18 +785,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -804,7 +804,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -818,7 +818,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -843,7 +843,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py index acb4e52340a6..9b0176fcf8d7 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -77,9 +77,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -94,15 +94,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -185,7 +185,7 @@ def __init__( tokenizer_2: CLIPTokenizer, text_encoder_3: T5EncoderModel, tokenizer_3: T5TokenizerFast, - pag_applied_layers: Union[str, List[str]] = "blocks.1", # 1st transformer block + pag_applied_layers: str | list[str] = "blocks.1", # 1st transformer block ): super().__init__() @@ -221,7 +221,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -278,7 +278,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -334,15 +334,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -354,12 +354,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -368,14 +368,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -685,19 +685,19 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -705,10 +705,10 @@ def __call__( negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, @@ -717,13 +717,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -733,7 +733,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -743,19 +743,19 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -791,7 +791,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py index e1819a79fb30..11143ae0dfd6 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -93,9 +93,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -110,15 +110,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -201,7 +201,7 @@ def __init__( tokenizer_2: CLIPTokenizer, text_encoder_3: T5EncoderModel, tokenizer_3: T5TokenizerFast, - pag_applied_layers: Union[str, List[str]] = "blocks.1", # 1st transformer block + pag_applied_layers: str | list[str] = "blocks.1", # 1st transformer block ): super().__init__() @@ -237,7 +237,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -294,7 +294,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -350,15 +350,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -370,12 +370,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -384,14 +384,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -736,21 +736,21 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, image: PipelineImageInput = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -758,10 +758,10 @@ def __call__( negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, @@ -770,16 +770,16 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -794,7 +794,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -804,19 +804,19 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -852,7 +852,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py index de13be9c4d22..881ed8c60d05 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection @@ -135,12 +135,12 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], + unet: UNet2DConditionModel | UNetMotionModel, motion_adapter: MotionAdapter, scheduler: KarrasDiffusionSchedulers, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, - pag_applied_layers: Union[str, List[str]] = "mid_block.*attn1", # ["mid"], ["down_blocks.1"] + pag_applied_layers: str | list[str] = "mid_block.*attn1", # ["mid"], ["down_blocks.1"] ): super().__init__() if isinstance(unet, UNet2DConditionModel): @@ -178,7 +178,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -186,7 +186,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -285,7 +285,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -576,27 +576,27 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, num_frames: Optional[int] = 16, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], decode_chunk_size: int = 16, pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, @@ -605,7 +605,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -620,13 +620,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -642,7 +642,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -663,7 +663,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py index 8351112ce409..515d27514b56 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -93,9 +93,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -110,15 +110,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -207,7 +207,7 @@ def __init__( feature_extractor: CLIPImageProcessor, image_encoder: CLIPVisionModelWithProjection = None, requires_safety_checker: bool = True, - pag_applied_layers: Union[str, List[str]] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] + pag_applied_layers: str | list[str] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] ): super().__init__() @@ -312,7 +312,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -320,7 +320,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -419,7 +419,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -783,29 +783,29 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -813,9 +813,9 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -830,18 +830,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. This parameter is modulated by `strength`. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -849,7 +849,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -859,7 +859,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -880,7 +880,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py index 6b1b294e10f5..30a75cabef2b 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -125,9 +125,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -142,15 +142,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -239,7 +239,7 @@ def __init__( feature_extractor: CLIPImageProcessor, image_encoder: CLIPVisionModelWithProjection = None, requires_safety_checker: bool = True, - pag_applied_layers: Union[str, List[str]] = "mid", + pag_applied_layers: str | list[str] = "mid", ): super().__init__() @@ -347,7 +347,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -355,7 +355,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -454,7 +454,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -911,7 +911,7 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: torch.Tensor = None, @@ -920,25 +920,25 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.9999, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -946,7 +946,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -955,18 +955,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -974,7 +974,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -988,7 +988,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1013,7 +1013,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py index 6b62ddcc7ca5..fd71487550a4 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -117,9 +117,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -134,15 +134,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -259,7 +259,7 @@ def __init__( feature_extractor: CLIPImageProcessor = None, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, - pag_applied_layers: Union[str, List[str]] = "mid", # ["mid"],["down.block_1"],["up.block_0.attentions_0"] + pag_applied_layers: str | list[str] = "mid", # ["mid"],["down.block_1"],["up.block_0.attentions_0"] ): super().__init__() @@ -314,9 +314,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -325,11 +325,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -447,7 +447,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -849,40 +849,40 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -890,10 +890,10 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -909,11 +909,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -930,11 +930,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -942,7 +942,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -964,7 +964,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -985,31 +985,31 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1019,7 +1019,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py index b6422b23648c..426848fe9525 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -135,9 +135,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -152,15 +152,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -281,7 +281,7 @@ def __init__( requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, - pag_applied_layers: Union[str, List[str]] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] + pag_applied_layers: str | list[str] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] ): super().__init__() @@ -331,9 +331,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -342,11 +342,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -464,7 +464,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1000,45 +1000,45 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, strength: float = 0.3, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -1046,13 +1046,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`): The image(s) to modify with the pipeline. strength (`float`, *optional*, defaults to 0.3): Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` @@ -1064,11 +1064,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1093,11 +1093,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1105,7 +1105,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1127,7 +1127,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1148,31 +1148,31 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1193,7 +1193,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py index 2a8f7a448d5e..19b455e567cb 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -148,9 +148,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -165,15 +165,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -297,7 +297,7 @@ def __init__( requires_aesthetics_score: bool = False, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, - pag_applied_layers: Union[str, List[str]] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] + pag_applied_layers: str | list[str] = "mid", # ["mid"], ["down.block_1", "up.block_0.attentions_0"] ): super().__init__() @@ -421,9 +421,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -432,11 +432,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -554,7 +554,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1091,8 +1091,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: torch.Tensor = None, @@ -1101,40 +1101,40 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.9999, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], pag_scale: float = 3.0, pag_adaptive_scale: float = 0.0, ): @@ -1142,10 +1142,10 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders image (`PIL.Image.Image`): @@ -1184,11 +1184,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1213,11 +1213,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -1235,7 +1235,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1262,31 +1262,31 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1307,7 +1307,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/paint_by_example/__init__.py b/src/diffusers/pipelines/paint_by_example/__init__.py index aaa775f690c3..e3121cbdbb66 100644 --- a/src/diffusers/pipelines/paint_by_example/__init__.py +++ b/src/diffusers/pipelines/paint_by_example/__init__.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np import PIL diff --git a/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py b/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py index c09992befbcb..f3c97395486f 100644 --- a/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +++ b/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -198,7 +198,7 @@ def __init__( vae: AutoencoderKL, image_encoder: PaintByExampleImageEncoder, unet: UNet2DConditionModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = False, @@ -270,7 +270,7 @@ def check_inputs(self, image, height, width, callback_steps): and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -397,17 +397,17 @@ def _encode_image(self, image, device, num_images_per_prompt, do_classifier_free @torch.no_grad() def __call__( self, - example_image: Union[torch.Tensor, PIL.Image.Image], - image: Union[torch.Tensor, PIL.Image.Image], - mask_image: Union[torch.Tensor, PIL.Image.Image], + example_image: torch.Tensor | PIL.Image.Image, + image: torch.Tensor | PIL.Image.Image, + mask_image: torch.Tensor | PIL.Image.Image, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -418,12 +418,12 @@ def __call__( The call function to the pipeline for generation. Args: - example_image (`torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + example_image (`torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]`): An example image to guide image generation. - image (`torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + image (`torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]`): `Image` or tensor representing an image batch to be inpainted (parts of the image are masked out with `mask_image` and repainted according to `prompt`). - mask_image (`torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + mask_image (`torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]`): `Image` or tensor representing an image batch to mask `image`. White pixels in the mask are repainted, while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the @@ -438,7 +438,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -446,7 +446,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/pia/pipeline_pia.py b/src/diffusers/pipelines/pia/pipeline_pia.py index dfc6e83fbd7c..d99e73eadbec 100644 --- a/src/diffusers/pipelines/pia/pipeline_pia.py +++ b/src/diffusers/pipelines/pia/pipeline_pia.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL @@ -85,7 +85,7 @@ ``` """ -RANGE_LIST = [ +RANGE_list = [ [1.0, 0.9, 0.85, 0.85, 0.85, 0.8], # 0 Small Motion [1.0, 0.8, 0.8, 0.8, 0.79, 0.78, 0.75], # Moderate Motion [1.0, 0.8, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.6, 0.5, 0.5], # Large Motion @@ -103,7 +103,7 @@ def prepare_mask_coef_by_statistics(num_frames: int, cond_frame: int, motion_sca assert num_frames > cond_frame, "video_length should be greater than cond_frame" - range_list = RANGE_LIST + range_list = RANGE_list assert motion_scale < len(range_list), f"motion_scale type{motion_scale} not implemented" @@ -122,13 +122,13 @@ class PIAPipelineOutput(BaseOutput): Output class for PIAPipeline. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): Nested list of length `batch_size` with denoised PIL image sequences of length `num_frames`, NumPy array of shape `(batch_size, num_frames, channels, height, width, Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] class PIAPipeline( @@ -179,15 +179,13 @@ def __init__( vae: AutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, - unet: Union[UNet2DConditionModel, UNetMotionModel], - scheduler: Union[ - DDIMScheduler, - PNDMScheduler, - LMSDiscreteScheduler, - EulerDiscreteScheduler, - EulerAncestralDiscreteScheduler, - DPMSolverMultistepScheduler, - ], + unet: UNet2DConditionModel | UNetMotionModel, + scheduler: DDIMScheduler + | PNDMScheduler + | LMSDiscreteScheduler + | EulerDiscreteScheduler + | EulerAncestralDiscreteScheduler + | DPMSolverMultistepScheduler, motion_adapter: Optional[MotionAdapter] = None, feature_extractor: CLIPImageProcessor = None, image_encoder: CLIPVisionModelWithProjection = None, @@ -226,7 +224,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -234,7 +232,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -333,7 +331,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -674,29 +672,29 @@ def num_timesteps(self): def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, strength: float = 1.0, num_frames: Optional[int] = 16, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, motion_scale: int = 0, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): r""" The call function to the pipeline for generation. @@ -704,7 +702,7 @@ def __call__( Args: image (`PipelineImageInput`): The input image to be used for video generation. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. strength (`float`, *optional*, defaults to 1.0): Indicates extent to transform the reference `image`. Must be between 0 and 1. @@ -721,13 +719,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -743,7 +741,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -769,7 +767,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/pipeline_flax_utils.py b/src/diffusers/pipelines/pipeline_flax_utils.py index 2724c764c771..d537f533a02f 100644 --- a/src/diffusers/pipelines/pipeline_flax_utils.py +++ b/src/diffusers/pipelines/pipeline_flax_utils.py @@ -17,7 +17,7 @@ import importlib import inspect import os -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Optional import flax import numpy as np @@ -90,12 +90,12 @@ class FlaxImagePipelineOutput(BaseOutput): Output class for image pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray class FlaxDiffusionPipeline(ConfigMixin, PushToHubMixin): @@ -150,8 +150,8 @@ def register_modules(self, **kwargs): def save_pretrained( self, - save_directory: Union[str, os.PathLike], - params: Union[Dict, FrozenDict], + save_directory: str | os.PathLike, + params: Dict | FrozenDict, push_to_hub: bool = False, **kwargs, ): @@ -168,7 +168,7 @@ class implements both a save and loading method. The pipeline is easily reloaded Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ self.save_config(save_directory) @@ -228,7 +228,7 @@ class implements both a save and loading method. The pipeline is easily reloaded @classmethod @validate_hf_hub_args - def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + def from_pretrained(cls, pretrained_model_name_or_path: Optional[str | os.PathLike], **kwargs): r""" Instantiate a Flax-based diffusion pipeline from pretrained pipeline weights. @@ -254,7 +254,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -544,7 +544,7 @@ def _get_signature_keys(cls, obj): return expected_modules, optional_parameters @property - def components(self) -> Dict[str, Any]: + def components(self) -> dict[str, Any]: r""" The `self.components` property can be useful to run different pipelines with the same weights and diff --git a/src/diffusers/pipelines/pipeline_loading_utils.py b/src/diffusers/pipelines/pipeline_loading_utils.py index dd542145d3fa..2978e4f39f3a 100644 --- a/src/diffusers/pipelines/pipeline_loading_utils.py +++ b/src/diffusers/pipelines/pipeline_loading_utils.py @@ -17,7 +17,7 @@ import re import warnings from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import httpx import requests @@ -208,7 +208,7 @@ def filter_with_regex(filenames, pattern_re): return {f for f in filenames if pattern_re.match(f.split("/")[-1]) is not None} -def variant_compatible_siblings(filenames, variant=None, ignore_patterns=None) -> Union[List[os.PathLike], str]: +def variant_compatible_siblings(filenames, variant=None, ignore_patterns=None) -> list[os.PathLike] | str: weight_names = [ WEIGHTS_NAME, SAFETENSORS_WEIGHTS_NAME, @@ -509,12 +509,12 @@ def _get_pipeline_class( def _load_empty_model( library_name: str, class_name: str, - importable_classes: List[Any], + importable_classes: list[Any], pipelines: Any, is_pipeline_module: bool, name: str, - torch_dtype: Union[str, torch.dtype], - cached_folder: Union[str, os.PathLike], + torch_dtype: str | torch.dtype, + cached_folder: str | os.PathLike, **kwargs, ): # retrieve class objects. @@ -591,7 +591,7 @@ def _load_empty_model( def _assign_components_to_devices( - module_sizes: Dict[str, float], device_memory: Dict[str, float], device_mapping_strategy: str = "balanced" + module_sizes: dict[str, float], device_memory: dict[str, float], device_mapping_strategy: str = "balanced" ): device_ids = list(device_memory.keys()) device_cycle = device_ids + device_ids[::-1] @@ -722,25 +722,25 @@ def _get_final_device_map(device_map, pipeline_class, passed_class_obj, init_dic def load_sub_model( library_name: str, class_name: str, - importable_classes: List[Any], + importable_classes: list[Any], pipelines: Any, is_pipeline_module: bool, pipeline_class: Any, torch_dtype: torch.dtype, provider: Any, sess_options: Any, - device_map: Optional[Union[Dict[str, torch.device], str]], - max_memory: Optional[Dict[Union[int, str], Union[int, str]]], - offload_folder: Optional[Union[str, os.PathLike]], + device_map: Optional[dict[str, torch.device] | str], + max_memory: Optional[dict[int | str, int | str]], + offload_folder: Optional[str | os.PathLike], offload_state_dict: bool, - model_variants: Dict[str, str], + model_variants: dict[str, str], name: str, from_flax: bool, variant: str, low_cpu_mem_usage: bool, - cached_folder: Union[str, os.PathLike], + cached_folder: str | os.PathLike, use_safetensors: bool, - dduf_entries: Optional[Dict[str, DDUFEntry]], + dduf_entries: Optional[dict[str, DDUFEntry]], provider_options: Any, quantization_config: Optional[Any] = None, ): @@ -1021,9 +1021,9 @@ def get_connected_passed_kwargs(prefix): def _get_custom_components_and_folders( pretrained_model_name: str, - config_dict: Dict[str, Any], - filenames: Optional[List[str]] = None, - variant_filenames: Optional[List[str]] = None, + config_dict: dict[str, Any], + filenames: Optional[list[str]] = None, + variant_filenames: Optional[list[str]] = None, variant: Optional[str] = None, ): config_dict = config_dict.copy() @@ -1057,15 +1057,15 @@ def _get_custom_components_and_folders( def _get_ignore_patterns( passed_components, - model_folder_names: List[str], - model_filenames: List[str], + model_folder_names: list[str], + model_filenames: list[str], use_safetensors: bool, from_flax: bool, allow_pickle: bool, use_onnx: bool, is_onnx: bool, variant: Optional[str] = None, -) -> List[str]: +) -> list[str]: if ( use_safetensors and not allow_pickle diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py index 392d5fb3feb4..2d1bb989044c 100644 --- a/src/diffusers/pipelines/pipeline_utils.py +++ b/src/diffusers/pipelines/pipeline_utils.py @@ -21,7 +21,7 @@ import sys from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union, get_args, get_origin +from typing import Any, Callable, Dict, Optional, Union, get_args, get_origin import httpx import numpy as np @@ -120,12 +120,12 @@ class ImagePipelineOutput(BaseOutput): Output class for image pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray @dataclass @@ -135,7 +135,7 @@ class AudioPipelineOutput(BaseOutput): Args: audios (`np.ndarray`) - List of denoised audio samples of a NumPy array of shape `(batch_size, num_channels, sample_rate)`. + list of denoised audio samples of a NumPy array of shape `(batch_size, num_channels, sample_rate)`. """ audios: np.ndarray @@ -192,7 +192,7 @@ class DiffusionPipeline(ConfigMixin, PushToHubMixin): - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the diffusion pipeline's components. - - **_optional_components** (`List[str]`) -- List of all optional components that don't have to be passed to the + - **_optional_components** (`list[str]`) -- list of all optional components that don't have to be passed to the pipeline to function (should be overridden by subclasses). """ @@ -236,10 +236,10 @@ def __setattr__(self, name: str, value: Any): def save_pretrained( self, - save_directory: Union[str, os.PathLike], + save_directory: str | os.PathLike, safe_serialization: bool = True, variant: Optional[str] = None, - max_shard_size: Optional[Union[int, str]] = None, + max_shard_size: Optional[int | str] = None, push_to_hub: bool = False, **kwargs, ): @@ -267,7 +267,7 @@ class implements both a save and loading method. The pipeline is easily reloaded repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ model_index_dict = dict(self.config) @@ -591,7 +591,7 @@ def dtype(self) -> torch.dtype: @classmethod @validate_hf_hub_args - def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs) -> Self: + def from_pretrained(cls, pretrained_model_name_or_path: Optional[str | os.PathLike], **kwargs) -> Self: r""" Instantiate a PyTorch diffusion pipeline from pretrained pipeline weights. @@ -648,7 +648,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -1147,7 +1147,7 @@ def remove_all_hooks(self): accelerate.hooks.remove_hook_from_module(model, recurse=True) self._all_hooks = [] - def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the accelerator when its @@ -1263,7 +1263,7 @@ def maybe_free_model_hooks(self): # make sure the model is in the same state as before calling it self.enable_model_cpu_offload(device=getattr(self, "_offload_device", "cuda")) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using 🤗 Accelerate, significantly reducing memory usage. When called, the state dicts of all `torch.nn.Module` components (except those in `self._exclude_from_cpu_offload`) are saved to CPU @@ -1341,7 +1341,7 @@ def enable_group_offload( record_stream: bool = False, low_cpu_mem_usage=False, offload_to_disk_path: Optional[str] = None, - exclude_modules: Optional[Union[str, List[str]]] = None, + exclude_modules: Optional[str | list[str]] = None, ) -> None: r""" Applies group offloading to the internal layers of a torch.nn.Module. To understand what group offloading is, @@ -1401,7 +1401,7 @@ def enable_group_offload( If True, the CPU memory usage is minimized by pinning tensors on-the-fly instead of pre-pinning them. This option only matters when using streamed CPU offloading (i.e. `use_stream=True`). This can be useful when the CPU memory is a bottleneck but may counteract the benefits of using streams. - exclude_modules (`Union[str, List[str]]`, defaults to `None`): List of modules to exclude from offloading. + exclude_modules (`Union[str, list[str]]`, defaults to `None`): list of modules to exclude from offloading. Example: ```python @@ -1472,7 +1472,7 @@ def reset_device_map(self): @classmethod @validate_hf_hub_args - def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: + def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike: r""" Download and cache a PyTorch diffusion pipeline from pretrained pipeline weights. @@ -1505,7 +1505,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -1568,7 +1568,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: use_onnx = kwargs.pop("use_onnx", None) load_connected_pipeline = kwargs.pop("load_connected_pipeline", False) trust_remote_code = kwargs.pop("trust_remote_code", False) - dduf_file: Optional[Dict[str, DDUFEntry]] = kwargs.pop("dduf_file", None) + dduf_file: Optional[dict[str, DDUFEntry]] = kwargs.pop("dduf_file", None) if dduf_file: if custom_pipeline: @@ -1818,14 +1818,14 @@ def _get_signature_types(cls): signature_types[k] = (v.annotation,) elif get_origin(v.annotation) == Union: signature_types[k] = get_args(v.annotation) - elif get_origin(v.annotation) in [List, Dict, list, dict]: + elif get_origin(v.annotation) in [list, Dict, list, dict]: signature_types[k] = (v.annotation,) else: logger.warning(f"cannot get type annotation for Parameter {k} of {cls}.") return signature_types @property - def parameters(self) -> Dict[str, Any]: + def parameters(self) -> dict[str, Any]: r""" The `self.parameters` property can be useful to run different pipelines with the same weights and configurations without reallocating additional memory. @@ -1855,7 +1855,7 @@ def parameters(self) -> Dict[str, Any]: return pipeline_parameters @property - def components(self) -> Dict[str, Any]: + def components(self) -> dict[str, Any]: r""" The `self.components` property can be useful to run different pipelines with the same weights and configurations without reallocating additional memory. @@ -1975,7 +1975,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): for module in modules: fn_recursive_set_mem_eff(module) - def enable_attention_slicing(self, slice_size: Optional[Union[str, int]] = "auto"): + def enable_attention_slicing(self, slice_size: Optional[str | int] = "auto"): r""" Enable sliced attention computation. When this option is enabled, the attention module splits the input tensor in slices to compute attention in several steps. For more than one attention head, the computation is performed diff --git a/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py b/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py index 1d718a4852a4..872ea43d8ea4 100644 --- a/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +++ b/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py @@ -16,7 +16,7 @@ import inspect import re import urllib.parse as ul -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -182,9 +182,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -199,15 +199,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -302,7 +302,7 @@ def __init__( # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -319,9 +319,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -687,17 +687,17 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = None, width: Optional[int] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -711,26 +711,26 @@ def __call__( use_resolution_binning: bool = True, max_sequence_length: int = 120, **kwargs, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -749,7 +749,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py b/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py index bb169ac5c443..39d1f960bb3d 100644 --- a/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py +++ b/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py @@ -16,7 +16,7 @@ import inspect import re import urllib.parse as ul -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import torch from transformers import T5EncoderModel, T5Tokenizer @@ -126,9 +126,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -143,15 +143,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -246,7 +246,7 @@ def __init__( # Copied from diffusers.pipelines.pixart_alpha.pipeline_pixart_alpha.PixArtAlphaPipeline.encode_prompt with 120->300 def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -263,9 +263,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -632,17 +632,17 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, height: Optional[int] = None, width: Optional[int] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -656,26 +656,26 @@ def __call__( use_resolution_binning: bool = True, max_sequence_length: int = 300, **kwargs, - ) -> Union[ImagePipelineOutput, Tuple]: + ) -> ImagePipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -694,7 +694,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/prx/pipeline_output.py b/src/diffusers/pipelines/prx/pipeline_output.py index ea1bc9bf418a..81f92c294735 100644 --- a/src/diffusers/pipelines/prx/pipeline_output.py +++ b/src/diffusers/pipelines/prx/pipeline_output.py @@ -13,7 +13,6 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -27,9 +26,9 @@ class PRXPipelineOutput(BaseOutput): Output class for PRX pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/prx/pipeline_prx.py b/src/diffusers/pipelines/prx/pipeline_prx.py index a3bd3e6b45e7..20ef5ac12be0 100644 --- a/src/diffusers/pipelines/prx/pipeline_prx.py +++ b/src/diffusers/pipelines/prx/pipeline_prx.py @@ -16,7 +16,7 @@ import inspect import re import urllib.parse as ul -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import ftfy import torch @@ -251,8 +251,8 @@ def __init__( transformer: PRXTransformer2DModel, scheduler: FlowMatchEulerDiscreteScheduler, text_encoder: T5GemmaEncoder, - tokenizer: Union[T5TokenizerFast, GemmaTokenizerFast, AutoTokenizer], - vae: Optional[Union[AutoencoderKL, AutoencoderDC]] = None, + tokenizer: T5TokenizerFast | GemmaTokenizerFast | AutoTokenizer, + vae: Optional[AutoencoderKL | AutoencoderDC] = None, default_sample_size: Optional[int] = DEFAULT_RESOLUTION, ): super().__init__() @@ -337,7 +337,7 @@ def prepare_latents( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, do_classifier_free_guidance: bool = True, negative_prompt: str = "", @@ -387,7 +387,7 @@ def encode_prompt( negative_prompt_attention_mask if do_classifier_free_guidance else None, ) - def _tokenize_prompts(self, prompts: List[str], device: torch.device): + def _tokenize_prompts(self, prompts: list[str], device: torch.device): """Tokenize and clean prompts.""" cleaned = [self.text_preprocessor.clean_text(text) for text in prompts] tokens = self.tokenizer( @@ -402,7 +402,7 @@ def _tokenize_prompts(self, prompts: List[str], device: torch.device): def _encode_prompt_standard( self, - prompt: List[str], + prompt: list[str], device: torch.device, do_classifier_free_guidance: bool = True, negative_prompt: str = "", @@ -440,11 +440,11 @@ def _encode_prompt_standard( def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: int, width: int, guidance_scale: float, - callback_on_step_end_tensor_inputs: Optional[List[str]] = None, + callback_on_step_end_tensor_inputs: Optional[list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, ): @@ -494,15 +494,15 @@ def check_inputs( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 4.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -512,13 +512,13 @@ def __call__( return_dict: bool = True, use_resolution_binning: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds` instead. negative_prompt (`str`, *optional*, defaults to `""`): @@ -531,7 +531,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 28): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -543,7 +543,7 @@ def __call__( usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -577,7 +577,7 @@ def __call__( with the following arguments: `callback_on_step_end(self, step, timestep, callback_kwargs)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include tensors that are listed in the `._callback_tensor_inputs` attribute. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_output.py b/src/diffusers/pipelines/qwenimage/pipeline_output.py index eef4b60e3770..e4ed06856e6a 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_output.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class QwenImagePipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py index 33dc2039b986..ed03ef1bd090 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -73,9 +73,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -90,15 +90,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -187,7 +187,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -225,7 +225,7 @@ def _get_qwen_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -235,7 +235,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -452,16 +452,16 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -469,19 +469,19 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -499,7 +499,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -515,7 +515,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -543,7 +543,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet.py index 5111096d93c1..460a8944089d 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -133,9 +133,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -150,15 +150,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -218,7 +218,7 @@ def __init__( text_encoder: Qwen2_5_VLForConditionalGeneration, tokenizer: Qwen2Tokenizer, transformer: QwenImageTransformer2DModel, - controlnet: Union[QwenImageControlNetModel, QwenImageMultiControlNetModel], + controlnet: QwenImageControlNetModel | QwenImageMultiControlNetModel, ): super().__init__() @@ -251,7 +251,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor # Coped from diffusers.pipelines.qwenimage.pipeline_qwenimage.get_qwen_prompt_embeds def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -290,7 +290,7 @@ def _get_qwen_prompt_embeds( # Coped from diffusers.pipelines.qwenimage.pipeline_qwenimage.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -300,7 +300,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -552,20 +552,20 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, control_image: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -573,19 +573,19 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -603,7 +603,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -619,7 +619,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -647,7 +647,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet_inpaint.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet_inpaint.py index 102a813ab582..8adccdc0f3a5 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet_inpaint.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -106,9 +106,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -123,15 +123,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -233,7 +233,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor # Coped from diffusers.pipelines.qwenimage.pipeline_qwenimage.get_qwen_prompt_embeds def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -272,7 +272,7 @@ def _get_qwen_prompt_embeds( # Coped from diffusers.pipelines.qwenimage.pipeline_qwenimage.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -281,7 +281,7 @@ def encode_prompt( ): r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -592,21 +592,21 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 1.0, - control_guidance_start: Union[float, List[float]] = 0.0, - control_guidance_end: Union[float, List[float]] = 1.0, + control_guidance_start: float | list[float] = 0.0, + control_guidance_end: float | list[float] = 1.0, control_image: PipelineImageInput = None, control_mask: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -614,19 +614,19 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -639,7 +639,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -651,7 +651,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -679,7 +679,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py index ed37b238c8c9..bf2603edd862 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -82,9 +82,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -99,15 +99,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -225,7 +225,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -272,7 +272,7 @@ def _get_qwen_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, @@ -283,7 +283,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded image (`torch.Tensor`, *optional*): image to be encoded @@ -548,16 +548,16 @@ def interrupt(self): def __call__( self, image: Optional[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -565,25 +565,25 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -601,7 +601,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -617,7 +617,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -645,7 +645,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_inpaint.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_inpaint.py index d54d1881fa4e..82f54cf57734 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_inpaint.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_inpaint.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -83,9 +83,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -100,15 +100,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -236,7 +236,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline._get_qwen_prompt_embeds def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -284,7 +284,7 @@ def _get_qwen_prompt_embeds( # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, @@ -295,7 +295,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded image (`torch.Tensor`, *optional*): image to be encoded @@ -681,8 +681,8 @@ def interrupt(self): def __call__( self, image: Optional[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, mask_image: PipelineImageInput = None, masked_image_latents: PipelineImageInput = None, true_cfg_scale: float = 4.0, @@ -691,10 +691,10 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -702,25 +702,25 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -731,14 +731,14 @@ def __call__( enabled by setting `true_cfg_scale > 1` and a provided `negative_prompt`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will ge generated by `mask_image`. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -761,7 +761,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -777,7 +777,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -805,7 +805,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_plus.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_plus.py index ec203edf166c..a2b771b2d86f 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_plus.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_plus.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -85,9 +85,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -102,15 +102,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -228,7 +228,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, @@ -286,7 +286,7 @@ def _get_qwen_prompt_embeds( # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], image: Optional[torch.Tensor] = None, device: Optional[torch.device] = None, num_images_per_prompt: int = 1, @@ -297,7 +297,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded image (`torch.Tensor`, *optional*): image to be encoded @@ -517,16 +517,16 @@ def interrupt(self): def __call__( self, image: Optional[PipelineImageInput] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -534,25 +534,25 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). @@ -570,7 +570,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -586,7 +586,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -614,7 +614,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_img2img.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_img2img.py index cb4c5d8016bb..c2e69da33876 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_img2img.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_img2img.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -75,9 +75,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -92,15 +92,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -194,7 +194,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._get_qwen_prompt_embeds def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -268,7 +268,7 @@ def get_timesteps(self, num_inference_steps, strength, device): # Copied fromCopied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -278,7 +278,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -526,18 +526,18 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -545,23 +545,23 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -587,7 +587,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -603,7 +603,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -631,7 +631,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_inpaint.py b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_inpaint.py index 1915c27eb2bb..66cf7391f664 100644 --- a/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_inpaint.py +++ b/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_inpaint.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -78,9 +78,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -95,15 +95,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -204,7 +204,7 @@ def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._get_qwen_prompt_embeds def _get_qwen_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): @@ -279,7 +279,7 @@ def get_timesteps(self, num_inference_steps, strength, device): # Copied fromCopied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -289,7 +289,7 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -636,8 +636,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, true_cfg_scale: float = 4.0, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, @@ -647,10 +647,10 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: Optional[float] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_mask: Optional[torch.Tensor] = None, @@ -658,23 +658,23 @@ def __call__( negative_prompt_embeds_mask: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is not greater than `1`). - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -687,14 +687,14 @@ def __call__( setting `true_cfg_scale > 1` and a provided `negative_prompt`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will be generated by `mask_image`. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -717,7 +717,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -733,7 +733,7 @@ def __call__( enable classifier-free guidance computations). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -761,7 +761,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/sana/pipeline_output.py b/src/diffusers/pipelines/sana/pipeline_output.py index f8ac12951644..b9d095906e06 100644 --- a/src/diffusers/pipelines/sana/pipeline_output.py +++ b/src/diffusers/pipelines/sana/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class SanaPipelineOutput(BaseOutput): Output class for Sana pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/sana/pipeline_sana.py b/src/diffusers/pipelines/sana/pipeline_sana.py index ac979305ca6d..096379cd3efb 100644 --- a/src/diffusers/pipelines/sana/pipeline_sana.py +++ b/src/diffusers/pipelines/sana/pipeline_sana.py @@ -17,7 +17,7 @@ import re import urllib.parse as ul import warnings -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import Gemma2PreTrainedModel, GemmaTokenizer, GemmaTokenizerFast @@ -131,9 +131,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -148,15 +148,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -201,7 +201,7 @@ class SanaPipeline(DiffusionPipeline, SanaLoraLoaderMixin): def __init__( self, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, text_encoder: Gemma2PreTrainedModel, vae: AutoencoderDC, transformer: SanaTransformer2DModel, @@ -275,18 +275,18 @@ def disable_vae_tiling(self): def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: torch.device, dtype: torch.dtype, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -333,7 +333,7 @@ def _get_gemma_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -344,16 +344,16 @@ def encode_prompt( negative_prompt_attention_mask: Optional[torch.Tensor] = None, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, lora_scale: Optional[float] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -728,17 +728,17 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, height: int = 1024, width: int = 1024, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -748,11 +748,11 @@ def __call__( return_dict: bool = True, clean_caption: bool = False, use_resolution_binning: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 300, - complex_human_instruction: List[str] = [ + complex_human_instruction: list[str] = [ "Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:", "- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.", "- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.", @@ -762,26 +762,26 @@ def __call__( "Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:", "User Prompt: ", ], - ) -> Union[SanaPipelineOutput, Tuple]: + ) -> SanaPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 20): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -800,7 +800,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -838,13 +838,13 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to `300`): Maximum sequence length to use with the `prompt`. - complex_human_instruction (`List[str]`, *optional*): + complex_human_instruction (`list[str]`, *optional*): Instructions for complex human attention: https://github.com/NVlabs/Sana/blob/main/configs/sana_app_config/Sana_1600M_app.yaml#L55. diff --git a/src/diffusers/pipelines/sana/pipeline_sana_controlnet.py b/src/diffusers/pipelines/sana/pipeline_sana_controlnet.py index 55ed7b84ebdf..1f0d0c2c7f0b 100644 --- a/src/diffusers/pipelines/sana/pipeline_sana_controlnet.py +++ b/src/diffusers/pipelines/sana/pipeline_sana_controlnet.py @@ -17,7 +17,7 @@ import re import urllib.parse as ul import warnings -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import Gemma2PreTrainedModel, GemmaTokenizer, GemmaTokenizerFast @@ -138,9 +138,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -155,15 +155,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -208,7 +208,7 @@ class SanaControlNetPipeline(DiffusionPipeline, SanaLoraLoaderMixin): def __init__( self, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, text_encoder: Gemma2PreTrainedModel, vae: AutoencoderDC, transformer: SanaTransformer2DModel, @@ -289,18 +289,18 @@ def disable_vae_tiling(self): # Copied from diffusers.pipelines.sana.pipeline_sana.SanaPipeline._get_gemma_prompt_embeds def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: torch.device, dtype: torch.dtype, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -348,7 +348,7 @@ def _get_gemma_prompt_embeds( # Copied from diffusers.pipelines.sana.pipeline_sana.SanaPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], do_classifier_free_guidance: bool = True, negative_prompt: str = "", num_images_per_prompt: int = 1, @@ -359,16 +359,16 @@ def encode_prompt( negative_prompt_attention_mask: Optional[torch.Tensor] = None, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, lora_scale: Optional[float] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For PixArt-Alpha, this should be "". @@ -777,19 +777,19 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, negative_prompt: str = "", num_inference_steps: int = 20, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 4.5, control_image: PipelineImageInput = None, - controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + controlnet_conditioning_scale: float | list[float] = 1.0, num_images_per_prompt: Optional[int] = 1, height: int = 1024, width: int = 1024, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -799,11 +799,11 @@ def __call__( return_dict: bool = True, clean_caption: bool = False, use_resolution_binning: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 300, - complex_human_instruction: List[str] = [ + complex_human_instruction: list[str] = [ "Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:", "- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.", "- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.", @@ -813,26 +813,26 @@ def __call__( "Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:", "User Prompt: ", ], - ) -> Union[SanaPipelineOutput, Tuple]: + ) -> SanaPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_inference_steps (`int`, *optional*, defaults to 20): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -842,15 +842,15 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: - `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, `list[np.ndarray]`,: + `list[list[torch.Tensor]]`, `list[list[np.ndarray]]` or `list[list[PIL.Image.Image]]`): The ControlNet input condition to provide guidance to the `unet` for generation. If the type is specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. - controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. @@ -863,7 +863,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -901,13 +901,13 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to `300`): Maximum sequence length to use with the `prompt`. - complex_human_instruction (`List[str]`, *optional*): + complex_human_instruction (`list[str]`, *optional*): Instructions for complex human attention: https://github.com/NVlabs/Sana/blob/main/configs/sana_app_config/Sana_1600M_app.yaml#L55. diff --git a/src/diffusers/pipelines/sana/pipeline_sana_sprint.py b/src/diffusers/pipelines/sana/pipeline_sana_sprint.py index 62b978829271..fd5cca63a930 100644 --- a/src/diffusers/pipelines/sana/pipeline_sana_sprint.py +++ b/src/diffusers/pipelines/sana/pipeline_sana_sprint.py @@ -17,7 +17,7 @@ import re import urllib.parse as ul import warnings -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import Gemma2PreTrainedModel, GemmaTokenizer, GemmaTokenizerFast @@ -82,9 +82,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -99,15 +99,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -152,7 +152,7 @@ class SanaSprintPipeline(DiffusionPipeline, SanaLoraLoaderMixin): def __init__( self, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, text_encoder: Gemma2PreTrainedModel, vae: AutoencoderDC, transformer: SanaTransformer2DModel, @@ -227,18 +227,18 @@ def disable_vae_tiling(self): # Copied from diffusers.pipelines.sana.pipeline_sana.SanaPipeline._get_gemma_prompt_embeds def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: torch.device, dtype: torch.dtype, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -285,21 +285,21 @@ def _get_gemma_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, lora_scale: Optional[float] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -616,9 +616,9 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 2, - timesteps: List[int] = None, + timesteps: list[int] = None, max_timesteps: float = 1.57080, intermediate_timesteps: float = 1.3, guidance_scale: float = 4.5, @@ -626,7 +626,7 @@ def __call__( height: int = 1024, width: int = 1024, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -634,11 +634,11 @@ def __call__( return_dict: bool = True, clean_caption: bool = False, use_resolution_binning: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 300, - complex_human_instruction: List[str] = [ + complex_human_instruction: list[str] = [ "Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:", "- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.", "- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.", @@ -648,12 +648,12 @@ def __call__( "Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:", "User Prompt: ", ], - ) -> Union[SanaPipelineOutput, Tuple]: + ) -> SanaPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 20): @@ -663,7 +663,7 @@ def __call__( The maximum timestep value used in the SCM scheduler. intermediate_timesteps (`float`, *optional*, defaults to 1.3): The intermediate timestep value used in SCM scheduler (only used when num_inference_steps=2). - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -682,7 +682,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -715,13 +715,13 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to `300`): Maximum sequence length to use with the `prompt`. - complex_human_instruction (`List[str]`, *optional*): + complex_human_instruction (`list[str]`, *optional*): Instructions for complex human attention: https://github.com/NVlabs/Sana/blob/main/configs/sana_app_config/Sana_1600M_app.yaml#L55. diff --git a/src/diffusers/pipelines/sana/pipeline_sana_sprint_img2img.py b/src/diffusers/pipelines/sana/pipeline_sana_sprint_img2img.py index 8899ed84c4e5..5e8ac2690a75 100644 --- a/src/diffusers/pipelines/sana/pipeline_sana_sprint_img2img.py +++ b/src/diffusers/pipelines/sana/pipeline_sana_sprint_img2img.py @@ -17,7 +17,7 @@ import re import urllib.parse as ul import warnings -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch import torch.nn.functional as F @@ -88,9 +88,9 @@ def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -105,15 +105,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -159,7 +159,7 @@ class SanaSprintImg2ImgPipeline(DiffusionPipeline, SanaLoraLoaderMixin): def __init__( self, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], + tokenizer: GemmaTokenizer | GemmaTokenizerFast, text_encoder: Gemma2PreTrainedModel, vae: AutoencoderDC, transformer: SanaTransformer2DModel, @@ -237,18 +237,18 @@ def disable_vae_tiling(self): # Copied from diffusers.pipelines.sana.pipeline_sana.SanaPipeline._get_gemma_prompt_embeds def _get_gemma_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], device: torch.device, dtype: torch.dtype, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on @@ -296,21 +296,21 @@ def _get_gemma_prompt_embeds( # Copied from diffusers.pipelines.sana.pipeline_sana_sprint.SanaSprintPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, clean_caption: bool = False, max_sequence_length: int = 300, - complex_human_instruction: Optional[List[str]] = None, + complex_human_instruction: Optional[list[str]] = None, lora_scale: Optional[float] = None, ): r""" Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -687,9 +687,9 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_inference_steps: int = 2, - timesteps: List[int] = None, + timesteps: list[int] = None, max_timesteps: float = 1.57080, intermediate_timesteps: float = 1.3, guidance_scale: float = 4.5, @@ -699,7 +699,7 @@ def __call__( height: int = 1024, width: int = 1024, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_attention_mask: Optional[torch.Tensor] = None, @@ -707,11 +707,11 @@ def __call__( return_dict: bool = True, clean_caption: bool = False, use_resolution_binning: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 300, - complex_human_instruction: List[str] = [ + complex_human_instruction: list[str] = [ "Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:", "- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.", "- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.", @@ -721,12 +721,12 @@ def __call__( "Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:", "User Prompt: ", ], - ) -> Union[SanaPipelineOutput, Tuple]: + ) -> SanaPipelineOutput | tuple: """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 20): @@ -736,7 +736,7 @@ def __call__( The maximum timestep value used in the SCM scheduler. intermediate_timesteps (`float`, *optional*, defaults to 1.3): The intermediate timestep value used in SCM scheduler (only used when num_inference_steps=2). - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. @@ -755,7 +755,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -788,13 +788,13 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to `300`): Maximum sequence length to use with the `prompt`. - complex_human_instruction (`List[str]`, *optional*): + complex_human_instruction (`list[str]`, *optional*): Instructions for complex human attention: https://github.com/NVlabs/Sana/blob/main/configs/sana_app_config/Sana_1600M_app.yaml#L55. diff --git a/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py b/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py index 349912993981..365dfd488ec2 100644 --- a/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py +++ b/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -13,13 +13,13 @@ class SemanticStableDiffusionPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`) - List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or + nsfw_content_detected (`list[bool]`) + list indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] diff --git a/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py b/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py index 49b09e205cc5..654a0bdbf0a0 100644 --- a/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +++ b/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py @@ -1,6 +1,6 @@ import inspect from itertools import repeat -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer @@ -223,37 +223,37 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - editing_prompt: Optional[Union[str, List[str]]] = None, + editing_prompt: Optional[str | list[str]] = None, editing_prompt_embeddings: Optional[torch.Tensor] = None, - reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, - edit_guidance_scale: Optional[Union[float, List[float]]] = 5, - edit_warmup_steps: Optional[Union[int, List[int]]] = 10, - edit_cooldown_steps: Optional[Union[int, List[int]]] = None, - edit_threshold: Optional[Union[float, List[float]]] = 0.9, + reverse_editing_direction: Optional[bool | list[bool]] = False, + edit_guidance_scale: Optional[float | list[float]] = 5, + edit_warmup_steps: Optional[int | list[int]] = 10, + edit_cooldown_steps: Optional[int | list[int]] = None, + edit_threshold: Optional[float | list[float]] = 0.9, edit_momentum_scale: Optional[float] = 0.1, edit_mom_beta: Optional[float] = 0.4, - edit_weights: Optional[List[float]] = None, - sem_guidance: Optional[List[torch.Tensor]] = None, + edit_weights: Optional[list[float]] = None, + sem_guidance: Optional[list[torch.Tensor]] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -265,7 +265,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -273,7 +273,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -291,24 +291,24 @@ def __call__( callback_steps (`int`, *optional*, defaults to 1): The frequency at which the `callback` function is called. If not specified, the callback is called at every step. - editing_prompt (`str` or `List[str]`, *optional*): + editing_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to use for semantic guidance. Semantic guidance is disabled by setting `editing_prompt = None`. Guidance direction of prompt should be specified via `reverse_editing_direction`. editing_prompt_embeddings (`torch.Tensor`, *optional*): Pre-computed embeddings to use for semantic guidance. Guidance direction of embedding should be specified via `reverse_editing_direction`. - reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + reverse_editing_direction (`bool` or `list[bool]`, *optional*, defaults to `False`): Whether the corresponding prompt in `editing_prompt` should be increased or decreased. - edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + edit_guidance_scale (`float` or `list[float]`, *optional*, defaults to 5): Guidance scale for semantic guidance. If provided as a list, values should correspond to `editing_prompt`. - edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + edit_warmup_steps (`float` or `list[float]`, *optional*, defaults to 10): Number of diffusion steps (for each prompt) for which semantic guidance is not applied. Momentum is calculated for those steps and applied once all warmup periods are over. - edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + edit_cooldown_steps (`float` or `list[float]`, *optional*, defaults to `None`): Number of diffusion steps (for each prompt) after which semantic guidance is longer applied. - edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + edit_threshold (`float` or `list[float]`, *optional*, defaults to 0.9): Threshold of semantic guidance. edit_momentum_scale (`float`, *optional*, defaults to 0.1): Scale of the momentum to be added to the semantic guidance at each diffusion step. If set to 0.0, @@ -318,11 +318,11 @@ def __call__( Defines how semantic guidance momentum builds up. `edit_mom_beta` indicates how much of the previous momentum is kept. Momentum is already built up during warmup (for diffusion steps smaller than `edit_warmup_steps`). - edit_weights (`List[float]`, *optional*, defaults to `None`): + edit_weights (`list[float]`, *optional*, defaults to `None`): Indicates how much each individual concept should influence the overall guidance. If no weights are provided all concepts are applied equally. - sem_guidance (`List[torch.Tensor]`, *optional*): - List of pre-generated guidance vectors to be applied at generation. Length of the list has to + sem_guidance (`list[torch.Tensor]`, *optional*): + list of pre-generated guidance vectors to be applied at generation. Length of the list has to correspond to `num_inference_steps`. Examples: @@ -458,7 +458,7 @@ def __call__( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): diff --git a/src/diffusers/pipelines/shap_e/camera.py b/src/diffusers/pipelines/shap_e/camera.py index 31e1759d6154..81807b6ff35a 100644 --- a/src/diffusers/pipelines/shap_e/camera.py +++ b/src/diffusers/pipelines/shap_e/camera.py @@ -13,7 +13,6 @@ # limitations under the License. from dataclasses import dataclass -from typing import Tuple import numpy as np import torch @@ -33,7 +32,7 @@ class DifferentiableProjectiveCamera: height: int x_fov: float y_fov: float - shape: Tuple[int] + shape: tuple[int] def __post_init__(self): assert self.x.shape[0] == self.y.shape[0] == self.z.shape[0] == self.origin.shape[0] diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py index 49ddfd1196bf..1a39b9b48437 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -82,7 +82,7 @@ class ShapEPipelineOutput(BaseOutput): A list of images for 3D rendering. """ - images: Union[List[List[PIL.Image.Image]], List[List[np.ndarray]]] + images: list[list[PIL.Image.Image]] | list[list[np.ndarray]] class ShapEPipeline(DiffusionPipeline): @@ -195,7 +195,7 @@ def __call__( prompt: str, num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, guidance_scale: float = 4.0, frame_size: int = 64, @@ -206,14 +206,14 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py index 55d8b85822c4..1db7c4a977cb 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -83,7 +83,7 @@ class ShapEPipelineOutput(BaseOutput): A list of images for 3D rendering. """ - images: Union[PIL.Image.Image, np.ndarray] + images: PIL.Image.Image | np.ndarray class ShapEImg2ImgPipeline(DiffusionPipeline): @@ -147,7 +147,7 @@ def _encode_image( num_images_per_prompt, do_classifier_free_guidance, ): - if isinstance(image, List) and isinstance(image[0], torch.Tensor): + if isinstance(image, list) and isinstance(image[0], torch.Tensor): image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) if not isinstance(image, torch.Tensor): @@ -174,10 +174,10 @@ def _encode_image( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image: Union[PIL.Image.Image, List[PIL.Image.Image]], + image: PIL.Image.Image | list[PIL.Image.Image], num_images_per_prompt: int = 1, num_inference_steps: int = 25, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, guidance_scale: float = 4.0, frame_size: int = 64, @@ -188,7 +188,7 @@ def __call__( The call function to the pipeline for generation. Args: - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be used as the starting point. Can also accept image latents as image, but if passing latents directly it is not encoded again. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -196,7 +196,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -231,7 +231,7 @@ def __call__( batch_size = len(image) else: raise ValueError( - f"`image` has to be of type `PIL.Image.Image`, `torch.Tensor`, `List[PIL.Image.Image]` or `List[torch.Tensor]` but is {type(image)}" + f"`image` has to be of type `PIL.Image.Image`, `torch.Tensor`, `list[PIL.Image.Image]` or `list[torch.Tensor]` but is {type(image)}" ) device = self._execution_device diff --git a/src/diffusers/pipelines/shap_e/renderer.py b/src/diffusers/pipelines/shap_e/renderer.py index b268eae806a6..9f816af42ef4 100644 --- a/src/diffusers/pipelines/shap_e/renderer.py +++ b/src/diffusers/pipelines/shap_e/renderer.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import Dict, Optional, Tuple +from typing import Optional import numpy as np import torch @@ -131,7 +131,7 @@ def _convert_srgb_to_linear(u: torch.Tensor): def _create_flat_edge_indices( flat_cube_indices: torch.Tensor, - grid_size: Tuple[int, int, int], + grid_size: tuple[int, int, int], ): num_xs = (grid_size[0] - 1) * grid_size[1] * grid_size[2] y_offset = num_xs @@ -479,7 +479,7 @@ class MeshDecoderOutput(BaseOutput): verts: torch.Tensor faces: torch.Tensor - vertex_channels: Dict[str, torch.Tensor] + vertex_channels: dict[str, torch.Tensor] class MeshDecoder(nn.Module): @@ -742,13 +742,13 @@ class ShapEParamsProjModel(ModelMixin, ConfigMixin): def __init__( self, *, - param_names: Tuple[str] = ( + param_names: tuple[str] = ( "nerstf.mlp.0.weight", "nerstf.mlp.1.weight", "nerstf.mlp.2.weight", "nerstf.mlp.3.weight", ), - param_shapes: Tuple[Tuple[int]] = ( + param_shapes: tuple[tuple[int]] = ( (256, 93), (256, 256), (256, 256), @@ -786,13 +786,13 @@ class ShapERenderer(ModelMixin, ConfigMixin): def __init__( self, *, - param_names: Tuple[str] = ( + param_names: tuple[str] = ( "nerstf.mlp.0.weight", "nerstf.mlp.1.weight", "nerstf.mlp.2.weight", "nerstf.mlp.3.weight", ), - param_shapes: Tuple[Tuple[int]] = ( + param_shapes: tuple[tuple[int]] = ( (256, 93), (256, 256), (256, 256), @@ -804,7 +804,7 @@ def __init__( n_hidden_layers: int = 6, act_fn: str = "swish", insert_direction_at: int = 4, - background: Tuple[float] = ( + background: tuple[float] = ( 255.0, 255.0, 255.0, @@ -953,7 +953,7 @@ def decode_to_mesh( device, grid_size: int = 128, query_batch_size: int = 4096, - texture_channels: Tuple = ("R", "G", "B"), + texture_channels: tuple = ("R", "G", "B"), ): # 1. project the parameters from the generated latents projected_params = self.params_proj(latents) diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_output.py b/src/diffusers/pipelines/skyreels_v2/pipeline_output.py index 7a170d24c39a..dac2316362ec 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_output.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_output.py @@ -11,8 +11,8 @@ class SkyReelsV2PipelineOutput(BaseOutput): Output class for SkyReelsV2 pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py index 8562a5eaf0e6..99b000d54972 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import regex as re import torch @@ -154,7 +154,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -196,8 +196,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -210,9 +210,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -326,7 +326,7 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -377,32 +377,32 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 97, num_inference_steps: int = 50, guidance_scale: float = 6.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, defaults to `544`): @@ -422,7 +422,7 @@ def __call__( usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -445,7 +445,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py index d0a4e118ce43..3b9eae3ef722 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py @@ -16,7 +16,7 @@ import math import re from copy import deepcopy -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import ftfy import torch @@ -176,7 +176,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -218,8 +218,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -232,9 +232,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -356,7 +356,7 @@ def prepare_latents( num_frames: int = 97, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, base_latent_num_frames: Optional[int] = None, video_latents: Optional[torch.Tensor] = None, @@ -462,7 +462,7 @@ def generate_timestep_matrix( num_latent_frames] - step_update_mask (torch.Tensor): Boolean mask indicating which frames to update Shape: [num_iterations, num_latent_frames] - - valid_interval (list[tuple]): List of (start, end) intervals for each iteration + - valid_interval (list[tuple]): list of (start, end) intervals for each iteration Raises: ValueError: If ar_step is too small for the given configuration @@ -599,25 +599,25 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str], + negative_prompt: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 97, num_inference_steps: int = 50, guidance_scale: float = 6.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, overlap_history: Optional[int] = None, addnoise_condition: float = 0, @@ -630,10 +630,10 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -654,7 +654,7 @@ def __call__( usually at the expense of lower image quality. (**6.0 for T2V**, **5.0 for I2V**) num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -680,7 +680,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py index 959cbb32f23a..acbb89a5de55 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py @@ -16,7 +16,7 @@ import math import re from copy import deepcopy -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import ftfy import PIL @@ -181,7 +181,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -223,8 +223,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -237,9 +237,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -375,7 +375,7 @@ def prepare_latents( num_frames: int = 97, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, last_image: Optional[torch.Tensor] = None, video_latents: Optional[torch.Tensor] = None, @@ -383,7 +383,7 @@ def prepare_latents( causal_block_size: Optional[int] = None, overlap_history_latent_frames: Optional[int] = None, long_video_iter: Optional[int] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 latent_height = height // self.vae_scale_factor_spatial latent_width = width // self.vae_scale_factor_spatial @@ -508,7 +508,7 @@ def generate_timestep_matrix( num_latent_frames] - step_update_mask (torch.Tensor): Boolean mask indicating which frames to update Shape: [num_iterations, num_latent_frames] - - valid_interval (list[tuple]): List of (start, end) intervals for each iteration + - valid_interval (list[tuple]): list of (start, end) intervals for each iteration Raises: ValueError: If ar_step is too small for the given configuration @@ -646,15 +646,15 @@ def attention_kwargs(self): def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 97, num_inference_steps: int = 50, guidance_scale: float = 5.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -662,11 +662,11 @@ def __call__( last_image: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, overlap_history: Optional[int] = None, addnoise_condition: float = 0, @@ -681,10 +681,10 @@ def __call__( Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -705,7 +705,7 @@ def __call__( usually at the expense of lower image quality. (**6.0 for T2V**, **5.0 for I2V**) num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -737,7 +737,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py index 6fedfc795a40..fca4cfbf7b67 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py @@ -17,7 +17,7 @@ import math import re from copy import deepcopy -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import ftfy import torch @@ -117,9 +117,9 @@ def prompt_clean(text): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -134,15 +134,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -237,7 +237,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -279,8 +279,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -293,9 +293,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -423,7 +423,7 @@ def prepare_latents( num_frames: int = 97, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, video_latents: Optional[torch.Tensor] = None, base_latent_num_frames: Optional[int] = None, @@ -546,7 +546,7 @@ def generate_timestep_matrix( num_latent_frames] - step_update_mask (torch.Tensor): Boolean mask indicating which frames to update Shape: [num_iterations, num_latent_frames] - - valid_interval (list[tuple]): List of (start, end) intervals for each iteration + - valid_interval (list[tuple]): list of (start, end) intervals for each iteration Raises: ValueError: If ar_step is too small for the given configuration @@ -683,26 +683,26 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - video: List[Image.Image], - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + video: list[Image.Image], + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 120, num_inference_steps: int = 50, guidance_scale: float = 6.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, overlap_history: Optional[int] = None, addnoise_condition: float = 0, @@ -715,12 +715,12 @@ def __call__( The call function to the pipeline for generation. Args: - video (`List[Image.Image]`): + video (`list[Image.Image]`): The video to guide the video generation. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the video generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the video generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -741,7 +741,7 @@ def __call__( usually at the expense of lower image quality. (**6.0 for T2V**, **5.0 for I2V**) num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -767,7 +767,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py index d59b4ce3cb17..9bb32f3e94d6 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL import regex as re @@ -184,7 +184,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan_i2v.WanImageToVideoPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -237,8 +237,8 @@ def encode_image( # Copied from diffusers.pipelines.wan.pipeline_wan_i2v.WanImageToVideoPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -251,9 +251,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -380,10 +380,10 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, last_image: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 latent_height = height // self.vae_scale_factor_spatial latent_width = width // self.vae_scale_factor_spatial @@ -478,15 +478,15 @@ def attention_kwargs(self): def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 544, width: int = 960, num_frames: int = 97, num_inference_steps: int = 50, guidance_scale: float = 5.0, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -494,11 +494,11 @@ def __call__( last_image: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" @@ -507,10 +507,10 @@ def __call__( Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -531,7 +531,7 @@ def __call__( usually at the expense of lower image quality. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -560,7 +560,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_audio/pipeline_stable_audio.py b/src/diffusers/pipelines/stable_audio/pipeline_stable_audio.py index b7faf097ab0d..8c34fcb75adb 100644 --- a/src/diffusers/pipelines/stable_audio/pipeline_stable_audio.py +++ b/src/diffusers/pipelines/stable_audio/pipeline_stable_audio.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch from transformers import ( @@ -108,7 +108,7 @@ def __init__( vae: AutoencoderOobleck, text_encoder: T5EncoderModel, projection_model: StableAudioProjectionModel, - tokenizer: Union[T5Tokenizer, T5TokenizerFast], + tokenizer: T5Tokenizer | T5TokenizerFast, transformer: StableAudioDiTModel, scheduler: EDMDPMSolverMultistepScheduler, ): @@ -206,7 +206,7 @@ def encode_prompt( prompt_embeds = prompt_embeds[0] if do_classifier_free_guidance and negative_prompt is not None: - uncond_tokens: List[str] + uncond_tokens: list[str] if type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -491,15 +491,15 @@ def prepare_latents( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, audio_end_in_s: Optional[float] = None, audio_start_in_s: Optional[float] = 0.0, num_inference_steps: int = 100, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_waveforms_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, initial_audio_waveforms: Optional[torch.Tensor] = None, initial_audio_sampling_rate: Optional[torch.Tensor] = None, @@ -516,7 +516,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. audio_end_in_s (`float`, *optional*, defaults to 47.55): Audio end index in seconds. @@ -528,7 +528,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.0): A higher guidance scale value encourages the model to generate audio that is closely linked to the text `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in audio generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_waveforms_per_prompt (`int`, *optional*, defaults to 1): @@ -536,7 +536,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py index aa39983c4e43..7902dfd322fe 100644 --- a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +++ b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import torch from transformers import CLIPTextModelWithProjection, CLIPTokenizer @@ -182,7 +182,7 @@ def encode_prompt( prompt_embeds_pooled = prompt_embeds_pooled.repeat_interleave(num_images_per_prompt, dim=0) if negative_prompt_embeds is None and do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -303,30 +303,30 @@ def get_timestep_ratio_conditioning(self, t, alphas_cumprod): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image_embeddings: Union[torch.Tensor, List[torch.Tensor]], - prompt: Union[str, List[str]] = None, + image_embeddings: torch.Tensor | list[torch.Tensor], + prompt: str | list[str] = None, num_inference_steps: int = 10, guidance_scale: float = 0.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_pooled: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds_pooled: Optional[torch.Tensor] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - image_embedding (`torch.Tensor` or `List[torch.Tensor]`): + image_embedding (`torch.Tensor` or `list[torch.Tensor]`): Image Embeddings either extracted from an image or generated by a Prior Model. - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. num_inference_steps (`int`, *optional*, defaults to 12): The number of denoising steps. More denoising steps usually lead to a higher quality image at the @@ -337,7 +337,7 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `decoder_guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): @@ -356,7 +356,7 @@ def __call__( input argument. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -373,7 +373,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py index b3dc23f2e571..b723122baa77 100644 --- a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +++ b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import PIL import torch @@ -125,7 +125,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` @@ -135,7 +135,7 @@ def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[t self.prior_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) self.decoder_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a @@ -157,38 +157,38 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - images: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None, + prompt: Optional[str | list[str]] = None, + images: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image] = None, height: int = 512, width: int = 512, prior_num_inference_steps: int = 60, prior_guidance_scale: float = 4.0, num_inference_steps: int = 12, decoder_guidance_scale: float = 0.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_pooled: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds_pooled: Optional[torch.Tensor] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prior_callback_on_step_end_tensor_inputs: list[str] = ["latents"], callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation for the prior and decoder. - images (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, *optional*): + images (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, *optional*): The images to guide the image generation for the prior. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): @@ -217,7 +217,7 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `prior_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - prior_num_inference_steps (`Union[int, Dict[float, int]]`, *optional*, defaults to 60): + prior_num_inference_steps (`Union[int, dict[float, int]]`, *optional*, defaults to 60): The number of prior denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. For more specific timestep spacing, you can pass customized `prior_timesteps` @@ -231,7 +231,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -247,7 +247,7 @@ def __call__( A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. - prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + prior_callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -256,7 +256,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py index 9e63b3489ccd..b86a2c0172fe 100644 --- a/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +++ b/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from math import ceil -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import PIL @@ -70,11 +70,11 @@ class StableCascadePriorPipelineOutput(BaseOutput): Text embeddings for the negative prompt. """ - image_embeddings: Union[torch.Tensor, np.ndarray] - prompt_embeds: Union[torch.Tensor, np.ndarray] - prompt_embeds_pooled: Union[torch.Tensor, np.ndarray] - negative_prompt_embeds: Union[torch.Tensor, np.ndarray] - negative_prompt_embeds_pooled: Union[torch.Tensor, np.ndarray] + image_embeddings: torch.Tensor | np.ndarray + prompt_embeds: torch.Tensor | np.ndarray + prompt_embeds_pooled: torch.Tensor | np.ndarray + negative_prompt_embeds: torch.Tensor | np.ndarray + negative_prompt_embeds_pooled: torch.Tensor | np.ndarray class StableCascadePriorPipeline(DiffusionPipeline): @@ -203,7 +203,7 @@ def encode_prompt( prompt_embeds_pooled = prompt_embeds_pooled.repeat_interleave(num_images_per_prompt, dim=0) if negative_prompt_embeds is None and do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -374,32 +374,32 @@ def get_timestep_ratio_conditioning(self, t, alphas_cumprod): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - images: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None, + prompt: Optional[str | list[str]] = None, + images: torch.Tensor | PIL.Image.Image | list[torch.Tensor] | list[PIL.Image.Image] = None, height: int = 1024, width: int = 1024, num_inference_steps: int = 20, - timesteps: List[float] = None, + timesteps: list[float] = None, guidance_scale: float = 4.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, prompt_embeds_pooled: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds_pooled: Optional[torch.Tensor] = None, image_embeds: Optional[torch.Tensor] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pt", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to 1024): The height in pixels of the generated image. @@ -414,7 +414,7 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `decoder_guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): @@ -436,7 +436,7 @@ def __call__( not provided, image embeddings will be generated from `image` input argument if existing. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -453,7 +453,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py index 6c0221d2092a..194d87e5130a 100644 --- a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +++ b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py @@ -17,7 +17,7 @@ import re from contextlib import nullcontext from io import BytesIO -from typing import Dict, Optional, Union +from typing import Optional import requests import torch @@ -1144,7 +1144,7 @@ def convert_controlnet_checkpoint( def download_from_original_stable_diffusion_ckpt( - checkpoint_path_or_dict: Union[str, Dict[str, torch.Tensor]], + checkpoint_path_or_dict: str | dict[str, torch.Tensor], original_config_file: str = None, image_size: Optional[int] = None, prediction_type: str = None, @@ -1237,7 +1237,7 @@ def download_from_original_stable_diffusion_ckpt( [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer) to use. If this parameter is `None`, the function will load a new instance of [CLIPTokenizer] by itself, if needed. - config_files (`Dict[str, str]`, *optional*, defaults to `None`): + config_files (`dict[str, str]`, *optional*, defaults to `None`): A dictionary mapping from config file names to their contents. If this parameter is `None`, the function will load the config files by itself, if needed. Valid keys are: - `v1`: Config file for Stable Diffusion v1 diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py index 6befe77aa4b1..a309eb4a6d14 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py @@ -14,7 +14,7 @@ import warnings from functools import partial -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import jax import jax.numpy as jnp @@ -112,9 +112,7 @@ def __init__( text_encoder: FlaxCLIPTextModel, tokenizer: CLIPTokenizer, unet: FlaxUNet2DConditionModel, - scheduler: Union[ - FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler - ], + scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler, safety_checker: FlaxStableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, dtype: jnp.dtype = jnp.float32, @@ -168,7 +166,7 @@ def __init__( ) self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 - def prepare_inputs(self, prompt: Union[str, List[str]]): + def prepare_inputs(self, prompt: str | list[str]): if not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") @@ -218,7 +216,7 @@ def _run_safety_checker(self, images, safety_model_params, jit=False): def _generate( self, prompt_ids: jnp.array, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int, height: int, @@ -315,12 +313,12 @@ def loop_body(step, args): def __call__( self, prompt_ids: jnp.array, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int = 50, height: Optional[int] = None, width: Optional[int] = None, - guidance_scale: Union[float, jnp.ndarray] = 7.5, + guidance_scale: float | jnp.ndarray = 7.5, latents: jnp.ndarray = None, neg_prompt_ids: jnp.ndarray = None, return_dict: bool = True, @@ -330,7 +328,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py index 81656beba7e1..b5698c32feba 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py @@ -14,7 +14,7 @@ import warnings from functools import partial -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import jax import jax.numpy as jnp @@ -136,9 +136,7 @@ def __init__( text_encoder: FlaxCLIPTextModel, tokenizer: CLIPTokenizer, unet: FlaxUNet2DConditionModel, - scheduler: Union[ - FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler - ], + scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler, safety_checker: FlaxStableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, dtype: jnp.dtype = jnp.float32, @@ -167,7 +165,7 @@ def __init__( ) self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 - def prepare_inputs(self, prompt: Union[str, List[str]], image: Union[Image.Image, List[Image.Image]]): + def prepare_inputs(self, prompt: str | list[str], image: Image.Image | list[Image.Image]): if not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") @@ -234,7 +232,7 @@ def _generate( self, prompt_ids: jnp.ndarray, image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, start_timestep: int, num_inference_steps: int, @@ -339,13 +337,13 @@ def __call__( self, prompt_ids: jnp.ndarray, image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, strength: float = 0.8, num_inference_steps: int = 50, height: Optional[int] = None, width: Optional[int] = None, - guidance_scale: Union[float, jnp.ndarray] = 7.5, + guidance_scale: float | jnp.ndarray = 7.5, noise: jnp.ndarray = None, neg_prompt_ids: jnp.ndarray = None, return_dict: bool = True, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py index 5938fe232a71..f043031b477c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py @@ -14,7 +14,7 @@ import warnings from functools import partial -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import jax import jax.numpy as jnp @@ -135,9 +135,7 @@ def __init__( text_encoder: FlaxCLIPTextModel, tokenizer: CLIPTokenizer, unet: FlaxUNet2DConditionModel, - scheduler: Union[ - FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler - ], + scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler, safety_checker: FlaxStableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, dtype: jnp.dtype = jnp.float32, @@ -193,9 +191,9 @@ def __init__( def prepare_inputs( self, - prompt: Union[str, List[str]], - image: Union[Image.Image, List[Image.Image]], - mask: Union[Image.Image, List[Image.Image]], + prompt: str | list[str], + image: Image.Image | list[Image.Image], + mask: Image.Image | list[Image.Image], ): if not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") @@ -269,7 +267,7 @@ def _generate( prompt_ids: jnp.ndarray, mask: jnp.ndarray, masked_image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int, height: int, @@ -397,12 +395,12 @@ def __call__( prompt_ids: jnp.ndarray, mask: jnp.ndarray, masked_image: jnp.ndarray, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int = 50, height: Optional[int] = None, width: Optional[int] = None, - guidance_scale: Union[float, jnp.ndarray] = 7.5, + guidance_scale: float | jnp.ndarray = 7.5, latents: jnp.ndarray = None, neg_prompt_ids: jnp.ndarray = None, return_dict: bool = True, @@ -412,7 +410,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py index 6ebe0986a1ab..9cea89ceb5fc 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import torch @@ -36,7 +36,7 @@ class OnnxStableDiffusionPipeline(DiffusionPipeline): text_encoder: OnnxRuntimeModel tokenizer: CLIPTokenizer unet: OnnxRuntimeModel - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler safety_checker: OnnxRuntimeModel feature_extractor: CLIPImageProcessor @@ -50,7 +50,7 @@ def __init__( text_encoder: OnnxRuntimeModel, tokenizer: CLIPTokenizer, unet: OnnxRuntimeModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: OnnxRuntimeModel, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = True, @@ -114,7 +114,7 @@ def __init__( def _encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: Optional[int], do_classifier_free_guidance: bool, negative_prompt: Optional[str], @@ -125,13 +125,13 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`np.ndarray`, *optional*): @@ -176,7 +176,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -217,7 +217,7 @@ def _encode_prompt( def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int], width: Optional[int], callback_steps: int, @@ -264,12 +264,12 @@ def check_inputs( def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, generator: Optional[np.random.RandomState] = None, @@ -285,10 +285,10 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - image (`PIL.Image.Image` or List[`PIL.Image.Image`] or `torch.Tensor`): + image (`PIL.Image.Image` or list[`PIL.Image.Image`] or `torch.Tensor`): `Image`, or tensor representing an image batch which will be upscaled. * num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the @@ -299,7 +299,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -470,7 +470,7 @@ def __init__( text_encoder: OnnxRuntimeModel, tokenizer: CLIPTokenizer, unet: OnnxRuntimeModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: OnnxRuntimeModel, feature_extractor: CLIPImageProcessor, ): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py index d63bf3bf4564..d292d2f5e249 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -89,7 +89,7 @@ class OnnxStableDiffusionImg2ImgPipeline(DiffusionPipeline): text_encoder: OnnxRuntimeModel tokenizer: CLIPTokenizer unet: OnnxRuntimeModel - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler safety_checker: OnnxRuntimeModel feature_extractor: CLIPImageProcessor @@ -103,7 +103,7 @@ def __init__( text_encoder: OnnxRuntimeModel, tokenizer: CLIPTokenizer, unet: OnnxRuntimeModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: OnnxRuntimeModel, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = True, @@ -168,7 +168,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt def _encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: Optional[int], do_classifier_free_guidance: bool, negative_prompt: Optional[str], @@ -179,13 +179,13 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`np.ndarray`, *optional*): @@ -230,7 +230,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -271,9 +271,9 @@ def _encode_prompt( def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], callback_steps: int, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[np.ndarray] = None, negative_prompt_embeds: Optional[np.ndarray] = None, ): @@ -313,12 +313,12 @@ def check_inputs( def __call__( self, - prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image] = None, + prompt: str | list[str], + image: np.ndarray | PIL.Image.Image = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, generator: Optional[np.random.RandomState] = None, @@ -333,7 +333,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`np.ndarray` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the @@ -353,7 +353,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py index 158bcabbebfd..27b7f58a267f 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -87,7 +87,7 @@ class OnnxStableDiffusionInpaintPipeline(DiffusionPipeline): text_encoder: OnnxRuntimeModel tokenizer: CLIPTokenizer unet: OnnxRuntimeModel - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler safety_checker: OnnxRuntimeModel feature_extractor: CLIPImageProcessor @@ -101,7 +101,7 @@ def __init__( text_encoder: OnnxRuntimeModel, tokenizer: CLIPTokenizer, unet: OnnxRuntimeModel, - scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + scheduler: DDIMScheduler | PNDMScheduler | LMSDiscreteScheduler, safety_checker: OnnxRuntimeModel, feature_extractor: CLIPImageProcessor, requires_safety_checker: bool = True, @@ -167,7 +167,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt def _encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: Optional[int], do_classifier_free_guidance: bool, negative_prompt: Optional[str], @@ -178,13 +178,13 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`np.ndarray`, *optional*): @@ -229,7 +229,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -271,7 +271,7 @@ def _encode_prompt( # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline.check_inputs def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int], width: Optional[int], callback_steps: int, @@ -319,14 +319,14 @@ def check_inputs( @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], image: PIL.Image.Image, mask_image: PIL.Image.Image, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, generator: Optional[np.random.RandomState] = None, @@ -342,7 +342,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`PIL.Image.Image`): `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will @@ -365,7 +365,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py index a765163175a2..4ae4ccd7758d 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -144,7 +144,7 @@ def __init__( def check_inputs( self, - prompt: Union[str, List[str]], + prompt: str | list[str], image, noise_level, callback_steps, @@ -245,7 +245,7 @@ def decode_latents(self, latents): def _encode_prompt( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: Optional[int], do_classifier_free_guidance: bool, negative_prompt: Optional[str], @@ -256,13 +256,13 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`np.ndarray`, *optional*): @@ -307,7 +307,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -348,15 +348,15 @@ def _encode_prompt( def __call__( self, - prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image, List[PIL.Image.Image]], + prompt: str | list[str], + image: np.ndarray | PIL.Image.Image | list[PIL.Image.Image], num_inference_steps: int = 75, guidance_scale: float = 9.0, noise_level: int = 20, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[np.random.RandomState, List[np.random.RandomState]]] = None, + generator: Optional[np.random.RandomState | list[np.random.RandomState]] = None, latents: Optional[np.ndarray] = None, prompt_embeds: Optional[np.ndarray] = None, negative_prompt_embeds: Optional[np.ndarray] = None, @@ -369,7 +369,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. image (`np.ndarray` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the @@ -385,7 +385,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. noise_level (`float`, defaults to 0.2): Deteremines the amount of noise to add to the initial image before performing upscaling. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_output.py b/src/diffusers/pipelines/stable_diffusion/pipeline_output.py index 5fb9b1a1412d..71031f63d41b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_output.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -13,16 +13,16 @@ class StableDiffusionPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`) - List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + nsfw_content_detected (`list[bool]`) + list indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] if is_flax_available(): @@ -36,10 +36,10 @@ class FlaxStableDiffusionPipelineOutput(BaseOutput): Args: images (`np.ndarray`): Denoised images of array shape of `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`): - List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content + nsfw_content_detected (`list[bool]`): + list indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or `None` if safety checking could not be performed. """ images: np.ndarray - nsfw_content_detected: List[bool] + nsfw_content_detected: list[bool] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index cb97f18efeff..a659a9bb6de7 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from packaging import version @@ -95,9 +95,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -112,15 +112,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -345,7 +345,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -353,7 +353,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -452,7 +452,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -779,38 +779,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -819,18 +819,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -838,7 +838,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -852,7 +852,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -877,7 +877,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index e957c6661f87..3f53c6dc5bf9 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -14,7 +14,7 @@ import contextlib import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -217,7 +217,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -225,7 +225,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -324,7 +324,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -634,33 +634,33 @@ def num_timesteps(self): @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, depth_map: Optional[torch.Tensor] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be used as the starting point. Can accept image latents as `image` only if `depth_map` is not `None`. depth_map (`torch.Tensor`, *optional*): @@ -678,7 +678,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -686,7 +686,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -711,7 +711,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py index d47e2f0593dd..660f96f4ea42 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import PIL.Image import torch @@ -218,7 +218,7 @@ def check_inputs(self, image, height, width, callback_steps): and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -259,14 +259,14 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.Tensor], + image: PIL.Image.Image | list[PIL.Image.Image] | torch.Tensor, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, @@ -277,7 +277,7 @@ def __call__( The call function to the pipeline for generation. Args: - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.Tensor`): Image or images to guide image generation. If you provide a tensor, it needs to be compatible with [`CLIPImageProcessor`](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): @@ -295,7 +295,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 95d3ab06f02a..ea4ad8f5300a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -123,9 +123,9 @@ def preprocess(image): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -140,15 +140,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -371,7 +371,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -379,7 +379,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -478,7 +478,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -859,38 +859,38 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: Optional[float] = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -905,18 +905,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. This parameter is modulated by `strength`. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -924,7 +924,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): @@ -934,7 +934,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -955,7 +955,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 148d7386a732..44d428be21cd 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -69,9 +69,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -86,15 +86,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -173,7 +173,7 @@ class StableDiffusionInpaintPipeline( def __init__( self, - vae: Union[AutoencoderKL, AsymmetricAutoencoderKL], + vae: AutoencoderKL | AsymmetricAutoencoderKL, text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, @@ -325,7 +325,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -333,7 +333,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -432,7 +432,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -880,7 +880,7 @@ def interrupt(self): @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: torch.Tensor = None, @@ -889,42 +889,42 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 1.0, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be inpainted (which parts of the image to be masked out with `mask_image` and repainted according to `prompt`). For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one @@ -951,18 +951,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. This parameter is modulated by `strength`. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -970,7 +970,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -984,7 +984,7 @@ def __call__( Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1005,7 +1005,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py index 843d25d67c10..a61ef462826b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -172,36 +172,36 @@ def __init__( @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, num_inference_steps: int = 100, guidance_scale: float = 7.5, image_guidance_scale: float = 1.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + cross_attention_kwargs: Optional[dict[str, Any]] = None, **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor` `np.ndarray`, `PIL.Image.Image`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be repainted according to `prompt`. Can also accept image latents as `image`, but if passing latents directly it is not encoded again. num_inference_steps (`int`, *optional*, defaults to 100): @@ -215,7 +215,7 @@ def __call__( `image_guidance_scale > 1`. Higher image guidance scale encourages generated images that are closely linked to the source `image`, usually at the expense of lower image quality. This pipeline requires a value of at least `1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -248,7 +248,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -504,7 +504,7 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -512,7 +512,7 @@ def _encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_ prompt (`str` or `List[str]`, *optional*): + negative_ prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -579,7 +579,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py index 66d5ffa6b849..5c8f6c42f8f1 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py @@ -13,7 +13,7 @@ # limitations under the License. import warnings -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -184,7 +184,7 @@ def encode_prompt( torch device do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.FloatTensor`, *optional*): @@ -243,7 +243,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: if negative_prompt_embeds is None or negative_pooled_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -396,12 +396,12 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, num_inference_steps: int = 75, guidance_scale: float = 9.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + negative_prompt: Optional[str | list[str]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -416,9 +416,9 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image upscaling. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be upscaled. If it's a tensor, it can be either a latent output from a Stable Diffusion model or an image tensor in the range `[-1, 1]`. It is considered a `latent` if `image.shape[1]` is `4`; otherwise, it is considered to be an image representation and @@ -429,13 +429,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py index 02f7526b7102..4e3a22cccf0d 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py @@ -14,7 +14,7 @@ import inspect import warnings -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -234,7 +234,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -242,7 +242,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -341,7 +341,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -548,15 +548,15 @@ def upcast_vae(self): @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, image: PipelineImageInput = None, num_inference_steps: int = 75, guidance_scale: float = 9.0, noise_level: int = 20, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -564,16 +564,16 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image` or tensor representing an image batch to be upscaled. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the @@ -581,7 +581,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -589,7 +589,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py index a134244e3ee4..073660dfb77a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer @@ -173,7 +173,7 @@ def _encode_prior_prompt( device, num_images_per_prompt, do_classifier_free_guidance, - text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_model_output: Optional[CLIPTextModelOutput | tuple] = None, text_attention_mask: Optional[torch.Tensor] = None, ): if text_model_output is None: @@ -310,7 +310,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -318,7 +318,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -417,7 +417,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -647,12 +647,12 @@ def noise_image_embeddings( def __call__( self, # regular denoising process args - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 20, guidance_scale: float = 10.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, generator: Optional[torch.Generator] = None, @@ -663,7 +663,7 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, noise_level: int = 0, # prior args prior_num_inference_steps: int = 25, @@ -675,7 +675,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -687,7 +687,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 10.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -695,7 +695,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py index abb4cc3a05d5..d1c5026e318c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import PIL.Image import torch @@ -272,7 +272,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -280,7 +280,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -379,7 +379,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -547,7 +547,7 @@ def check_inputs( and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -625,13 +625,13 @@ def noise_image_embeddings( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image: Union[torch.Tensor, PIL.Image.Image] = None, - prompt: Union[str, List[str]] = None, + image: torch.Tensor | PIL.Image.Image = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 20, guidance_scale: float = 10, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, generator: Optional[torch.Generator] = None, @@ -642,7 +642,7 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, noise_level: int = 0, image_embeds: Optional[torch.Tensor] = None, clip_skip: Optional[int] = None, @@ -651,7 +651,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, either `prompt_embeds` will be used or prompt is initialized to `""`. image (`torch.Tensor` or `PIL.Image.Image`): @@ -668,7 +668,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 10.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -676,7 +676,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py b/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py index 55d5023b6869..a072bc229fe2 100644 --- a/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py +++ b/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Optional import jax import jax.numpy as jnp @@ -76,7 +76,7 @@ class FlaxStableDiffusionSafetyChecker(FlaxPreTrainedModel): def __init__( self, config: CLIPConfig, - input_shape: Optional[Tuple] = None, + input_shape: Optional[tuple] = None, seed: int = 0, dtype: jnp.dtype = jnp.float32, _do_init: bool = True, @@ -87,7 +87,7 @@ def __init__( module = self.module_class(config=config, dtype=dtype, **kwargs) super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) - def init_weights(self, rng: jax.Array, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + def init_weights(self, rng: jax.Array, input_shape: tuple, params: FrozenDict = None) -> FrozenDict: # init input tensor clip_input = jax.random.normal(rng, input_shape) diff --git a/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py b/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py index ffd66792fe46..5d287781f400 100644 --- a/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py +++ b/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Union +from typing import Optional import torch from torch import nn @@ -41,7 +41,7 @@ def __init__( def to( self, - torch_device: Optional[Union[str, torch.device]] = None, + torch_device: Optional[str | torch.device] = None, torch_dtype: Optional[torch.dtype] = None, ): self.mean = nn.Parameter(self.mean.to(torch_device).to(torch_dtype)) diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_output.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_output.py index 4655f446102a..43f801d81e2e 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_output.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,9 +12,9 @@ class StableDiffusion3PipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py index 1618f89a49e3..ca9e4c858f65 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -89,9 +89,9 @@ def calculate_shift( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -106,15 +106,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -232,7 +232,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -288,7 +288,7 @@ def _get_t5_prompt_embeds( def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -343,15 +343,15 @@ def _get_clip_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -363,12 +363,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -377,14 +377,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -773,19 +773,19 @@ def enable_sequential_cpu_offload(self, *args, **kwargs): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 28, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -795,12 +795,12 @@ def __call__( ip_adapter_image_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, - skip_guidance_layers: List[int] = None, + skip_guidance_layers: list[int] = None, skip_layer_guidance_scale: float = 2.8, skip_layer_guidance_stop: float = 0.2, skip_layer_guidance_start: float = 0.01, @@ -810,13 +810,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -826,7 +826,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -836,19 +836,19 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -890,12 +890,12 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. max_sequence_length (`int` defaults to 256): Maximum sequence length to use with the `prompt`. - skip_guidance_layers (`List[int]`, *optional*): + skip_guidance_layers (`list[int]`, *optional*): A list of integers that specify layers to skip during guidance. If not provided, all layers will be used for guidance. If provided, the guidance will only be applied to the layers specified in the list. Recommended value by StabiltyAI for Stable Diffusion 3.5 Medium is [7, 8, 9]. diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py index 7e97909f42ca..d8b285ef2289 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -109,9 +109,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -126,15 +126,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -256,7 +256,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -313,7 +313,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -369,15 +369,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -389,12 +389,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -403,14 +403,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -830,21 +830,21 @@ def enable_sequential_cpu_offload(self, *args, **kwargs): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, image: PipelineImageInput = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -854,10 +854,10 @@ def __call__( ip_adapter_image: Optional[PipelineImageInput] = None, ip_adapter_image_embeds: Optional[torch.Tensor] = None, return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, mu: Optional[float] = None, ): @@ -865,13 +865,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead height (`int`, *optional*, defaults to self.transformer.config.sample_size * self.vae_scale_factor): @@ -881,7 +881,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -891,19 +891,19 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -945,7 +945,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py index 5b2cca0378cd..7d467891993f 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -108,9 +108,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -125,15 +125,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -262,7 +262,7 @@ def __init__( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 256, device: Optional[torch.device] = None, @@ -319,7 +319,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, clip_skip: Optional[int] = None, @@ -375,15 +375,15 @@ def _get_clip_prompt_embeds( # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - prompt_2: Union[str, List[str]], - prompt_3: Union[str, List[str]], + prompt: str | list[str], + prompt_2: str | list[str], + prompt_3: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, do_classifier_free_guidance: bool = True, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, @@ -395,12 +395,12 @@ def encode_prompt( r""" Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in all text-encoders - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is used in all text-encoders device: (`torch.device`): @@ -409,14 +409,14 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders. prompt_embeds (`torch.FloatTensor`, *optional*): @@ -922,9 +922,9 @@ def enable_sequential_cpu_offload(self, *args, **kwargs): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - prompt_3: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, + prompt_3: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: PipelineImageInput = None, @@ -933,13 +933,13 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.6, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 7.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - negative_prompt_3: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, + negative_prompt_3: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -949,10 +949,10 @@ def __call__( ip_adapter_image_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 256, mu: Optional[float] = None, ): @@ -960,29 +960,29 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is will be used instead - prompt_3 (`str` or `List[str]`, *optional*): + prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is will be used instead - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but if passing latents directly it is not encoded again. - mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, 1)`, or `(H, W)`. - mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`): + mask_image_latent (`torch.Tensor`, `list[torch.Tensor]`): `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask latents tensor will be generated by `mask_image`. height (`int`, *optional*, defaults to self.transformer.config.sample_size * self.vae_scale_factor): @@ -1005,7 +1005,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1015,19 +1015,19 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used instead - negative_prompt_3 (`str` or `List[str]`, *optional*): + negative_prompt_3 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `negative_prompt` is used instead num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -1069,7 +1069,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py b/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py index a1ff99b6aa34..9f7fa359ae6f 100644 --- a/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +++ b/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py @@ -14,7 +14,7 @@ import inspect import math -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -111,7 +111,7 @@ def get_average_attention(self): average_attention = self.attention_store return average_attention - def aggregate_attention(self, from_where: List[str]) -> torch.Tensor: + def aggregate_attention(self, from_where: list[str]) -> torch.Tensor: """Aggregates the attention across the different layers and heads at the specified resolution.""" out = [] attention_maps = self.get_average_attention() @@ -309,7 +309,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -317,7 +317,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -416,7 +416,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -619,8 +619,8 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @staticmethod def _compute_max_attention_per_index( attention_maps: torch.Tensor, - indices: List[int], - ) -> List[torch.Tensor]: + indices: list[int], + ) -> list[torch.Tensor]: """Computes the maximum attention value for each of the tokens we wish to alter.""" attention_for_text = attention_maps[:, :, 1:-1] attention_for_text *= 100 @@ -641,7 +641,7 @@ def _compute_max_attention_per_index( def _aggregate_and_get_max_attention_per_token( self, - indices: List[int], + indices: list[int], ): """Aggregates the attention for each token and computes the max activation value for each token to alter.""" attention_maps = self.attention_store.aggregate_attention( @@ -654,7 +654,7 @@ def _aggregate_and_get_max_attention_per_token( return max_attention_per_index @staticmethod - def _compute_loss(max_attention_per_index: List[torch.Tensor]) -> torch.Tensor: + def _compute_loss(max_attention_per_index: list[torch.Tensor]) -> torch.Tensor: """Computes the attend-and-excite loss using the maximum attention value for each token.""" losses = [max(0, 1.0 - curr_max) for curr_max in max_attention_per_index] loss = max(losses) @@ -670,7 +670,7 @@ def _update_latent(latents: torch.Tensor, loss: torch.Tensor, step_size: float) def _perform_iterative_refinement_step( self, latents: torch.Tensor, - indices: List[int], + indices: list[int], loss: torch.Tensor, threshold: float, text_embeddings: torch.Tensor, @@ -740,7 +740,7 @@ def register_attention_control(self): self.unet.set_attn_processor(attn_procs) self.attention_store.num_att_layers = cross_att_count - def get_indices(self, prompt: str) -> Dict[str, int]: + def get_indices(self, prompt: str) -> dict[str, int]: """Utility function to list the indices of the tokens you wish to alte""" ids = self.tokenizer(prompt).input_ids indices = {i: tok for tok, i in zip(self.tokenizer.convert_ids_to_tokens(ids), range(len(ids)))} @@ -750,16 +750,16 @@ def get_indices(self, prompt: str) -> Dict[str, int]: @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]], - token_indices: Union[List[int], List[List[int]]], + prompt: str | list[str], + token_indices: list[int] | list[list[int]], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -767,20 +767,20 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, max_iter_to_alter: int = 25, thresholds: dict = {0: 0.05, 10: 0.5, 20: 0.8}, scale_factor: int = 20, - attn_res: Optional[Tuple[int]] = (16, 16), + attn_res: Optional[tuple[int]] = (16, 16), clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - token_indices (`List[int]`): + token_indices (`list[int]`): The token indices to alter with attend-and-excite. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -792,7 +792,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -800,7 +800,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py index 65c25ffbe492..f66b68531e7d 100644 --- a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +++ b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -63,14 +63,14 @@ class DiffEditInversionPipelineOutput(BaseOutput): Args: latents (`torch.Tensor`) inverted latents tensor - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `num_timesteps * batch_size` or numpy array of shape `(num_timesteps, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `num_timesteps * batch_size` or numpy array of shape `(num_timesteps, batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ latents: torch.Tensor - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray EXAMPLE_DOC_STRING = """ @@ -435,7 +435,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -443,7 +443,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -542,7 +542,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -844,13 +844,13 @@ def get_epsilon(self, model_output: torch.Tensor, sample: torch.Tensor, timestep @replace_example_docstring(EXAMPLE_DOC_STRING) def generate_mask( self, - image: Union[torch.Tensor, PIL.Image.Image] = None, - target_prompt: Optional[Union[str, List[str]]] = None, - target_negative_prompt: Optional[Union[str, List[str]]] = None, + image: torch.Tensor | PIL.Image.Image = None, + target_prompt: Optional[str | list[str]] = None, + target_negative_prompt: Optional[str | list[str]] = None, target_prompt_embeds: Optional[torch.Tensor] = None, target_negative_prompt_embeds: Optional[torch.Tensor] = None, - source_prompt: Optional[Union[str, List[str]]] = None, - source_negative_prompt: Optional[Union[str, List[str]]] = None, + source_prompt: Optional[str | list[str]] = None, + source_negative_prompt: Optional[str | list[str]] = None, source_prompt_embeds: Optional[torch.Tensor] = None, source_negative_prompt_embeds: Optional[torch.Tensor] = None, num_maps_per_mask: Optional[int] = 10, @@ -858,9 +858,9 @@ def generate_mask( mask_thresholding_ratio: Optional[float] = 3.0, num_inference_steps: int = 50, guidance_scale: float = 7.5, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, output_type: Optional[str] = "np", - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, ): r""" Generate a latent mask given a mask prompt, a target prompt, and an image. @@ -868,10 +868,10 @@ def generate_mask( Args: image (`PIL.Image.Image`): `Image` or tensor representing an image batch to be used for computing the mask. - target_prompt (`str` or `List[str]`, *optional*): + target_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide semantic mask generation. If not defined, you need to pass `prompt_embeds`. - target_negative_prompt (`str` or `List[str]`, *optional*): + target_negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). target_prompt_embeds (`torch.Tensor`, *optional*): @@ -880,10 +880,10 @@ def generate_mask( target_negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. - source_prompt (`str` or `List[str]`, *optional*): + source_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide semantic mask generation using DiffEdit. If not defined, you need to pass `source_prompt_embeds` or `source_image` instead. - source_negative_prompt (`str` or `List[str]`, *optional*): + source_negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide semantic mask generation away from using DiffEdit. If not defined, you need to pass `source_negative_prompt_embeds` or `source_image` instead. source_prompt_embeds (`torch.Tensor`, *optional*): @@ -908,7 +908,7 @@ def generate_mask( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): @@ -921,8 +921,8 @@ def generate_mask( Examples: Returns: - `List[PIL.Image.Image]` or `np.array`: - When returning a `List[PIL.Image.Image]`, the list consists of a batch of single-channel binary images + `list[PIL.Image.Image]` or `np.array`: + When returning a `list[PIL.Image.Image]`, the list consists of a batch of single-channel binary images with dimensions `(height // self.vae_scale_factor, width // self.vae_scale_factor)`. If it's `np.array`, the shape is `(batch_size, height // self.vae_scale_factor, width // self.vae_scale_factor)`. @@ -1063,13 +1063,13 @@ def generate_mask( @replace_example_docstring(EXAMPLE_INVERT_DOC_STRING) def invert( self, - prompt: Optional[Union[str, List[str]]] = None, - image: Union[torch.Tensor, PIL.Image.Image] = None, + prompt: Optional[str | list[str]] = None, + image: torch.Tensor | PIL.Image.Image = None, num_inference_steps: int = 50, inpaint_strength: float = 0.8, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + negative_prompt: Optional[str | list[str]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, decode_latents: bool = False, @@ -1077,7 +1077,7 @@ def invert( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, lambda_auto_corr: float = 20.0, lambda_kl: float = 20.0, num_reg_steps: int = 0, @@ -1087,7 +1087,7 @@ def invert( Generate inverted latents given a prompt and image. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. image (`PIL.Image.Image`): `Image` or tensor representing an image batch to produce the inverted latents guided by `prompt`. @@ -1102,7 +1102,7 @@ def invert( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). generator (`torch.Generator`, *optional*): @@ -1301,16 +1301,16 @@ def invert( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - mask_image: Union[torch.Tensor, PIL.Image.Image] = None, - image_latents: Union[torch.Tensor, PIL.Image.Image] = None, + prompt: Optional[str | list[str]] = None, + mask_image: torch.Tensor | PIL.Image.Image = None, + image_latents: torch.Tensor | PIL.Image.Image = None, inpaint_strength: Optional[float] = 0.8, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -1318,14 +1318,14 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: int = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. mask_image (`PIL.Image.Image`): `Image` or tensor representing an image batch to mask the generated image. White pixels in the mask are @@ -1345,7 +1345,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): diff --git a/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py b/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py index 78b026684cfa..bc491755d1b2 100644 --- a/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +++ b/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py @@ -14,7 +14,7 @@ import inspect import warnings -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import PIL.Image import torch @@ -233,7 +233,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -241,7 +241,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -340,7 +340,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -540,19 +540,19 @@ def target_size_center_crop(self, im, new_hw): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, gligen_scheduled_sampling_beta: float = 0.3, - gligen_phrases: List[str] = None, - gligen_boxes: List[List[float]] = None, + gligen_phrases: list[str] = None, + gligen_boxes: list[list[float]] = None, gligen_inpaint_image: Optional[PIL.Image.Image] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -560,14 +560,14 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -579,13 +579,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - gligen_phrases (`List[str]`): + gligen_phrases (`list[str]`): The phrases to guide what to include in each of the regions defined by the corresponding `gligen_boxes`. There should only be one phrase per bounding box. - gligen_boxes (`List[List[float]]`): + gligen_boxes (`list[list[float]]`): The bounding boxes that identify rectangular regions of the image that are going to be filled with the content described by the corresponding `gligen_phrases`. Each rectangular box is defined as a - `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. + `list[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. gligen_inpaint_image (`PIL.Image.Image`, *optional*): The input image, if provided, is inpainted with objects described by the `gligen_boxes` and `gligen_phrases`. Otherwise, it is treated as a generation task on a blank input image. @@ -593,7 +593,7 @@ def __call__( Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://huggingface.co/papers/2301.07093). Scheduled Sampling factor is only varied for scheduled sampling during inference for improved quality and controllability. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -601,7 +601,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py b/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py index 05cbad139d92..bf093e5c3bfc 100644 --- a/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +++ b/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py @@ -14,7 +14,7 @@ import inspect import warnings -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import PIL.Image import torch @@ -264,7 +264,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -272,7 +272,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -371,7 +371,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -715,22 +715,22 @@ def get_cross_attention_kwargs_without_grounded(self, hidden_size, repeat_batch, @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, gligen_scheduled_sampling_beta: float = 0.3, - gligen_phrases: List[str] = None, - gligen_images: List[PIL.Image.Image] = None, - input_phrases_mask: Union[int, List[int]] = None, - input_images_mask: Union[int, List[int]] = None, - gligen_boxes: List[List[float]] = None, + gligen_phrases: list[str] = None, + gligen_images: list[PIL.Image.Image] = None, + input_phrases_mask: int | list[int] = None, + input_images_mask: int | list[int] = None, + gligen_boxes: list[list[float]] = None, gligen_inpaint_image: Optional[PIL.Image.Image] = None, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -738,7 +738,7 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, gligen_normalize_constant: float = 28.7, clip_skip: int = None, ): @@ -746,7 +746,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -758,20 +758,20 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - gligen_phrases (`List[str]`): + gligen_phrases (`list[str]`): The phrases to guide what to include in each of the regions defined by the corresponding `gligen_boxes`. There should only be one phrase per bounding box. - gligen_images (`List[PIL.Image.Image]`): + gligen_images (`list[PIL.Image.Image]`): The images to guide what to include in each of the regions defined by the corresponding `gligen_boxes`. There should only be one image per bounding box - input_phrases_mask (`int` or `List[int]`): + input_phrases_mask (`int` or `list[int]`): pre phrases mask input defined by the correspongding `input_phrases_mask` - input_images_mask (`int` or `List[int]`): + input_images_mask (`int` or `list[int]`): pre images mask input defined by the correspongding `input_images_mask` - gligen_boxes (`List[List[float]]`): + gligen_boxes (`list[list[float]]`): The bounding boxes that identify rectangular regions of the image that are going to be filled with the content described by the corresponding `gligen_phrases`. Each rectangular box is defined as a - `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. + `list[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. gligen_inpaint_image (`PIL.Image.Image`, *optional*): The input image, if provided, is inpainted with objects described by the `gligen_boxes` and `gligen_phrases`. Otherwise, it is treated as a generation task on a blank input image. @@ -779,7 +779,7 @@ def __call__( Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://huggingface.co/papers/2301.07093). Scheduled Sampling factor is only varied for scheduled sampling during inference for improved quality and controllability. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -787,7 +787,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py b/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py index feebd6adf8f8..0638adb99276 100755 --- a/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py @@ -14,7 +14,7 @@ import importlib import inspect -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import torch from k_diffusion.external import CompVisDenoiser, CompVisVDenoiser @@ -115,7 +115,7 @@ def __init__( self, vae: AutoencoderKL, text_encoder: CLIPTextModel, - tokenizer: Union[CLIPTokenizer, CLIPTokenizerFast], + tokenizer: CLIPTokenizer | CLIPTokenizerFast, unet: UNet2DConditionModel, scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, @@ -215,7 +215,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -223,7 +223,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -322,7 +322,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -480,15 +480,15 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -504,7 +504,7 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -520,7 +520,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). diff --git a/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py b/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py index 766ca37d8142..9842c69e2874 100644 --- a/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py @@ -14,7 +14,7 @@ import importlib import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import torch from k_diffusion.external import CompVisDenoiser, CompVisVDenoiser @@ -223,9 +223,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -234,11 +234,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -356,7 +356,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -581,16 +581,16 @@ def do_classifier_free_guidance(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -598,12 +598,12 @@ def __call__( negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, use_karras_sigmas: Optional[bool] = False, noise_sampler_seed: Optional[int] = None, clip_skip: Optional[int] = None, @@ -612,10 +612,10 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -637,16 +637,16 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -673,31 +673,31 @@ def __call__( return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead of a plain tuple. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more diff --git a/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py b/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py index c32121c88c9b..51ccb9fcf8c7 100644 --- a/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +++ b/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -99,9 +99,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -116,15 +116,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -161,20 +161,20 @@ class LDM3DPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - rgb (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + rgb (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - depth (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + depth (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`List[bool]`) - List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + nsfw_content_detected (`list[bool]`) + list indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or `None` if safety checking could not be performed. """ - rgb: Union[List[PIL.Image.Image], np.ndarray] - depth: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + rgb: list[PIL.Image.Image] | np.ndarray + depth: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] class StableDiffusionLDM3DPipeline( @@ -320,7 +320,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -328,7 +328,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -427,7 +427,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -748,36 +748,36 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 49, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -786,18 +786,18 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. guidance_scale (`float`, *optional*, defaults to 5.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -805,7 +805,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -820,7 +820,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -841,7 +841,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py b/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py index 295095947a12..74fa559256c6 100644 --- a/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +++ b/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py @@ -13,7 +13,7 @@ import copy import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection @@ -99,9 +99,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -116,15 +116,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -296,7 +296,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -304,7 +304,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -403,7 +403,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -735,7 +735,7 @@ def get_views( window_size: int = 64, stride: int = 8, circular_padding: bool = False, - ) -> List[Tuple[int, int, int, int]]: + ) -> list[tuple[int, int, int, int]]: """ Generates a list of views based on the given parameters. Here, we define the mappings F_i (see Eq. 7 in the MultiDiffusion paper https://huggingface.co/papers/2302.08113). If panorama's height/width < window_size, @@ -749,7 +749,7 @@ def get_views( circular_padding (bool, optional): Whether to apply circular padding. Defaults to False. Returns: - List[Tuple[int, int, int, int]]: A list of tuples representing the views. Each tuple contains four integers + list[tuple[int, int, int, int]]: A list of tuples representing the views. Each tuple contains four integers representing the start and end coordinates of the window in the panorama. """ @@ -802,37 +802,37 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = 512, width: Optional[int] = 2048, num_inference_steps: int = 50, - timesteps: List[int] = None, + timesteps: list[int] = None, guidance_scale: float = 7.5, view_batch_size: int = 1, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, circular_padding: bool = False, clip_skip: Optional[int] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs: Any, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -842,7 +842,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): The timesteps at which to generate the images. If not specified, then the default timestep spacing strategy of the scheduler is used. guidance_scale (`float`, *optional*, defaults to 7.5): @@ -851,7 +851,7 @@ def __call__( view_batch_size (`int`, *optional*, defaults to 1): The batch size to denoise split views. For some GPUs with high performance, higher view batch size can speedup the generation and increase the VRAM usage. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -859,7 +859,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -874,7 +874,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -902,7 +902,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List[str]`, *optional*): + callback_on_step_end_tensor_inputs (`list[str]`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_safe/__init__.py b/src/diffusers/pipelines/stable_diffusion_safe/__init__.py index b432b9418c46..69394ca22891 100644 --- a/src/diffusers/pipelines/stable_diffusion_safe/__init__.py +++ b/src/diffusers/pipelines/stable_diffusion_safe/__init__.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np import PIL diff --git a/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py b/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py index 69a064d6638d..042aaad9571a 100644 --- a/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py +++ b/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -15,20 +15,20 @@ class StableDiffusionSafePipelineOutput(BaseOutput): Output class for Safe Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. - nsfw_content_detected (`List[bool]`) - List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + nsfw_content_detected (`list[bool]`) + list of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) content, or `None` if safety checking could not be performed. - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images that were flagged by the safety checker any may contain "not-safe-for-work" + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images that were flagged by the safety checker any may contain "not-safe-for-work" (nsfw) content, or `None` if no safety check was performed or no images were flagged. applied_safety_concept (`str`) The safety concept that was applied for safety guidance, or `None` if safety guidance was disabled """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] - unsafe_images: Optional[Union[List[PIL.Image.Image], np.ndarray]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] + unsafe_images: Optional[list[PIL.Image.Image] | np.ndarray] applied_safety_concept: Optional[str] diff --git a/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py b/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py index d334107b0703..37a848fdd472 100644 --- a/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +++ b/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py @@ -1,6 +1,6 @@ import inspect import warnings -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import torch @@ -199,7 +199,7 @@ def _encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): prompt to be encoded device: (`torch.device`): torch device @@ -207,7 +207,7 @@ def _encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`): + negative_prompt (`str` or `list[str]`): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). """ @@ -248,7 +248,7 @@ def _encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -520,15 +520,15 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, output_type: Optional[str] = "pil", @@ -545,7 +545,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -557,7 +557,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -565,7 +565,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py b/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py index 48add535a81d..c9ae1c019cbd 100644 --- a/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +++ b/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import torch import torch.nn.functional as F @@ -224,7 +224,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -232,7 +232,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -331,7 +331,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -574,33 +574,33 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, sag_scale: float = 0.75, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: Optional[int] = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated image. @@ -614,7 +614,7 @@ def __call__( `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. sag_scale (`float`, *optional*, defaults to 0.75): Chosen between [0, 1.0] for better quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -622,7 +622,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -637,7 +637,7 @@ def __call__( not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. If not provided, embeddings are computed from the `ip_adapter_image` input argument. output_type (`str`, *optional*, defaults to `"pil"`): diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py index 3227fd9a08a4..d131aefa9f70 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py @@ -13,7 +13,7 @@ # limitations under the License. from functools import partial -from typing import Dict, List, Optional, Union +from typing import Dict, Optional import jax import jax.numpy as jnp @@ -48,9 +48,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: FlaxUNet2DConditionModel, - scheduler: Union[ - FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler - ], + scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler, dtype: jnp.dtype = jnp.float32, ): super().__init__() @@ -67,7 +65,7 @@ def __init__( ) self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 - def prepare_inputs(self, prompt: Union[str, List[str]]): + def prepare_inputs(self, prompt: str | list[str]): if not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") @@ -88,10 +86,10 @@ def prepare_inputs(self, prompt: Union[str, List[str]]): def __call__( self, prompt_ids: jax.Array, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int = 50, - guidance_scale: Union[float, jax.Array] = 7.5, + guidance_scale: float | jax.Array = 7.5, height: Optional[int] = None, width: Optional[int] = None, latents: jnp.array = None, @@ -169,7 +167,7 @@ def _get_add_time_ids(self, original_size, crops_coords_top_left, target_size, b def _generate( self, prompt_ids: jnp.array, - params: Union[Dict, FrozenDict], + params: Dict | FrozenDict, prng_seed: jax.Array, num_inference_steps: int, height: int, diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py index 0783f44486ee..6617e7cd058c 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL.Image @@ -13,12 +12,12 @@ class StableDiffusionXLPipelineOutput(BaseOutput): Output class for Stable Diffusion pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray if is_flax_available(): diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py index b97cf6f1f6f8..8acacae73a6d 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import torch from transformers import ( @@ -116,9 +116,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -133,15 +133,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -305,9 +305,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -316,11 +316,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -438,7 +438,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -837,52 +837,52 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): @@ -898,11 +898,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -919,11 +919,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -931,7 +931,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -953,7 +953,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -974,31 +974,31 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1008,7 +1008,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index 44e8f4fe4b54..b12464d11a74 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import torch @@ -133,9 +133,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -150,15 +150,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -322,9 +322,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -333,11 +333,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -455,7 +455,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -987,58 +987,58 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, strength: float = 0.3, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`): The image(s) to modify with the pipeline. strength (`float`, *optional*, defaults to 0.3): Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` @@ -1050,11 +1050,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1079,11 +1079,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -1091,7 +1091,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -1113,7 +1113,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1134,31 +1134,31 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1179,7 +1179,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 88cc7515b051..e72f42766963 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import numpy as np import PIL.Image @@ -160,9 +160,9 @@ def retrieve_latents( def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -177,15 +177,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -426,9 +426,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -437,11 +437,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -559,7 +559,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -1092,8 +1092,8 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, mask_image: PipelineImageInput = None, masked_image_latents: torch.Tensor = None, @@ -1102,50 +1102,50 @@ def __call__( padding_mask_crop: Optional[int] = None, strength: float = 0.9999, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_start: Optional[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, aesthetic_score: float = 6.0, negative_aesthetic_score: float = 2.5, clip_skip: Optional[int] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders image (`PIL.Image.Image`): @@ -1184,11 +1184,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -1213,11 +1213,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -1235,7 +1235,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1262,31 +1262,31 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more @@ -1307,7 +1307,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py index 58b008361782..3158171c9e9e 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import PIL.Image import torch @@ -234,9 +234,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -245,11 +245,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -355,7 +355,7 @@ def encode_prompt( negative_prompt = negative_prompt or "" negative_prompt_2 = negative_prompt_2 or negative_prompt - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -611,8 +611,8 @@ def upcast_vae(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, @@ -620,11 +620,11 @@ def __call__( denoising_end: Optional[float] = None, guidance_scale: float = 5.0, image_guidance_scale: float = 1.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -634,23 +634,23 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Tuple[int, int] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Tuple[int, int] = None, + original_size: tuple[int, int] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: tuple[int, int] = None, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`): The image(s) to modify with the pipeline. height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): The height in pixels of the generated image. @@ -677,11 +677,11 @@ def __call__( scale is enabled by setting `image_guidance_scale > 1`. Higher image guidance scale encourages to generate images that are closely linked to the source image `image`, usually at the expense of lower image quality. This pipeline requires a value of at least `1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -689,7 +689,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -732,17 +732,17 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). diff --git a/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py b/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py index 6d9053faaec8..517d019f7590 100644 --- a/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +++ b/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import PIL.Image @@ -74,9 +74,9 @@ def _append_dims(x, target_dims): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -91,15 +91,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -136,12 +136,12 @@ class StableVideoDiffusionPipelineOutput(BaseOutput): Output class for Stable Video Diffusion pipeline. Args: - frames (`[List[List[PIL.Image.Image]]`, `np.ndarray`, `torch.Tensor`]): - List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + frames (`[list[list[PIL.Image.Image]]`, `np.ndarray`, `torch.Tensor`]): + list of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, num_frames, height, width, num_channels)`. """ - frames: Union[List[List[PIL.Image.Image]], np.ndarray, torch.Tensor] + frames: list[list[PIL.Image.Image]] | np.ndarray | torch.Tensor class StableVideoDiffusionPipeline(DiffusionPipeline): @@ -191,7 +191,7 @@ def __init__( def _encode_image( self, image: PipelineImageInput, - device: Union[str, torch.device], + device: str | torch.device, num_videos_per_prompt: int, do_classifier_free_guidance: bool, ) -> torch.Tensor: @@ -239,7 +239,7 @@ def _encode_image( def _encode_vae_image( self, image: torch.Tensor, - device: Union[str, torch.device], + device: str | torch.device, num_videos_per_prompt: int, do_classifier_free_guidance: bool, ): @@ -323,7 +323,7 @@ def check_inputs(self, image, height, width): and not isinstance(image, list) ): raise ValueError( - "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is" f" {type(image)}" ) @@ -338,7 +338,7 @@ def prepare_latents( height: int, width: int, dtype: torch.dtype, - device: Union[str, torch.device], + device: str | torch.device, generator: torch.Generator, latents: Optional[torch.Tensor] = None, ): @@ -385,12 +385,12 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.Tensor], + image: PIL.Image.Image | list[PIL.Image.Image] | torch.Tensor, height: int = 576, width: int = 1024, num_frames: Optional[int] = None, num_inference_steps: int = 25, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, min_guidance_scale: float = 1.0, max_guidance_scale: float = 3.0, fps: int = 7, @@ -398,18 +398,18 @@ def __call__( noise_aug_strength: float = 0.02, decode_chunk_size: Optional[int] = None, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], return_dict: bool = True, ): r""" The call function to the pipeline for generation. Args: - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.Tensor`): Image(s) to guide image generation. If you provide a tensor, the expected value range is between `[0, 1]`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): @@ -422,7 +422,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference. This parameter is modulated by `strength`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -445,7 +445,7 @@ def __call__( For lower memory usage, reduce `decode_chunk_size`. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -460,7 +460,7 @@ def __call__( `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -473,7 +473,7 @@ def __call__( Returns: [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] is - returned, otherwise a `tuple` of (`List[List[PIL.Image.Image]]` or `np.ndarray` or `torch.Tensor`) is + returned, otherwise a `tuple` of (`list[list[PIL.Image.Image]]` or `np.ndarray` or `torch.Tensor`) is returned. """ # 0. Default height and width to unet @@ -691,7 +691,7 @@ def _filter2d(input, kernel): height, width = tmp_kernel.shape[-2:] - padding_shape: List[int] = _compute_padding([height, width]) + padding_shape: list[int] = _compute_padding([height, width]) input = torch.nn.functional.pad(input, padding_shape, mode="reflect") # kernel and input tensor reshape to align element-wise or batch-wise params diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index 1ce6987114a7..9153269cbaeb 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -14,7 +14,7 @@ import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -54,16 +54,16 @@ class StableDiffusionAdapterPipelineOutput(BaseOutput): """ Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. - nsfw_content_detected (`List[bool]`) - List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + nsfw_content_detected (`list[bool]`) + list of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) content, or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -132,9 +132,9 @@ def _preprocess_adapter_image(image, height, width): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -149,15 +149,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -197,11 +197,11 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline, StableDiffusionMixin, Fr library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) Args: - adapter ([`T2IAdapter`] or [`MultiAdapter`] or `List[T2IAdapter]`): + adapter ([`T2IAdapter`] or [`MultiAdapter`] or `list[T2IAdapter]`): Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a list, the outputs from each Adapter are added together to create one combined additional conditioning. - adapter_weights (`List[float]`, *optional*, defaults to None): - List of floats representing the weight which will be multiply to each adapter's output before adding them + adapter_weights (`list[float]`, *optional*, defaults to None): + list of floats representing the weight which will be multiply to each adapter's output before adding them together. vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. @@ -233,7 +233,7 @@ def __init__( text_encoder: CLIPTextModel, tokenizer: CLIPTokenizer, unet: UNet2DConditionModel, - adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]], + adapter: T2IAdapter | MultiAdapter | list[T2IAdapter], scheduler: KarrasDiffusionSchedulers, safety_checker: StableDiffusionSafetyChecker, feature_extractor: CLIPImageProcessor, @@ -324,7 +324,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -332,7 +332,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -431,7 +431,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -690,18 +690,18 @@ def do_classifier_free_guidance(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - image: Union[torch.Tensor, PIL.Image.Image, List[PIL.Image.Image]] = None, + prompt: str | list[str] = None, + image: torch.Tensor | PIL.Image.Image | list[PIL.Image.Image] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -709,18 +709,18 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - adapter_conditioning_scale: Union[float, List[float]] = 1.0, + cross_attention_kwargs: Optional[dict[str, Any]] = None, + adapter_conditioning_scale: float | list[float] = 1.0, clip_skip: Optional[int] = None, ): r""" Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[list[PIL.Image.Image]]`): The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the type is specified as `torch.Tensor`, it is passed to Adapter as is. PIL.Image.Image` can also be accepted as an image. The control image is automatically resized to fit the output image. @@ -731,11 +731,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -745,7 +745,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -754,7 +754,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -784,7 +784,7 @@ def __call__( A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - adapter_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + adapter_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the residual in the original unet. If multiple adapters are specified in init, you can set the corresponding scale as a list. diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index 2802d690f3cc..c2fe8e414ae5 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import numpy as np import PIL.Image @@ -157,9 +157,9 @@ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -174,15 +174,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -236,11 +236,11 @@ class StableDiffusionXLAdapterPipeline( - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: - adapter ([`T2IAdapter`] or [`MultiAdapter`] or `List[T2IAdapter]`): + adapter ([`T2IAdapter`] or [`MultiAdapter`] or `list[T2IAdapter]`): Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a list, the outputs from each Adapter are added together to create one combined additional conditioning. - adapter_weights (`List[float]`, *optional*, defaults to None): - List of floats representing the weight which will be multiply to each adapter's output before adding them + adapter_weights (`list[float]`, *optional*, defaults to None): + list of floats representing the weight which will be multiply to each adapter's output before adding them together. vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. @@ -281,7 +281,7 @@ def __init__( tokenizer: CLIPTokenizer, tokenizer_2: CLIPTokenizer, unet: UNet2DConditionModel, - adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]], + adapter: T2IAdapter | MultiAdapter | list[T2IAdapter], scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, feature_extractor: CLIPImageProcessor = None, @@ -331,9 +331,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -342,11 +342,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -464,7 +464,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -869,41 +869,41 @@ def do_classifier_free_guidance(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str] = None, + prompt_2: Optional[str | list[str]] = None, image: PipelineImageInput = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, - timesteps: List[int] = None, - sigmas: List[float] = None, + timesteps: list[int] = None, + sigmas: list[float] = None, denoising_end: Optional[float] = None, guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, ip_adapter_image: Optional[PipelineImageInput] = None, - ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + ip_adapter_image_embeds: Optional[list[torch.Tensor]] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, - adapter_conditioning_scale: Union[float, List[float]] = 1.0, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, + negative_original_size: Optional[tuple[int, int]] = None, + negative_crops_coords_top_left: tuple[int, int] = (0, 0), + negative_target_size: Optional[tuple[int, int]] = None, + adapter_conditioning_scale: float | list[float] = 1.0, adapter_conditioning_factor: float = 1.0, clip_skip: Optional[int] = None, ): @@ -911,13 +911,13 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders - image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]` or `List[PIL.Image.Image]` or `List[List[PIL.Image.Image]]`): + image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[list[PIL.Image.Image]]`): The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the type is specified as `torch.Tensor`, it is passed to Adapter as is. PIL.Image.Image` can also be accepted as an image. The control image is automatically resized to fit the output image. @@ -932,11 +932,11 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -953,11 +953,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -965,7 +965,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -987,7 +987,7 @@ def __call__( weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not @@ -1014,37 +1014,37 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a specific image resolution. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): To negatively condition the generation process based on a target image resolution. It should be as same as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - adapter_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + adapter_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0): The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the residual in the original unet. If multiple adapters are specified in init, you can set the corresponding scale as a list. diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py index 040bf0efba84..c94c5d2d144a 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import List, Union import numpy as np import PIL @@ -16,11 +15,11 @@ class TextToVideoSDPipelineOutput(BaseOutput): Output class for text-to-video pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)` """ - frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + frames: torch.Tensor | np.ndarray | list[list[PIL.Image.Image]] diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py index 3ce7b4d1990f..c6fbcf2d66dc 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import torch from transformers import CLIPTextModel, CLIPTokenizer @@ -172,7 +172,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -180,7 +180,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -279,7 +279,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -450,15 +450,15 @@ def prepare_latents( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: int = 16, num_inference_steps: int = 50, guidance_scale: float = 9.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -466,14 +466,14 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. @@ -488,7 +488,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -496,7 +496,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py index 9d0b7e3dbc32..30b5a6296186 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional import numpy as np import torch @@ -207,7 +207,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -215,7 +215,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -314,7 +314,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -515,14 +515,14 @@ def prepare_latents(self, video, timestep, batch_size, dtype, device, generator= @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - video: Union[List[np.ndarray], torch.Tensor] = None, + prompt: str | list[str] = None, + video: list[np.ndarray] | torch.Tensor = None, strength: float = 0.6, num_inference_steps: int = 50, guidance_scale: float = 15.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -530,16 +530,16 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, clip_skip: Optional[int] = None, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. - video (`List[np.ndarray]` or `torch.Tensor`): + video (`list[np.ndarray]` or `torch.Tensor`): `video` frames or tensor representing a video batch to be used as the starting point for the process. Can also accept video latents as `image`, if passing latents directly, it will not be encoded again. strength (`float`, *optional*, defaults to 0.8): @@ -554,13 +554,13 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in video generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py index 96316f8e91e5..613f0d5d41d6 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py @@ -1,7 +1,7 @@ import copy import inspect from dataclasses import dataclass -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -199,16 +199,16 @@ class TextToVideoPipelineOutput(BaseOutput): Output class for zero-shot text-to-video pipeline. Args: - images (`[List[PIL.Image.Image]`, `np.ndarray`]): - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`[list[PIL.Image.Image]`, `np.ndarray`]): + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - nsfw_content_detected (`[List[bool]]`): - List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + nsfw_content_detected (`[list[bool]]`): + list indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or `None` if safety checking could not be performed. """ - images: Union[List[PIL.Image.Image], np.ndarray] - nsfw_content_detected: Optional[List[bool]] + images: list[PIL.Image.Image] | np.ndarray + nsfw_content_detected: Optional[list[bool]] def coords_grid(batch, ht, wd, device): @@ -374,7 +374,7 @@ def forward_loop(self, x_t0, t0, t1, generator): Timestep at t0. t1: Timestamp at t1. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. @@ -545,16 +545,16 @@ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], + prompt: str | list[str], video_length: Optional[int] = 8, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, motion_field_strength_x: float = 12, motion_field_strength_y: float = 12, @@ -564,13 +564,13 @@ def __call__( callback_steps: Optional[int] = 1, t0: int = 44, t1: int = 47, - frame_ids: Optional[List[int]] = None, + frame_ids: Optional[list[int]] = None, ): """ The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. video_length (`int`, *optional*, defaults to 8): The number of generated video frames. @@ -584,7 +584,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in video generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -592,7 +592,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -623,7 +623,7 @@ def __call__( t1 (`int`, *optional*, defaults to 47): Timestep t0. Should be in the range [t0 + 1, num_inference_steps - 1]. See the [paper](https://huggingface.co/papers/2303.13439), Sect. 3.3.1. - frame_ids (`List[int]`, *optional*): + frame_ids (`list[int]`, *optional*): Indexes of the frames that are being generated. This is used when generating longer videos chunk-by-chunk. @@ -830,7 +830,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -838,7 +838,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -937,7 +937,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py index 288aae6c0d44..eb490c4dcd0b 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py @@ -1,7 +1,7 @@ import copy import inspect from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Optional import numpy as np import PIL @@ -222,12 +222,12 @@ class TextToVideoSDXLPipelineOutput(BaseOutput): Output class for zero-shot text-to-video pipeline. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. """ - images: Union[List[PIL.Image.Image], np.ndarray] + images: list[PIL.Image.Image] | np.ndarray # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.coords_grid @@ -616,9 +616,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders device: (`torch.device`): @@ -627,11 +627,11 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders prompt_embeds (`torch.Tensor`, *optional*): @@ -749,7 +749,7 @@ def encode_prompt( batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 ) - uncond_tokens: List[str] + uncond_tokens: list[str] if prompt is not None and type(prompt) is not type(negative_prompt): raise TypeError( f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" @@ -846,7 +846,7 @@ def forward_loop(self, x_t0, t0, t1, generator): Timestep at t0. t1: Timestamp at t1. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. @@ -951,20 +951,20 @@ def backward_loop( @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]], - prompt_2: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + prompt_2: Optional[str | list[str]] = None, video_length: Optional[int] = 8, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, denoising_end: Optional[float] = None, guidance_scale: float = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, + negative_prompt_2: Optional[str | list[str]] = None, num_videos_per_prompt: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, - frame_ids: Optional[List[int]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, + frame_ids: Optional[list[int]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, pooled_prompt_embeds: Optional[torch.Tensor] = None, @@ -976,11 +976,11 @@ def __call__( return_dict: bool = True, callback: Optional[Callable[[int, int, torch.Tensor], None]] = None, callback_steps: int = 1, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, + cross_attention_kwargs: Optional[dict[str, Any]] = None, guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, + original_size: Optional[tuple[int, int]] = None, + crops_coords_top_left: tuple[int, int] = (0, 0), + target_size: Optional[tuple[int, int]] = None, t0: int = 44, t1: int = 47, ): @@ -988,10 +988,10 @@ def __call__( Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - prompt_2 (`str` or `List[str]`, *optional*): + prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is used in both text-encoders video_length (`int`, *optional*, defaults to 8): @@ -1016,11 +1016,11 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): + negative_prompt_2 (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders num_videos_per_prompt (`int`, *optional*, defaults to 1): @@ -1028,10 +1028,10 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. - frame_ids (`List[int]`, *optional*): + frame_ids (`list[int]`, *optional*): Indexes of the frames that are being generated. This is used when generating longer videos chunk-by-chunk. prompt_embeds (`torch.Tensor`, *optional*): @@ -1080,17 +1080,17 @@ def __call__( [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)): `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)): For most cases, `target_size` should be set to the desired height and width of the generated image. If not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). diff --git a/src/diffusers/pipelines/transformers_loading_utils.py b/src/diffusers/pipelines/transformers_loading_utils.py index b52d154d6ba2..c750292f599e 100644 --- a/src/diffusers/pipelines/transformers_loading_utils.py +++ b/src/diffusers/pipelines/transformers_loading_utils.py @@ -15,7 +15,7 @@ import contextlib import os import tempfile -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from huggingface_hub import DDUFEntry from tqdm import tqdm @@ -34,7 +34,7 @@ def _load_tokenizer_from_dduf( - cls: "PreTrainedTokenizer", name: str, dduf_entries: Dict[str, DDUFEntry], **kwargs + cls: "PreTrainedTokenizer", name: str, dduf_entries: dict[str, DDUFEntry], **kwargs ) -> "PreTrainedTokenizer": """ Load a tokenizer from a DDUF archive. @@ -57,7 +57,7 @@ def _load_tokenizer_from_dduf( def _load_transformers_model_from_dduf( - cls: "PreTrainedModel", name: str, dduf_entries: Dict[str, DDUFEntry], **kwargs + cls: "PreTrainedModel", name: str, dduf_entries: dict[str, DDUFEntry], **kwargs ) -> "PreTrainedModel": """ Load a transformers model from a DDUF archive. diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip.py b/src/diffusers/pipelines/unclip/pipeline_unclip.py index bbb9b0eb3ab2..42270f877431 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Tuple, Union +from typing import Optional import torch from torch.nn import functional as F @@ -131,7 +131,7 @@ def _encode_prompt( device, num_images_per_prompt, do_classifier_free_guidance, - text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_model_output: Optional[CLIPTextModelOutput | tuple] = None, text_attention_mask: Optional[torch.Tensor] = None, ): if text_model_output is None: @@ -219,16 +219,16 @@ def _encode_prompt( @torch.no_grad() def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, prior_num_inference_steps: int = 25, decoder_num_inference_steps: int = 25, super_res_num_inference_steps: int = 7, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, prior_latents: Optional[torch.Tensor] = None, decoder_latents: Optional[torch.Tensor] = None, super_res_latents: Optional[torch.Tensor] = None, - text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_model_output: Optional[CLIPTextModelOutput | tuple] = None, text_attention_mask: Optional[torch.Tensor] = None, prior_guidance_scale: float = 4.0, decoder_guidance_scale: float = 8.0, @@ -239,7 +239,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide image generation. This can only be left undefined if `text_model_output` and `text_attention_mask` is passed. num_images_per_prompt (`int`, *optional*, defaults to 1): @@ -253,7 +253,7 @@ def __call__( super_res_num_inference_steps (`int`, *optional*, defaults to 7): The number of denoising steps for super resolution. More denoising steps usually lead to a higher quality image at the expense of slower inference. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prior_latents (`torch.Tensor` of shape (batch size, embeddings dimension), *optional*): diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py index 31710a000e0a..39ac9df2cd30 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py @@ -13,7 +13,7 @@ # limitations under the License. import inspect -from typing import List, Optional, Union +from typing import Optional import PIL.Image import torch @@ -207,7 +207,7 @@ def _encode_image(self, image, device, num_images_per_prompt, image_embeddings: @torch.no_grad() def __call__( self, - image: Optional[Union[PIL.Image.Image, List[PIL.Image.Image], torch.Tensor]] = None, + image: Optional[PIL.Image.Image | list[PIL.Image.Image] | torch.Tensor] = None, num_images_per_prompt: int = 1, decoder_num_inference_steps: int = 25, super_res_num_inference_steps: int = 7, @@ -223,7 +223,7 @@ def __call__( The call function to the pipeline for generation. Args: - image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.Tensor`): + image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.Tensor`): `Image` or tensor representing an image batch to be used as the starting point. If you provide a tensor, it needs to be compatible with the [`CLIPImageProcessor`] [configuration](https://huggingface.co/fusing/karlo-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). diff --git a/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py b/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py index 0ddcbf735770..0d101adc0779 100644 --- a/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py +++ b/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py @@ -180,7 +180,7 @@ def generate_captions(self, features, eos_token_id, device): Device to perform text generation on. Returns: - `List[str]`: A list of strings generated from the decoder model. + `list[str]`: A list of strings generated from the decoder model. """ features = torch.split(features, 1, dim=0) @@ -233,7 +233,7 @@ def generate_beam( The temperature to use when performing the softmax over logits from the decoding model. Returns: - `Tuple(torch.Tensor, torch.Tensor)`: A tuple of tensors where the first element is a tensor of generated + `tuple(torch.Tensor, torch.Tensor)`: A tuple of tensors where the first element is a tensor of generated token sequences sorted by score in descending order, and the second element is the sequence lengths corresponding to those sequences. """ diff --git a/src/diffusers/pipelines/unidiffuser/modeling_uvit.py b/src/diffusers/pipelines/unidiffuser/modeling_uvit.py index 2a04ec2e4030..45837853fccb 100644 --- a/src/diffusers/pipelines/unidiffuser/modeling_uvit.py +++ b/src/diffusers/pipelines/unidiffuser/modeling_uvit.py @@ -1,5 +1,5 @@ import math -from typing import Optional, Union +from typing import Optional import torch from torch import nn @@ -1040,9 +1040,9 @@ def forward( latent_image_embeds: torch.Tensor, image_embeds: torch.Tensor, prompt_embeds: torch.Tensor, - timestep_img: Union[torch.Tensor, float, int], - timestep_text: Union[torch.Tensor, float, int], - data_type: Optional[Union[torch.Tensor, float, int]] = 1, + timestep_img: torch.Tensor | float | int, + timestep_text: torch.Tensor | float | int, + data_type: Optional[torch.Tensor | float | int] = 1, encoder_hidden_states=None, cross_attention_kwargs=None, ): diff --git a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py index f9298d5b86f8..5edf69e645b9 100644 --- a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +++ b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py @@ -1,6 +1,6 @@ import inspect from dataclasses import dataclass -from typing import Callable, List, Optional, Union +from typing import Callable, Optional import numpy as np import PIL.Image @@ -50,16 +50,16 @@ class ImageTextPipelineOutput(BaseOutput): Output class for joint image-text pipelines. Args: - images (`List[PIL.Image.Image]` or `np.ndarray`) - List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + images (`list[PIL.Image.Image]` or `np.ndarray`) + list of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, num_channels)`. - text (`List[str]` or `List[List[str]]`) - List of generated text strings of length `batch_size` or a list of list of strings whose outer list has + text (`list[str]` or `list[list[str]]`) + list of generated text strings of length `batch_size` or a list of list of strings whose outer list has length `batch_size`. """ - images: Optional[Union[List[PIL.Image.Image], np.ndarray]] - text: Optional[Union[List[str], List[List[str]]]] + images: Optional[list[PIL.Image.Image] | np.ndarray] + text: Optional[list[str] | list[list[str]]] class UniDiffuserPipeline(DeprecatedPipelineMixin, DiffusionPipeline): @@ -434,7 +434,7 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device @@ -442,7 +442,7 @@ def encode_prompt( number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -541,7 +541,7 @@ def encode_prompt( # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance and negative_prompt_embeds is None: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif prompt is not None and type(prompt) is not type(negative_prompt): @@ -1119,18 +1119,18 @@ def check_inputs( @torch.no_grad() def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, - image: Optional[Union[torch.Tensor, PIL.Image.Image]] = None, + prompt: Optional[str | list[str]] = None, + image: Optional[torch.Tensor | PIL.Image.Image] = None, height: Optional[int] = None, width: Optional[int] = None, data_type: Optional[int] = 1, num_inference_steps: int = 50, guidance_scale: float = 8.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: Optional[int] = 1, num_prompts_per_image: Optional[int] = 1, eta: float = 0.0, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_latents: Optional[torch.Tensor] = None, vae_latents: Optional[torch.Tensor] = None, @@ -1146,7 +1146,7 @@ def __call__( The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. Required for text-conditioned image generation (`text2img`) mode. image (`torch.Tensor` or `PIL.Image.Image`, *optional*): @@ -1166,7 +1166,7 @@ def __call__( guidance_scale (`float`, *optional*, defaults to 8.0): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). Used in text-conditioned image generation (`text2img`) mode. @@ -1181,7 +1181,7 @@ def __call__( eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): diff --git a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py index 91a54e1ae82f..0509fa30dde4 100644 --- a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py +++ b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import torch from PIL import Image @@ -250,24 +250,24 @@ def check_inputs( @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - task_prompt: Union[str, List[str]] = None, - content_prompt: Union[str, List[str]] = None, + task_prompt: str | list[str] = None, + content_prompt: str | list[str] = None, image: Optional[torch.FloatTensor] = None, upsampling_height: Optional[int] = None, upsampling_width: Optional[int] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 30.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, upsampling_strength: float = 1.0, ): @@ -275,11 +275,11 @@ def __call__( Function invoked when calling the VisualCloze pipeline for generation. Args: - task_prompt (`str` or `List[str]`, *optional*): + task_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the task intention. - content_prompt (`str` or `List[str]`, *optional*): + content_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the content or caption of the target image to be generated. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -297,7 +297,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -309,7 +309,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -336,7 +336,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py index e12995106bcf..f1601f9dfce7 100644 --- a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py +++ b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import numpy as np import torch @@ -192,7 +192,7 @@ def __init__( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_images_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -242,7 +242,7 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds def _get_clip_prompt_embeds( self, - prompt: Union[str, List[str]], + prompt: str | list[str], num_images_per_prompt: int = 1, device: Optional[torch.device] = None, ): @@ -287,9 +287,9 @@ def _get_clip_prompt_embeds( # Modified from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt def encode_prompt( self, - layout_prompt: Union[str, List[str]], - task_prompt: Union[str, List[str]], - content_prompt: Union[str, List[str]], + layout_prompt: str | list[str], + task_prompt: str | list[str], + content_prompt: str | list[str], device: Optional[torch.device] = None, num_images_per_prompt: int = 1, prompt_embeds: Optional[torch.FloatTensor] = None, @@ -300,12 +300,12 @@ def encode_prompt( r""" Args: - layout_prompt (`str` or `List[str]`, *optional*): + layout_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the number of in-context examples and the number of images involved in the task. - task_prompt (`str` or `List[str]`, *optional*): + task_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the task intention. - content_prompt (`str` or `List[str]`, *optional*): + content_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the content or caption of the target image to be generated. device: (`torch.device`): torch device @@ -709,33 +709,33 @@ def interrupt(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - task_prompt: Union[str, List[str]] = None, - content_prompt: Union[str, List[str]] = None, + task_prompt: str | list[str] = None, + content_prompt: str | list[str] = None, image: Optional[torch.FloatTensor] = None, num_inference_steps: int = 50, - sigmas: Optional[List[float]] = None, + sigmas: Optional[list[float]] = None, guidance_scale: float = 30.0, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.FloatTensor] = None, prompt_embeds: Optional[torch.FloatTensor] = None, pooled_prompt_embeds: Optional[torch.FloatTensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, - joint_attention_kwargs: Optional[Dict[str, Any]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" Function invoked when calling the VisualCloze pipeline for generation. Args: - task_prompt (`str` or `List[str]`, *optional*): + task_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the task intention. - content_prompt (`str` or `List[str]`, *optional*): + content_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to define the content or caption of the target image to be generated. - image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`): `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a @@ -743,7 +743,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. @@ -755,7 +755,7 @@ def __call__( the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): @@ -782,7 +782,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/visualcloze/visualcloze_utils.py b/src/diffusers/pipelines/visualcloze/visualcloze_utils.py index efe5dff47623..a2c08c833fa1 100644 --- a/src/diffusers/pipelines/visualcloze/visualcloze_utils.py +++ b/src/diffusers/pipelines/visualcloze/visualcloze_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Optional import torch from PIL import Image @@ -40,8 +40,8 @@ def __init__(self, *args, resolution: int = 384, **kwargs): self.resolution = resolution def preprocess_image( - self, input_images: List[List[Optional[Image.Image]]], vae_scale_factor: int - ) -> Tuple[List[List[torch.Tensor]], List[List[List[int]]], List[int]]: + self, input_images: list[list[Optional[Image.Image]]], vae_scale_factor: int + ) -> tuple[list[list[torch.Tensor]], list[list[list[int]]], list[int]]: """ Preprocesses input images for the VisualCloze pipeline. @@ -52,7 +52,7 @@ def preprocess_image( 4. Tracking image sizes and positions of target images Args: - input_images (List[List[Optional[Image.Image]]]): + input_images (list[list[Optional[Image.Image]]]): A nested list of PIL Images where: - Outer list represents different samples, including in-context examples and the query - Inner list contains images for the task @@ -61,17 +61,17 @@ def preprocess_image( The scale factor used by the VAE for resizing images Returns: - Tuple containing: - - List[List[torch.Tensor]]: Preprocessed images in tensor format - - List[List[List[int]]]: Dimensions of each processed image [height, width] - - List[int]: Target positions indicating which images are to be generated + tuple containing: + - list[list[torch.Tensor]]: Preprocessed images in tensor format + - list[list[list[int]]]: Dimensions of each processed image [height, width] + - list[int]: Target positions indicating which images are to be generated """ n_samples, n_task_images = len(input_images), len(input_images[0]) divisible = 2 * vae_scale_factor - processed_images: List[List[Image.Image]] = [[] for _ in range(n_samples)] - resize_size: List[Optional[Tuple[int, int]]] = [None for _ in range(n_samples)] - target_position: List[int] = [] + processed_images: list[list[Image.Image]] = [[] for _ in range(n_samples)] + resize_size: list[Optional[tuple[int, int]]] = [None for _ in range(n_samples)] + target_position: list[int] = [] # Process each sample for i in range(n_samples): @@ -125,19 +125,19 @@ def preprocess_image( return processed_images, image_sizes, target_position def preprocess_mask( - self, input_images: List[List[Image.Image]], target_position: List[int] - ) -> List[List[torch.Tensor]]: + self, input_images: list[list[Image.Image]], target_position: list[int] + ) -> list[list[torch.Tensor]]: """ Generate masks for the VisualCloze pipeline. Args: - input_images (List[List[Image.Image]]): + input_images (list[list[Image.Image]]): Processed images from preprocess_image - target_position (List[int]): + target_position (list[int]): Binary list marking the positions of target images (1 for target, 0 for condition) Returns: - List[List[torch.Tensor]]: + list[list[torch.Tensor]]: A nested list of mask tensors (1 for target positions, 0 for condition images) """ mask = [] @@ -155,10 +155,10 @@ def preprocess_mask( def preprocess_image_upsampling( self, - input_images: List[List[Image.Image]], + input_images: list[list[Image.Image]], height: int, width: int, - ) -> Tuple[List[List[Image.Image]], List[List[List[int]]]]: + ) -> tuple[list[list[Image.Image]], list[list[list[int]]]]: """Process images for the upsampling stage in the VisualCloze pipeline. Args: @@ -167,7 +167,7 @@ def preprocess_image_upsampling( width: Target width Returns: - Tuple of processed image and its size + tuple of processed image and its size """ image = self.resize(input_images[0][0], height, width) image = self.pil_to_numpy(image) # to np @@ -178,10 +178,10 @@ def preprocess_image_upsampling( image_sizes = [[[height, width]]] return input_images, image_sizes - def preprocess_mask_upsampling(self, input_images: List[List[Image.Image]]) -> List[List[torch.Tensor]]: + def preprocess_mask_upsampling(self, input_images: list[list[Image.Image]]) -> list[list[torch.Tensor]]: return [[torch.ones((1, 1, input_images[0][0].shape[2], input_images[0][0].shape[3]))]] - def get_layout_prompt(self, size: Tuple[int, int]) -> str: + def get_layout_prompt(self, size: tuple[int, int]) -> str: layout_instruction = ( f"A grid layout with {size[0]} rows and {size[1]} columns, displaying {size[0] * size[1]} images arranged side by side.", ) @@ -189,9 +189,9 @@ def get_layout_prompt(self, size: Tuple[int, int]) -> str: def preprocess( self, - task_prompt: Union[str, List[str]], - content_prompt: Union[str, List[str]], - input_images: Optional[List[List[List[Optional[str]]]]] = None, + task_prompt: str | list[str], + content_prompt: str | list[str], + input_images: Optional[list[list[list[Optional[str]]]]] = None, height: Optional[int] = None, width: Optional[int] = None, upsampling: bool = False, @@ -202,7 +202,7 @@ def preprocess( Args: task_prompt: Task description(s) content_prompt: Content description(s) - input_images: List of images or None for the target images + input_images: list of images or None for the target images height: Optional target height for upsampling stage width: Optional target width for upsampling stage upsampling: Whether this is in the upsampling processing stage diff --git a/src/diffusers/pipelines/wan/pipeline_output.py b/src/diffusers/pipelines/wan/pipeline_output.py index 88907ad0f0a1..7c7f8a4f0ad9 100644 --- a/src/diffusers/pipelines/wan/pipeline_output.py +++ b/src/diffusers/pipelines/wan/pipeline_output.py @@ -11,8 +11,8 @@ class WanPipelineOutput(BaseOutput): Output class for Wan pipelines. Args: - frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): - List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]): + list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. """ diff --git a/src/diffusers/pipelines/wan/pipeline_wan.py b/src/diffusers/pipelines/wan/pipeline_wan.py index 78fe71ea9138..c10deb594317 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan.py +++ b/src/diffusers/pipelines/wan/pipeline_wan.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import regex as re import torch @@ -156,7 +156,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -197,8 +197,8 @@ def _get_t5_prompt_embeds( def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -211,9 +211,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -330,7 +330,7 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -381,8 +381,8 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 480, width: int = 832, num_frames: int = 81, @@ -390,26 +390,26 @@ def __call__( guidance_scale: float = 5.0, guidance_scale_2: Optional[float] = None, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, pass `prompt_embeds` instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts to avoid during image generation. If not defined, pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale` < `1`). height (`int`, defaults to `480`): @@ -433,7 +433,7 @@ def __call__( and the pipeline's `boundary_ratio` are not None. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -456,7 +456,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index b7fd0b05980f..79c9e7990f2c 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import PIL import regex as re @@ -198,7 +198,7 @@ def __init__( def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 512, device: Optional[torch.device] = None, @@ -250,8 +250,8 @@ def encode_image( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -264,9 +264,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -400,10 +400,10 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, last_image: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> tuple[torch.Tensor, torch.Tensor]: num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 latent_height = height // self.vae_scale_factor_spatial latent_width = width // self.vae_scale_factor_spatial @@ -509,8 +509,8 @@ def attention_kwargs(self): def __call__( self, image: PipelineImageInput, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 480, width: int = 832, num_frames: int = 81, @@ -518,7 +518,7 @@ def __call__( guidance_scale: float = 5.0, guidance_scale_2: Optional[float] = None, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, @@ -526,11 +526,11 @@ def __call__( last_image: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" @@ -539,10 +539,10 @@ def __call__( Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -567,7 +567,7 @@ def __call__( and the pipeline's `boundary_ratio` are not None. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -596,7 +596,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wan/pipeline_wan_vace.py b/src/diffusers/pipelines/wan/pipeline_wan_vace.py index 2b1890afec97..52d4f4fe1c56 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_vace.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_vace.py @@ -13,7 +13,7 @@ # limitations under the License. import html -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import PIL.Image import regex as re @@ -201,7 +201,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -243,8 +243,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -257,9 +257,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -403,9 +403,9 @@ def check_inputs( def preprocess_conditions( self, - video: Optional[List[PipelineImageInput]] = None, - mask: Optional[List[PipelineImageInput]] = None, - reference_images: Optional[Union[PIL.Image.Image, List[PIL.Image.Image], List[List[PIL.Image.Image]]]] = None, + video: Optional[list[PipelineImageInput]] = None, + mask: Optional[list[PipelineImageInput]] = None, + reference_images: Optional[PIL.Image.Image | list[PIL.Image.Image] | list[list[PIL.Image.Image]]] = None, batch_size: int = 1, height: int = 480, width: int = 832, @@ -501,8 +501,8 @@ def prepare_video_latents( self, video: torch.Tensor, mask: torch.Tensor, - reference_images: Optional[List[List[torch.Tensor]]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + reference_images: Optional[list[list[torch.Tensor]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, device: Optional[torch.device] = None, ) -> torch.Tensor: device = device or self._execution_device @@ -567,8 +567,8 @@ def prepare_video_latents( def prepare_masks( self, mask: torch.Tensor, - reference_images: Optional[List[torch.Tensor]] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + reference_images: Optional[list[torch.Tensor]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, ) -> torch.Tensor: if isinstance(generator, list): # TODO: support this @@ -621,7 +621,7 @@ def prepare_latents( num_frames: int = 81, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, ) -> torch.Tensor: if latents is not None: @@ -672,12 +672,12 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, - video: Optional[List[PipelineImageInput]] = None, - mask: Optional[List[PipelineImageInput]] = None, - reference_images: Optional[List[PipelineImageInput]] = None, - conditioning_scale: Union[float, List[float], torch.Tensor] = 1.0, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, + video: Optional[list[PipelineImageInput]] = None, + mask: Optional[list[PipelineImageInput]] = None, + reference_images: Optional[list[PipelineImageInput]] = None, + conditioning_scale: float | list[float] | torch.Tensor = 1.0, height: int = 480, width: int = 832, num_frames: int = 81, @@ -685,47 +685,47 @@ def __call__( guidance_scale: float = 5.0, guidance_scale_2: Optional[float] = None, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds` instead. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). - video (`List[PIL.Image.Image]`, *optional*): + video (`list[PIL.Image.Image]`, *optional*): The input video or videos to be used as a starting point for the generation. The video should be a list of PIL images, a numpy array, or a torch tensor. Currently, the pipeline only supports generating one video at a time. - mask (`List[PIL.Image.Image]`, *optional*): + mask (`list[PIL.Image.Image]`, *optional*): The input mask defines which video regions to condition on and which to generate. Black areas in the mask indicate conditioning regions, while white areas indicate regions for generation. The mask should be a list of PIL images, a numpy array, or a torch tensor. Currently supports generating a single video at a time. - reference_images (`List[PIL.Image.Image]`, *optional*): + reference_images (`list[PIL.Image.Image]`, *optional*): A list of one or more reference images as extra conditioning for the generation. For example, if you are trying to inpaint a video to change the character, you can pass reference images of the new character here. Refer to the Diffusers [examples](https://github.com/huggingface/diffusers/pull/11582) and original [user guide](https://github.com/ali-vilab/VACE/blob/0897c6d055d7d9ea9e191dce763006664d9780f8/UserGuide.md) for a full list of supported tasks and use cases. - conditioning_scale (`float`, `List[float]`, `torch.Tensor`, defaults to `1.0`): + conditioning_scale (`float`, `list[float]`, `torch.Tensor`, defaults to `1.0`): The conditioning scale to be applied when adding the control conditioning latent stream to the denoising latent stream in each control layer of the model. If a float is provided, it will be applied uniformly to all layers. If a list or tensor is provided, it should have the same length as the number @@ -751,7 +751,7 @@ def __call__( and the pipeline's `boundary_ratio` are not None. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -774,7 +774,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py index a976126da7fe..aa3d65554108 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py @@ -14,7 +14,7 @@ import html import inspect -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, Optional import regex as re import torch @@ -101,9 +101,9 @@ def prompt_clean(text): def retrieve_timesteps( scheduler, num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: Optional[str | torch.device] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, **kwargs, ): r""" @@ -118,15 +118,15 @@ def retrieve_timesteps( must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. """ if timesteps is not None and sigmas is not None: @@ -221,7 +221,7 @@ def __init__( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds def _get_t5_prompt_embeds( self, - prompt: Union[str, List[str]] = None, + prompt: str | list[str] = None, num_videos_per_prompt: int = 1, max_sequence_length: int = 226, device: Optional[torch.device] = None, @@ -263,8 +263,8 @@ def _get_t5_prompt_embeds( # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( self, - prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, + prompt: str | list[str], + negative_prompt: Optional[str | list[str]] = None, do_classifier_free_guidance: bool = True, num_videos_per_prompt: int = 1, prompt_embeds: Optional[torch.Tensor] = None, @@ -277,9 +277,9 @@ def encode_prompt( Encodes the prompt into text encoder hidden states. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): prompt to be encoded - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). @@ -480,34 +480,34 @@ def attention_kwargs(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - video: List[Image.Image] = None, - prompt: Union[str, List[str]] = None, - negative_prompt: Union[str, List[str]] = None, + video: list[Image.Image] = None, + prompt: str | list[str] = None, + negative_prompt: str | list[str] = None, height: int = 480, width: int = 832, num_inference_steps: int = 50, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, guidance_scale: float = 5.0, strength: float = 0.8, num_videos_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, output_type: Optional[str] = "np", return_dict: bool = True, - attention_kwargs: Optional[Dict[str, Any]] = None, + attention_kwargs: Optional[dict[str, Any]] = None, callback_on_step_end: Optional[ - Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + Callable[[int, int, Dict], None] | PipelineCallback | MultiPipelineCallbacks ] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ): r""" The call function to the pipeline for generation. Args: - prompt (`str` or `List[str]`, *optional*): + prompt (`str` or `list[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds` instead. height (`int`, defaults to `480`): @@ -529,7 +529,7 @@ def __call__( Higher strength leads to more differences between original image and generated video. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -552,7 +552,7 @@ def __call__( each denoising step during the inference. with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py b/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py index 5ab206b15176..932c7ac618f6 100644 --- a/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +++ b/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union import torch import torch.nn as nn @@ -142,7 +141,7 @@ def encode(self, x: torch.Tensor, return_dict: bool = True) -> VQEncoderOutput: @apply_forward_hook def decode( self, h: torch.Tensor, force_not_quantize: bool = True, return_dict: bool = True - ) -> Union[DecoderOutput, torch.Tensor]: + ) -> DecoderOutput | torch.Tensor: if not force_not_quantize: quant, _, _ = self.vquantizer(h) else: @@ -155,7 +154,7 @@ def decode( return DecoderOutput(sample=dec) - def forward(self, sample: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + def forward(self, sample: torch.Tensor, return_dict: bool = True) -> DecoderOutput | torch.Tensor: r""" Args: sample (`torch.Tensor`): Input sample. diff --git a/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py b/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py index ae821a55788a..dc6f41082901 100644 --- a/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +++ b/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py @@ -14,7 +14,6 @@ # limitations under the License. import math -from typing import Dict, Union import torch import torch.nn as nn @@ -63,7 +62,7 @@ def __init__(self, c_in=16, c=1280, c_cond=1024, c_r=64, depth=16, nhead=16, dro @property # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors - def attn_processors(self) -> Dict[str, AttentionProcessor]: + def attn_processors(self) -> dict[str, AttentionProcessor]: r""" Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with @@ -72,7 +71,7 @@ def attn_processors(self) -> Dict[str, AttentionProcessor]: # set recursively processors = {} - def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: dict[str, AttentionProcessor]): if hasattr(module, "get_processor"): processors[f"{name}.processor"] = module.get_processor() @@ -87,7 +86,7 @@ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: return processors # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor - def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + def set_attn_processor(self, processor: AttentionProcessor | dict[str, AttentionProcessor]): r""" Sets the attention processor to use to compute attention. diff --git a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py index bbdb60471fd1..f1a153b02843 100644 --- a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +++ b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -156,7 +156,7 @@ def encode_prompt( uncond_text_encoder_hidden_states = None if do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -217,33 +217,33 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - image_embeddings: Union[torch.Tensor, List[torch.Tensor]], - prompt: Union[str, List[str]] = None, + image_embeddings: torch.Tensor | list[torch.Tensor], + prompt: str | list[str] = None, num_inference_steps: int = 12, - timesteps: Optional[List[float]] = None, + timesteps: Optional[list[float]] = None, guidance_scale: float = 0.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - image_embedding (`torch.Tensor` or `List[torch.Tensor]`): + image_embedding (`torch.Tensor` or `list[torch.Tensor]`): Image Embeddings either extracted from an image or generated by a Prior Model. - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. num_inference_steps (`int`, *optional*, defaults to 12): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 0.0): @@ -252,12 +252,12 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `decoder_guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -274,7 +274,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py index c54c1fefe8fe..699b18625e7e 100644 --- a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +++ b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import torch from transformers import CLIPTextModel, CLIPTokenizer @@ -113,7 +113,7 @@ def __init__( def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) - def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` @@ -123,7 +123,7 @@ def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[t self.prior_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) self.decoder_pipe.enable_model_cpu_offload(gpu_id=gpu_id, device=device) - def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = None): + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: torch.device | str = None): r""" Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a @@ -145,36 +145,36 @@ def set_progress_bar_config(self, **kwargs): @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, height: int = 512, width: int = 512, prior_num_inference_steps: int = 60, - prior_timesteps: Optional[List[float]] = None, + prior_timesteps: Optional[list[float]] = None, prior_guidance_scale: float = 4.0, num_inference_steps: int = 12, - decoder_timesteps: Optional[List[float]] = None, + decoder_timesteps: Optional[list[float]] = None, decoder_guidance_scale: float = 0.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, num_images_per_prompt: int = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pil", return_dict: bool = True, prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prior_callback_on_step_end_tensor_inputs: list[str] = ["latents"], callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation for the prior and decoder. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): @@ -196,7 +196,7 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `prior_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - prior_num_inference_steps (`Union[int, Dict[float, int]]`, *optional*, defaults to 60): + prior_num_inference_steps (`Union[int, dict[float, int]]`, *optional*, defaults to 60): The number of prior denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. For more specific timestep spacing, you can pass customized `prior_timesteps` @@ -204,10 +204,10 @@ def __call__( The number of decoder denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. For more specific timestep spacing, you can pass customized `timesteps` - prior_timesteps (`List[float]`, *optional*): + prior_timesteps (`list[float]`, *optional*): Custom timesteps to use for the denoising process for the prior. If not defined, equal spaced `prior_num_inference_steps` timesteps are used. Must be in descending order. - decoder_timesteps (`List[float]`, *optional*): + decoder_timesteps (`list[float]`, *optional*): Custom timesteps to use for the denoising process for the decoder. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. decoder_guidance_scale (`float`, *optional*, defaults to 0.0): @@ -216,7 +216,7 @@ def __call__( of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -232,7 +232,7 @@ def __call__( A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. - prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + prior_callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. @@ -241,7 +241,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py index e138b6e805c8..9438b2a3e568 100644 --- a/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +++ b/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from math import ceil -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Optional import numpy as np import torch @@ -67,7 +67,7 @@ class WuerstchenPriorPipelineOutput(BaseOutput): """ - image_embeddings: Union[torch.Tensor, np.ndarray] + image_embeddings: torch.Tensor | np.ndarray class WuerstchenPriorPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixin): @@ -191,7 +191,7 @@ def encode_prompt( prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) if negative_prompt_embeds is None and do_classifier_free_guidance: - uncond_tokens: List[str] + uncond_tokens: list[str] if negative_prompt is None: uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): @@ -290,29 +290,29 @@ def num_timesteps(self): @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, - prompt: Optional[Union[str, List[str]]] = None, + prompt: Optional[str | list[str]] = None, height: int = 1024, width: int = 1024, num_inference_steps: int = 60, - timesteps: List[float] = None, + timesteps: list[float] = None, guidance_scale: float = 8.0, - negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[str | list[str]] = None, prompt_embeds: Optional[torch.Tensor] = None, negative_prompt_embeds: Optional[torch.Tensor] = None, num_images_per_prompt: Optional[int] = 1, - generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + generator: Optional[torch.Generator | list[torch.Generator]] = None, latents: Optional[torch.Tensor] = None, output_type: Optional[str] = "pt", return_dict: bool = True, callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end_tensor_inputs: list[str] = ["latents"], **kwargs, ): """ Function invoked when calling the pipeline for generation. Args: - prompt (`str` or `List[str]`): + prompt (`str` or `list[str]`): The prompt or prompts to guide the image generation. height (`int`, *optional*, defaults to 1024): The height in pixels of the generated image. @@ -321,7 +321,7 @@ def __call__( num_inference_steps (`int`, *optional*, defaults to 60): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 8.0): @@ -330,7 +330,7 @@ def __call__( equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): + negative_prompt (`str` or `list[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `decoder_guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): @@ -342,7 +342,7 @@ def __call__( argument. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + generator (`torch.Generator` or `list[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): @@ -359,7 +359,7 @@ def __call__( with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): + callback_on_step_end_tensor_inputs (`list`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. diff --git a/src/diffusers/quantizers/auto.py b/src/diffusers/quantizers/auto.py index 070bcd0b2151..3e4967ef50fb 100644 --- a/src/diffusers/quantizers/auto.py +++ b/src/diffusers/quantizers/auto.py @@ -17,7 +17,7 @@ """ import warnings -from typing import Dict, Optional, Union +from typing import Dict, Optional from .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer from .gguf import GGUFQuantizer @@ -82,7 +82,7 @@ def from_dict(cls, quantization_config_dict: Dict): return target_cls.from_dict(quantization_config_dict) @classmethod - def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs): + def from_config(cls, quantization_config: QuantizationConfigMixin | Dict, **kwargs): # Convert it to a QuantizationConfig if the q_config is a dict if isinstance(quantization_config, dict): quantization_config = cls.from_dict(quantization_config) @@ -123,7 +123,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): @classmethod def merge_quantization_configs( cls, - quantization_config: Union[dict, QuantizationConfigMixin], + quantization_config: dict | QuantizationConfigMixin, quantization_config_from_args: Optional[QuantizationConfigMixin], ): """ diff --git a/src/diffusers/quantizers/base.py b/src/diffusers/quantizers/base.py index 24fc724b4c88..f0060767d7da 100644 --- a/src/diffusers/quantizers/base.py +++ b/src/diffusers/quantizers/base.py @@ -18,7 +18,7 @@ """ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional from ..utils import is_torch_available from .quantization_config import QuantizationConfigMixin @@ -40,9 +40,9 @@ class DiffusersQuantizer(ABC): Attributes quantization_config (`diffusers.quantizers.quantization_config.QuantizationConfigMixin`): The quantization config that defines the quantization parameters of your model that you want to quantize. - modules_to_not_convert (`List[str]`, *optional*): + modules_to_not_convert (`list[str]`, *optional*): The list of module names to not convert when quantizing the model. - required_packages (`List[str]`, *optional*): + required_packages (`list[str]`, *optional*): The list of required pip packages to install prior to using the quantizer requires_calibration (`bool`): Whether the quantization method requires to calibrate the model before using it. @@ -76,7 +76,7 @@ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype": """ return torch_dtype - def update_device_map(self, device_map: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + def update_device_map(self, device_map: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: """ Override this method if you want to pass a override the existing device map with a new one. E.g. for bitsandbytes, since `accelerate` is a hard requirement, if no device_map is passed, the device_map is set to @@ -100,17 +100,17 @@ def adjust_target_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype": """ return torch_dtype - def update_missing_keys(self, model, missing_keys: List[str], prefix: str) -> List[str]: + def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]: """ Override this method if you want to adjust the `missing_keys`. Args: - missing_keys (`List[str]`, *optional*): + missing_keys (`list[str]`, *optional*): The list of missing keys in the checkpoint compared to the state dict of the model """ return missing_keys - def get_special_dtypes_update(self, model, torch_dtype: "torch.dtype") -> Dict[str, "torch.dtype"]: + def get_special_dtypes_update(self, model, torch_dtype: "torch.dtype") -> dict[str, "torch.dtype"]: """ returns dtypes for modules that are not quantized - used for the computation of the device_map in case one passes a str as a device_map. The method will use the `modules_to_not_convert` that is modified in @@ -130,7 +130,7 @@ def get_special_dtypes_update(self, model, torch_dtype: "torch.dtype") -> Dict[s if any(m in name for m in self.modules_to_not_convert) } - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: """adjust max_memory argument for infer_auto_device_map() if extra memory is needed for quantization""" return max_memory @@ -139,7 +139,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ) -> bool: """ diff --git a/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py b/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py index 0dfdff019b79..e7aef4aa9a7a 100644 --- a/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py +++ b/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py @@ -16,7 +16,7 @@ https://github.com/huggingface/transformers/blob/c409cd81777fb27aadc043ed3d8339dbc020fb3b/src/transformers/quantizers/quantizer_bnb_4bit.py """ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional from ...utils import get_module_from_name from ..base import DiffusersQuantizer @@ -111,7 +111,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ) -> bool: import bitsandbytes as bnb @@ -133,8 +133,8 @@ def create_quantized_param( param_value: "torch.Tensor", param_name: str, target_device: "torch.device", - state_dict: Dict[str, Any], - unexpected_keys: Optional[List[str]] = None, + state_dict: dict[str, Any], + unexpected_keys: Optional[list[str]] = None, **kwargs, ): import bitsandbytes as bnb @@ -218,7 +218,7 @@ def check_quantized_param_shape(self, param_name, current_param, loaded_param): else: return True - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: # need more space for buffers that are created during quantization max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory @@ -255,7 +255,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): from .utils import replace_with_bnb_linear @@ -387,7 +387,7 @@ def validate_environment(self, *args, **kwargs): ) # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.adjust_max_memory - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: # need more space for buffers that are created during quantization max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory @@ -432,7 +432,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ): import bitsandbytes as bnb @@ -455,8 +455,8 @@ def create_quantized_param( param_value: "torch.Tensor", param_name: str, target_device: "torch.device", - state_dict: Dict[str, Any], - unexpected_keys: Optional[List[str]] = None, + state_dict: dict[str, Any], + unexpected_keys: Optional[list[str]] = None, **kwargs, ): import bitsandbytes as bnb @@ -513,7 +513,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): from .utils import replace_with_bnb_linear diff --git a/src/diffusers/quantizers/bitsandbytes/utils.py b/src/diffusers/quantizers/bitsandbytes/utils.py index 429aabb8fae6..aea4e7dda57f 100644 --- a/src/diffusers/quantizers/bitsandbytes/utils.py +++ b/src/diffusers/quantizers/bitsandbytes/utils.py @@ -18,7 +18,6 @@ import inspect from inspect import signature -from typing import Union from ...utils import is_accelerate_available, is_bitsandbytes_available, is_torch_available, logging from ..quantization_config import QuantizationMethod @@ -128,10 +127,10 @@ def replace_with_bnb_linear(model, modules_to_not_convert=None, current_key_name Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. - modules_to_not_convert (`List[`str`]`, *optional*, defaults to `[]`): + modules_to_not_convert (`list[`str`]`, *optional*, defaults to `[]`): Names of the modules to not convert in `Linear8bitLt`. In practice we keep the `modules_to_not_convert` in full precision for numerical stability reasons. - current_key_name (`List[`str`]`, *optional*): + current_key_name (`list[`str`]`, *optional*): An array to track the current key of the recursion. This is used to check whether the current key (part of it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or `disk`). @@ -305,7 +304,7 @@ def dequantize_and_replace( return model -def _check_bnb_status(module) -> Union[bool, bool]: +def _check_bnb_status(module) -> bool | bool: is_loaded_in_4bit_bnb = ( hasattr(module, "is_loaded_in_4bit") and module.is_loaded_in_4bit diff --git a/src/diffusers/quantizers/gguf/gguf_quantizer.py b/src/diffusers/quantizers/gguf/gguf_quantizer.py index aa5ebf5711a3..b91d1cc9869e 100644 --- a/src/diffusers/quantizers/gguf/gguf_quantizer.py +++ b/src/diffusers/quantizers/gguf/gguf_quantizer.py @@ -1,4 +1,6 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional from ..base import DiffusersQuantizer @@ -57,7 +59,7 @@ def validate_environment(self, *args, **kwargs): ) # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.adjust_max_memory - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: # need more space for buffers that are created during quantization max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory @@ -90,9 +92,9 @@ def check_quantized_param_shape(self, param_name, current_param, loaded_param): def check_if_quantized_param( self, model: "ModelMixin", - param_value: Union["GGUFParameter", "torch.Tensor"], + param_value: "GGUFParameter" | "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ) -> bool: if isinstance(param_value, GGUFParameter): @@ -103,11 +105,11 @@ def check_if_quantized_param( def create_quantized_param( self, model: "ModelMixin", - param_value: Union["GGUFParameter", "torch.Tensor"], + param_value: "GGUFParameter" | "torch.Tensor", param_name: str, target_device: "torch.device", - state_dict: Optional[Dict[str, Any]] = None, - unexpected_keys: Optional[List[str]] = None, + state_dict: Optional[dict[str, Any]] = None, + unexpected_keys: Optional[list[str]] = None, **kwargs, ): module, tensor_name = get_module_from_name(model, param_name) @@ -123,7 +125,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): state_dict = kwargs.get("state_dict", None) diff --git a/src/diffusers/quantizers/modelopt/modelopt_quantizer.py b/src/diffusers/quantizers/modelopt/modelopt_quantizer.py index 534f752321b3..78321f403568 100644 --- a/src/diffusers/quantizers/modelopt/modelopt_quantizer.py +++ b/src/diffusers/quantizers/modelopt/modelopt_quantizer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Union +from typing import TYPE_CHECKING, Any from ...utils import ( get_module_from_name, @@ -61,7 +61,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ): # ModelOpt imports diffusers internally. This is here to prevent circular imports @@ -101,7 +101,7 @@ def create_quantized_param( mtq.compress(module) module.weight.requires_grad = False - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory @@ -116,7 +116,7 @@ def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype" torch_dtype = torch.float32 return torch_dtype - def get_conv_param_names(self, model: "ModelMixin") -> List[str]: + def get_conv_param_names(self, model: "ModelMixin") -> list[str]: """ Get parameter names for all convolutional layers in a HuggingFace ModelMixin. Includes Conv1d/2d/3d and ConvTranspose1d/2d/3d. @@ -142,7 +142,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): # ModelOpt imports diffusers internally. This is here to prevent circular imports diff --git a/src/diffusers/quantizers/pipe_quant_config.py b/src/diffusers/quantizers/pipe_quant_config.py index f75a337341a9..3be180514f5d 100644 --- a/src/diffusers/quantizers/pipe_quant_config.py +++ b/src/diffusers/quantizers/pipe_quant_config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import inspect -from typing import Dict, List, Optional, Union +from typing import Optional from ..utils import is_transformers_available, logging from .quantization_config import QuantizationConfigMixin as DiffQuantConfigMixin @@ -47,9 +49,9 @@ class PipelineQuantizationConfig: def __init__( self, quant_backend: str = None, - quant_kwargs: Dict[str, Union[str, float, int, dict]] = None, - components_to_quantize: Optional[Union[List[str], str]] = None, - quant_mapping: Dict[str, Union[DiffQuantConfigMixin, "TransformersQuantConfigMixin"]] = None, + quant_kwargs: dict[str, str | float | int | dict] = None, + components_to_quantize: Optional[list[str] | str] = None, + quant_mapping: dict[str, DiffQuantConfigMixin | "TransformersQuantConfigMixin"] = None, ): self.quant_backend = quant_backend # Initialize kwargs to be {} to set to the defaults. diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index 5dd8f56717df..eb4567e4e808 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -20,6 +20,8 @@ https://github.com/huggingface/transformers/blob/52cb4034ada381fe1ffe8d428a1076e5411a8026/src/transformers/utils/quantization_config.py """ +from __future__ import annotations + import copy import dataclasses import importlib.metadata @@ -30,7 +32,7 @@ from dataclasses import dataclass, is_dataclass from enum import Enum from functools import partial -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Optional from packaging import version @@ -76,12 +78,12 @@ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs): Instantiates a [`QuantizationConfigMixin`] from a Python dictionary of parameters. Args: - config_dict (`Dict[str, Any]`): + config_dict (`dict[str, Any]`): Dictionary that will be used to instantiate the configuration object. return_unused_kwargs (`bool`, *optional*, defaults to `False`): Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in `PreTrainedModel`. - kwargs (`Dict[str, Any]`): + kwargs (`dict[str, Any]`): Additional parameters from which to initialize the configuration object. Returns: @@ -103,7 +105,7 @@ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs): else: return config - def to_json_file(self, json_file_path: Union[str, os.PathLike]): + def to_json_file(self, json_file_path: str | os.PathLike): """ Save this instance to a JSON file. @@ -120,10 +122,10 @@ def to_json_file(self, json_file_path: Union[str, os.PathLike]): writer.write(json_string) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Serializes this instance to a Python dictionary. Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. """ return copy.deepcopy(self.__dict__) @@ -159,11 +161,11 @@ def update(self, **kwargs): returning all the unused kwargs. Args: - kwargs (`Dict[str, Any]`): + kwargs (`dict[str, Any]`): Dictionary of attributes to tentatively update this class. Returns: - `Dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance. + `dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance. """ to_remove = [] for key, value in kwargs.items(): @@ -202,7 +204,7 @@ class BitsAndBytesConfig(QuantizationConfigMixin): These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning). - llm_int8_skip_modules (`List[str]`, *optional*): + llm_int8_skip_modules (`list[str]`, *optional*): An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as Jukebox that has several heads in different places and not necessarily at the last position. For example for `CausalLM` models, the last `lm_head` is typically kept in its original `dtype`. @@ -225,7 +227,7 @@ class BitsAndBytesConfig(QuantizationConfigMixin): quantized again. bnb_4bit_quant_storage (`torch.dtype` or str, *optional*, defaults to `torch.uint8`): This sets the storage type to pack the quanitzed 4-bit prarams. - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional parameters from which to initialize the configuration object. """ @@ -376,10 +378,10 @@ def quantization_method(self): else: return None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Serializes this instance to a Python dictionary. Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. """ output = copy.deepcopy(self.__dict__) output["bnb_4bit_compute_dtype"] = str(output["bnb_4bit_compute_dtype"]).split(".")[1] @@ -393,13 +395,13 @@ def __repr__(self): config_dict = self.to_dict() return f"{self.__class__.__name__} {json.dumps(config_dict, indent=2, sort_keys=True)}\n" - def to_diff_dict(self) -> Dict[str, Any]: + def to_diff_dict(self) -> dict[str, Any]: """ Removes all attributes from config which correspond to the default config attributes for better readability and serializes to a Python dictionary. Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance, + `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance, """ config_dict = self.to_dict() @@ -467,10 +469,10 @@ class TorchAoConfig(QuantizationConfigMixin): - Full function names: `uintx_weight_only` - Shorthands: `uint1wo`, `uint2wo`, `uint3wo`, `uint4wo`, `uint5wo`, `uint6wo`, `uint7wo` - An AOBaseConfig instance: for more advanced configuration options. - modules_to_not_convert (`List[str]`, *optional*, default to `None`): + modules_to_not_convert (`list[str]`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision. - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): The keyword arguments for the chosen type of quantization, for example, int4_weight_only quantization supports two keyword arguments `group_size` and `inner_k_tiles` currently. More API examples and documentation of arguments can be found in @@ -498,8 +500,8 @@ class TorchAoConfig(QuantizationConfigMixin): def __init__( self, - quant_type: Union[str, "AOBaseConfig"], # noqa: F821 - modules_to_not_convert: Optional[List[str]] = None, + quant_type: str | "AOBaseConfig", # noqa: F821 + modules_to_not_convert: Optional[list[str]] = None, **kwargs, ) -> None: self.quant_method = QuantizationMethod.TORCHAO @@ -829,7 +831,7 @@ class QuantoConfig(QuantizationConfigMixin): def __init__( self, weights_dtype: str = "int8", - modules_to_not_convert: Optional[List[str]] = None, + modules_to_not_convert: Optional[list[str]] = None, **kwargs, ): self.quant_method = QuantizationMethod.QUANTO @@ -861,7 +863,7 @@ class NVIDIAModelOptConfig(QuantizationConfigMixin): - INT4 - NF4 - NVFP4 - modules_to_not_convert (`List[str]`, *optional*, default to `None`): + modules_to_not_convert (`list[str]`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some weight_only (`bool`, *optional*, default to `False`): If set to `True`, the quantization will be applied only to the weights of the model. @@ -881,7 +883,7 @@ class NVIDIAModelOptConfig(QuantizationConfigMixin): The modelopt config, useful for passing custom configs to modelopt. disable_conv_quantization (`bool`, *optional*, default to `False`): If set to `True`, the quantization will be disabled for convolutional layers. - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional parameters which are to be used for calibration. """ @@ -900,7 +902,7 @@ class NVIDIAModelOptConfig(QuantizationConfigMixin): def __init__( self, quant_type: str, - modules_to_not_convert: Optional[List[str]] = None, + modules_to_not_convert: Optional[list[str]] = None, weight_only: bool = True, channel_quantize: Optional[int] = None, block_quantize: Optional[int] = None, @@ -970,7 +972,7 @@ def _normalize_quant_type(self, quant_type: str) -> str: act_type = None self.quant_type = w_type + ("_" + act_type if act_type is not None else "") - def get_config_from_quant_type(self) -> Dict[str, Any]: + def get_config_from_quant_type(self) -> dict[str, Any]: """ Get the config from the quantization type. """ diff --git a/src/diffusers/quantizers/quanto/quanto_quantizer.py b/src/diffusers/quantizers/quanto/quanto_quantizer.py index c5f71f816fc3..a036dabfe6f4 100644 --- a/src/diffusers/quantizers/quanto/quanto_quantizer.py +++ b/src/diffusers/quantizers/quanto/quanto_quantizer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Union +from typing import TYPE_CHECKING, Any from diffusers.utils.import_utils import is_optimum_quanto_version @@ -68,7 +68,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ): # Quanto imports diffusers internally. This is here to prevent circular imports @@ -105,7 +105,7 @@ def create_quantized_param( module.freeze() module.weight.requires_grad = False - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory @@ -127,7 +127,7 @@ def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype" torch_dtype = torch.float32 return torch_dtype - def update_missing_keys(self, model, missing_keys: List[str], prefix: str) -> List[str]: + def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]: # Quanto imports diffusers internally. This is here to prevent circular imports from optimum.quanto import QModuleMixin @@ -147,7 +147,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): self.modules_to_not_convert = self.quantization_config.modules_to_not_convert diff --git a/src/diffusers/quantizers/torchao/torchao_quantizer.py b/src/diffusers/quantizers/torchao/torchao_quantizer.py index 2334c7af8630..3359f431b7ee 100644 --- a/src/diffusers/quantizers/torchao/torchao_quantizer.py +++ b/src/diffusers/quantizers/torchao/torchao_quantizer.py @@ -21,7 +21,7 @@ import re import types from fnmatch import fnmatch -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional from packaging import version @@ -263,7 +263,7 @@ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": f"dtype you are using should be supported, please open an issue at https://github.com/huggingface/diffusers/issues." ) - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]: max_memory = {key: val * 0.9 for key, val in max_memory.items()} return max_memory @@ -272,7 +272,7 @@ def check_if_quantized_param( model: "ModelMixin", param_value: "torch.Tensor", param_name: str, - state_dict: Dict[str, Any], + state_dict: dict[str, Any], **kwargs, ) -> bool: param_device = kwargs.pop("param_device", None) @@ -293,8 +293,8 @@ def create_quantized_param( param_value: "torch.Tensor", param_name: str, target_device: "torch.device", - state_dict: Dict[str, Any], - unexpected_keys: List[str], + state_dict: dict[str, Any], + unexpected_keys: list[str], **kwargs, ): r""" @@ -358,7 +358,7 @@ def _process_model_before_weight_loading( self, model: "ModelMixin", device_map, - keep_in_fp32_modules: List[str] = [], + keep_in_fp32_modules: list[str] = [], **kwargs, ): self.modules_to_not_convert = self.quantization_config.modules_to_not_convert diff --git a/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py b/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py index 9206ee80a6b6..3f55d92c2369 100644 --- a/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py +++ b/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py @@ -14,7 +14,7 @@ from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -110,7 +110,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -134,7 +134,7 @@ def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.devic def add_noise_to_input( self, sample: torch.Tensor, sigma: float, generator: Optional[torch.Generator] = None - ) -> Tuple[torch.Tensor, float]: + ) -> tuple[torch.Tensor, float]: """ Explicit Langevin-like "churn" step of adding noise to the sample according to a `gamma_i ≥ 0` to reach a higher noise level `sigma_hat = sigma_i + gamma_i*sigma_i`. @@ -165,7 +165,7 @@ def step( sigma_prev: float, sample_hat: torch.Tensor, return_dict: bool = True, - ) -> Union[KarrasVeOutput, Tuple]: + ) -> KarrasVeOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). @@ -206,7 +206,7 @@ def step_correct( sample_prev: torch.Tensor, derivative: torch.Tensor, return_dict: bool = True, - ) -> Union[KarrasVeOutput, Tuple]: + ) -> KarrasVeOutput | tuple: """ Corrects the predicted sample based on the `model_output` of the network. diff --git a/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py b/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py index 5088bdb49761..99fa408e70c0 100644 --- a/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py +++ b/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py @@ -15,7 +15,6 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/yang-song/score_sde_pytorch import math -from typing import Union import torch @@ -48,7 +47,7 @@ def __init__(self, num_train_timesteps=2000, beta_min=0.1, beta_max=20, sampling self.discrete_sigmas = None self.timesteps = None - def set_timesteps(self, num_inference_steps, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps, device: str | torch.device = None): """ Sets the continuous timesteps used for the diffusion chain (to be run before inference). diff --git a/src/diffusers/schedulers/scheduling_amused.py b/src/diffusers/schedulers/scheduling_amused.py index 238b8d869171..dccf78e04a79 100644 --- a/src/diffusers/schedulers/scheduling_amused.py +++ b/src/diffusers/schedulers/scheduling_amused.py @@ -1,6 +1,6 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -58,8 +58,8 @@ def __init__( def set_timesteps( self, num_inference_steps: int, - temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), - device: Union[str, torch.device] = None, + temperature: int | tuple[int, int] | list[int] = (2, 0), + device: str | torch.device = None, ): self.timesteps = torch.arange(num_inference_steps, device=device).flip(0) @@ -76,7 +76,7 @@ def step( starting_mask_ratio: int = 1, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[AmusedSchedulerOutput, Tuple]: + ) -> AmusedSchedulerOutput | tuple: two_dim_input = sample.ndim == 3 and model_output.ndim == 4 if two_dim_input: diff --git a/src/diffusers/schedulers/scheduling_consistency_decoder.py b/src/diffusers/schedulers/scheduling_consistency_decoder.py index d7af018b284a..fb25ad5b7d0f 100644 --- a/src/diffusers/schedulers/scheduling_consistency_decoder.py +++ b/src/diffusers/schedulers/scheduling_consistency_decoder.py @@ -1,6 +1,6 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch @@ -97,7 +97,7 @@ def __init__( def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, ): if num_inference_steps != 2: raise ValueError("Currently more than 2 inference steps are not supported.") @@ -133,11 +133,11 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[ConsistencyDecoderSchedulerOutput, Tuple]: + ) -> ConsistencyDecoderSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_consistency_models.py b/src/diffusers/schedulers/scheduling_consistency_models.py index 5d81d5eb8ac0..79b365c44f7c 100644 --- a/src/diffusers/schedulers/scheduling_consistency_models.py +++ b/src/diffusers/schedulers/scheduling_consistency_models.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -66,7 +66,7 @@ class CMStochasticIterativeScheduler(SchedulerMixin, ConfigMixin): [paper](https://huggingface.co/papers/2206.00364). Defaults to 7.0 from the original implementation. clip_denoised (`bool`, defaults to `True`): Whether to clip the denoised outputs to `(-1, 1)`. - timesteps (`List` or `np.ndarray` or `torch.Tensor`, *optional*): + timesteps (`list` or `np.ndarray` or `torch.Tensor`, *optional*): An explicit timestep schedule that can be optionally specified. The timesteps are expected to be in increasing order. """ @@ -126,7 +126,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Scales the consistency model input by `(sigma**2 + sigma_data**2) ** 0.5`. @@ -151,7 +151,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T self.is_scale_input_called = True return sample - def sigma_to_t(self, sigmas: Union[float, np.ndarray]): + def sigma_to_t(self, sigmas: float | np.ndarray): """ Gets scaled timesteps from the Karras sigmas for input to the consistency model. @@ -173,8 +173,8 @@ def sigma_to_t(self, sigmas: Union[float, np.ndarray]): def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, - timesteps: Optional[List[int]] = None, + device: str | torch.device = None, + timesteps: Optional[list[int]] = None, ): """ Sets the timesteps used for the diffusion chain (to be run before inference). @@ -184,7 +184,7 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -313,11 +313,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[CMStochasticIterativeSchedulerOutput, Tuple]: + ) -> CMStochasticIterativeSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py b/src/diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py index b9567f2c47d5..3b2b4ae6fd93 100644 --- a/src/diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py +++ b/src/diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver and https://github.com/NVlabs/edm import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -171,7 +171,7 @@ def precondition_outputs(self, sample, model_output, sigma): return denoised # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.scale_model_input - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. @@ -195,7 +195,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T self.is_scale_input_called = True return sample - def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int = None, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -359,7 +359,7 @@ def dpm_solver_first_order_update( def multistep_dpm_solver_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, ) -> torch.Tensor: @@ -367,7 +367,7 @@ def multistep_dpm_solver_second_order_update( One step for the second-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by the diffusion process. @@ -451,11 +451,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep DPMSolver. diff --git a/src/diffusers/schedulers/scheduling_ddim.py b/src/diffusers/schedulers/scheduling_ddim.py index 5ee0d084f060..c637d187a729 100644 --- a/src/diffusers/schedulers/scheduling_ddim.py +++ b/src/diffusers/schedulers/scheduling_ddim.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -188,7 +188,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, clip_sample: bool = True, set_alpha_to_one: bool = True, steps_offset: int = 0, @@ -294,7 +294,7 @@ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -349,7 +349,7 @@ def step( generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[DDIMSchedulerOutput, Tuple]: + ) -> DDIMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddim_cogvideox.py b/src/diffusers/schedulers/scheduling_ddim_cogvideox.py index c19efdc7834d..6bcd52b52b05 100644 --- a/src/diffusers/schedulers/scheduling_ddim_cogvideox.py +++ b/src/diffusers/schedulers/scheduling_ddim_cogvideox.py @@ -18,7 +18,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -183,7 +183,7 @@ def __init__( beta_start: float = 0.00085, beta_end: float = 0.0120, beta_schedule: str = "scaled_linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, clip_sample: bool = True, set_alpha_to_one: bool = True, steps_offset: int = 0, @@ -257,7 +257,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -312,7 +312,7 @@ def step( generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[DDIMSchedulerOutput, Tuple]: + ) -> DDIMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddim_flax.py b/src/diffusers/schedulers/scheduling_ddim_flax.py index 802d8f79779d..51d5c9f71d1b 100644 --- a/src/diffusers/schedulers/scheduling_ddim_flax.py +++ b/src/diffusers/schedulers/scheduling_ddim_flax.py @@ -16,7 +16,7 @@ # and https://github.com/hojonathanho/diffusion from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax.numpy as jnp @@ -166,7 +166,7 @@ def scale_model_input( return sample def set_timesteps( - self, state: DDIMSchedulerState, num_inference_steps: int, shape: Tuple = () + self, state: DDIMSchedulerState, num_inference_steps: int, shape: tuple = () ) -> DDIMSchedulerState: """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -207,7 +207,7 @@ def step( sample: jnp.ndarray, eta: float = 0.0, return_dict: bool = True, - ) -> Union[FlaxDDIMSchedulerOutput, Tuple]: + ) -> FlaxDDIMSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddim_inverse.py b/src/diffusers/schedulers/scheduling_ddim_inverse.py index 49dba840d089..5d5e154d6055 100644 --- a/src/diffusers/schedulers/scheduling_ddim_inverse.py +++ b/src/diffusers/schedulers/scheduling_ddim_inverse.py @@ -16,7 +16,7 @@ # and https://github.com/hojonathanho/diffusion import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -180,7 +180,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, clip_sample: bool = True, set_alpha_to_one: bool = True, steps_offset: int = 0, @@ -248,7 +248,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -292,7 +292,7 @@ def step( timestep: int, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[DDIMSchedulerOutput, Tuple]: + ) -> DDIMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddim_parallel.py b/src/diffusers/schedulers/scheduling_ddim_parallel.py index 7c3f03a8dbe1..2cf5bb1d3828 100644 --- a/src/diffusers/schedulers/scheduling_ddim_parallel.py +++ b/src/diffusers/schedulers/scheduling_ddim_parallel.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -195,7 +195,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, clip_sample: bool = True, set_alpha_to_one: bool = True, steps_offset: int = 0, @@ -317,7 +317,7 @@ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: return sample # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler.set_timesteps - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -372,7 +372,7 @@ def step( generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[DDIMParallelSchedulerOutput, Tuple]: + ) -> DDIMParallelSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). @@ -490,7 +490,7 @@ def step( def batch_step_no_noise( self, model_output: torch.Tensor, - timesteps: List[int], + timesteps: list[int], sample: torch.Tensor, eta: float = 0.0, use_clipped_model_output: bool = False, @@ -505,7 +505,7 @@ def batch_step_no_noise( Args: model_output (`torch.Tensor`): direct output from learned diffusion model. - timesteps (`List[int]`): + timesteps (`list[int]`): current discrete timesteps in the diffusion chain. This is now a list of integers. sample (`torch.Tensor`): current instance of sample being created by diffusion process. diff --git a/src/diffusers/schedulers/scheduling_ddpm.py b/src/diffusers/schedulers/scheduling_ddpm.py index 0fab6d910a82..ac51aa2769fe 100644 --- a/src/diffusers/schedulers/scheduling_ddpm.py +++ b/src/diffusers/schedulers/scheduling_ddpm.py @@ -16,7 +16,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -184,7 +184,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, variance_type: str = "fixed_small", clip_sample: bool = True, prediction_type: str = "epsilon", @@ -251,8 +251,8 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, - timesteps: Optional[List[int]] = None, + device: str | torch.device = None, + timesteps: Optional[list[int]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -263,7 +263,7 @@ def set_timesteps( `timesteps` must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -402,7 +402,7 @@ def step( sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[DDPMSchedulerOutput, Tuple]: + ) -> DDPMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddpm_flax.py b/src/diffusers/schedulers/scheduling_ddpm_flax.py index a3264f54f572..cb0cc120910f 100644 --- a/src/diffusers/schedulers/scheduling_ddpm_flax.py +++ b/src/diffusers/schedulers/scheduling_ddpm_flax.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax @@ -137,7 +137,7 @@ def scale_model_input( return sample def set_timesteps( - self, state: DDPMSchedulerState, num_inference_steps: int, shape: Tuple = () + self, state: DDPMSchedulerState, num_inference_steps: int, shape: tuple = () ) -> DDPMSchedulerState: """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -200,7 +200,7 @@ def step( sample: jnp.ndarray, key: Optional[jax.Array] = None, return_dict: bool = True, - ) -> Union[FlaxDDPMSchedulerOutput, Tuple]: + ) -> FlaxDDPMSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ddpm_parallel.py b/src/diffusers/schedulers/scheduling_ddpm_parallel.py index ec741f9ecb7d..9a258e46fef2 100644 --- a/src/diffusers/schedulers/scheduling_ddpm_parallel.py +++ b/src/diffusers/schedulers/scheduling_ddpm_parallel.py @@ -16,7 +16,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -192,7 +192,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, variance_type: str = "fixed_small", clip_sample: bool = True, prediction_type: str = "epsilon", @@ -261,8 +261,8 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, - timesteps: Optional[List[int]] = None, + device: str | torch.device = None, + timesteps: Optional[list[int]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -273,7 +273,7 @@ def set_timesteps( `timesteps` must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -414,7 +414,7 @@ def step( sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[DDPMParallelSchedulerOutput, Tuple]: + ) -> DDPMParallelSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). @@ -509,7 +509,7 @@ def step( def batch_step_no_noise( self, model_output: torch.Tensor, - timesteps: List[int], + timesteps: list[int], sample: torch.Tensor, ) -> torch.Tensor: """ @@ -522,7 +522,7 @@ def batch_step_no_noise( Args: model_output (`torch.Tensor`): direct output from learned diffusion model. - timesteps (`List[int]`): + timesteps (`list[int]`): current discrete timesteps in the diffusion chain. This is now a list of integers. sample (`torch.Tensor`): current instance of sample being created by diffusion process. diff --git a/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py b/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py index 71f08277ebd7..fb6b4cdf5195 100644 --- a/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py +++ b/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -142,14 +142,14 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None def set_timesteps( self, num_inference_steps: int = None, - timesteps: Optional[List[int]] = None, - device: Union[str, torch.device] = None, + timesteps: Optional[list[int]] = None, + device: str | torch.device = None, ): """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. Args: - num_inference_steps (`Dict[float, int]`): + num_inference_steps (`dict[float, int]`): the number of diffusion steps used when generating samples with a pre-trained model. If passed, then `timesteps` must be `None`. device (`str` or `torch.device`, optional): @@ -168,7 +168,7 @@ def step( sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[DDPMWuerstchenSchedulerOutput, Tuple]: + ) -> DDPMWuerstchenSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_deis_multistep.py b/src/diffusers/schedulers/scheduling_deis_multistep.py index 7d8685ba10c3..caa91148c427 100644 --- a/src/diffusers/schedulers/scheduling_deis_multistep.py +++ b/src/diffusers/schedulers/scheduling_deis_multistep.py @@ -16,7 +16,7 @@ # The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -234,9 +234,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def set_timesteps( - self, num_inference_steps: int, device: Union[str, torch.device] = None, mu: Optional[float] = None - ): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None, mu: Optional[float] = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -586,7 +584,7 @@ def deis_first_order_update( def multistep_deis_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, **kwargs, @@ -595,7 +593,7 @@ def multistep_deis_second_order_update( One step for the second-order multistep DEIS. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by the diffusion process. @@ -655,7 +653,7 @@ def ind_fn(t, b, c): def multistep_deis_third_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, **kwargs, @@ -664,7 +662,7 @@ def multistep_deis_third_order_update( One step for the third-order multistep DEIS. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by diffusion process. @@ -777,10 +775,10 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep DEIS. diff --git a/src/diffusers/schedulers/scheduling_dpm_cogvideox.py b/src/diffusers/schedulers/scheduling_dpm_cogvideox.py index f7b63720e107..b0ab6ee4a7bd 100644 --- a/src/diffusers/schedulers/scheduling_dpm_cogvideox.py +++ b/src/diffusers/schedulers/scheduling_dpm_cogvideox.py @@ -18,7 +18,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -184,7 +184,7 @@ def __init__( beta_start: float = 0.00085, beta_end: float = 0.0120, beta_schedule: str = "scaled_linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, clip_sample: bool = True, set_alpha_to_one: bool = True, steps_offset: int = 0, @@ -258,7 +258,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -339,7 +339,7 @@ def step( generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = False, - ) -> Union[DDIMSchedulerOutput, Tuple]: + ) -> DDIMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py index 8b523cd13f1f..2ea577f353e9 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -208,7 +208,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, solver_order: int = 2, prediction_type: str = "epsilon", thresholding: bool = False, @@ -331,9 +331,9 @@ def set_begin_index(self, begin_index: int = 0): def set_timesteps( self, num_inference_steps: int = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, mu: Optional[float] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -343,7 +343,7 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary timesteps schedule. If `None`, timesteps will be generated based on the `timestep_spacing` attribute. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`, and `timestep_spacing` attribute will be ignored. @@ -787,7 +787,7 @@ def dpm_solver_first_order_update( def multistep_dpm_solver_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -797,7 +797,7 @@ def multistep_dpm_solver_second_order_update( One step for the second-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by the diffusion process. @@ -910,7 +910,7 @@ def multistep_dpm_solver_second_order_update( def multistep_dpm_solver_third_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -920,7 +920,7 @@ def multistep_dpm_solver_third_order_update( One step for the third-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by diffusion process. @@ -1037,12 +1037,12 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep DPMSolver. diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py index 71b9960bf2ff..784c6b73b810 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import flax import jax @@ -203,7 +203,7 @@ def create_state(self, common: Optional[CommonSchedulerState] = None) -> DPMSolv ) def set_timesteps( - self, state: DPMSolverMultistepSchedulerState, num_inference_steps: int, shape: Tuple + self, state: DPMSolverMultistepSchedulerState, num_inference_steps: int, shape: tuple ) -> DPMSolverMultistepSchedulerState: """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -213,7 +213,7 @@ def set_timesteps( the `FlaxDPMSolverMultistepScheduler` state data class instance. num_inference_steps (`int`): the number of diffusion steps used when generating samples with a pre-trained model. - shape (`Tuple`): + shape (`tuple`): the shape of the samples to be generated. """ last_timestep = self.config.num_train_timesteps @@ -365,7 +365,7 @@ def multistep_dpm_solver_second_order_update( self, state: DPMSolverMultistepSchedulerState, model_output_list: jnp.ndarray, - timestep_list: List[int], + timestep_list: list[int], prev_timestep: int, sample: jnp.ndarray, ) -> jnp.ndarray: @@ -373,7 +373,7 @@ def multistep_dpm_solver_second_order_update( One step for the second-order multistep DPM-Solver. Args: - model_output_list (`List[jnp.ndarray]`): + model_output_list (`list[jnp.ndarray]`): direct outputs from learned diffusion model at current and latter timesteps. timestep (`int`): current and latter discrete timestep in the diffusion chain. prev_timestep (`int`): previous discrete timestep in the diffusion chain. @@ -425,7 +425,7 @@ def multistep_dpm_solver_third_order_update( self, state: DPMSolverMultistepSchedulerState, model_output_list: jnp.ndarray, - timestep_list: List[int], + timestep_list: list[int], prev_timestep: int, sample: jnp.ndarray, ) -> jnp.ndarray: @@ -433,7 +433,7 @@ def multistep_dpm_solver_third_order_update( One step for the third-order multistep DPM-Solver. Args: - model_output_list (`List[jnp.ndarray]`): + model_output_list (`list[jnp.ndarray]`): direct outputs from learned diffusion model at current and latter timesteps. timestep (`int`): current and latter discrete timestep in the diffusion chain. prev_timestep (`int`): previous discrete timestep in the diffusion chain. @@ -484,7 +484,7 @@ def step( timestep: int, sample: jnp.ndarray, return_dict: bool = True, - ) -> Union[FlaxDPMSolverMultistepSchedulerOutput, Tuple]: + ) -> FlaxDPMSolverMultistepSchedulerOutput | tuple: """ Predict the sample at the previous timestep by DPM-Solver. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py index f1a1ac3d8216..b9917bcc76b2 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -156,7 +156,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, solver_order: int = 2, prediction_type: str = "epsilon", thresholding: bool = False, @@ -242,7 +242,7 @@ def step_index(self): """ return self._step_index - def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int = None, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -650,7 +650,7 @@ def dpm_solver_first_order_update( # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_second_order_update def multistep_dpm_solver_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -660,7 +660,7 @@ def multistep_dpm_solver_second_order_update( One step for the second-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by the diffusion process. @@ -774,7 +774,7 @@ def multistep_dpm_solver_second_order_update( # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_third_order_update def multistep_dpm_solver_third_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -784,7 +784,7 @@ def multistep_dpm_solver_third_order_update( One step for the third-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by diffusion process. @@ -889,12 +889,12 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, generator=None, variance_noise: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep DPMSolver. diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_sde.py b/src/diffusers/schedulers/scheduling_dpmsolver_sde.py index eeb06773d977..1ef28fca1e98 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_sde.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_sde.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -96,7 +96,7 @@ class BrownianTreeNoiseSampler: random samples. sigma_min (float): The low end of the valid interval. sigma_max (float): The high end of the valid interval. - seed (int or List[int]): The random seed. If a list of seeds is + seed (int or list[int]): The random seed. If a list of seeds is supplied instead of a single integer, then the noise sampler will use one BrownianTree per batch item, each with its own seed. transform (callable): A function that maps sigma to the sampler's @@ -209,7 +209,7 @@ def __init__( beta_start: float = 0.00085, # sensible defaults beta_end: float = 0.012, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, prediction_type: str = "epsilon", use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, @@ -309,7 +309,7 @@ def set_begin_index(self, begin_index: int = 0): def scale_model_input( self, sample: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, ) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the @@ -336,7 +336,7 @@ def scale_model_input( def set_timesteps( self, num_inference_steps: int, - device: Union[str, torch.device] = None, + device: str | torch.device = None, num_train_timesteps: Optional[int] = None, ): """ @@ -525,12 +525,12 @@ def state_in_first_order(self): def step( self, - model_output: Union[torch.Tensor, np.ndarray], - timestep: Union[float, torch.Tensor], - sample: Union[torch.Tensor, np.ndarray], + model_output: torch.Tensor | np.ndarray, + timestep: float | torch.Tensor, + sample: torch.Tensor | np.ndarray, return_dict: bool = True, s_noise: float = 1.0, - ) -> Union[DPMSolverSDESchedulerOutput, Tuple]: + ) -> DPMSolverSDESchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py b/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py index 1ae824973034..d8b91d73b0d5 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -234,7 +234,7 @@ def __init__( self._begin_index = None self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication - def get_order_list(self, num_inference_steps: int) -> List[int]: + def get_order_list(self, num_inference_steps: int) -> list[int]: """ Computes the solver order at each time step. @@ -302,9 +302,9 @@ def set_begin_index(self, begin_index: int = 0): def set_timesteps( self, num_inference_steps: int = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, mu: Optional[float] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -314,7 +314,7 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps schedule is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -718,7 +718,7 @@ def dpm_solver_first_order_update( def singlestep_dpm_solver_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -729,7 +729,7 @@ def singlestep_dpm_solver_second_order_update( time `timestep_list[-2]`. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. timestep (`int`): The current and latter discrete timestep in the diffusion chain. @@ -829,7 +829,7 @@ def singlestep_dpm_solver_second_order_update( def singlestep_dpm_solver_third_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, @@ -840,7 +840,7 @@ def singlestep_dpm_solver_third_order_update( time `timestep_list[-3]`. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. timestep (`int`): The current and latter discrete timestep in the diffusion chain. @@ -951,7 +951,7 @@ def singlestep_dpm_solver_third_order_update( def singlestep_dpm_solver_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], *args, sample: torch.Tensor = None, order: int = None, @@ -962,7 +962,7 @@ def singlestep_dpm_solver_update( One step for the singlestep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. timestep (`int`): The current and latter discrete timestep in the diffusion chain. @@ -1048,11 +1048,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the singlestep DPMSolver. diff --git a/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py b/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py index e9ba695e1f39..0c99469291b5 100644 --- a/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +++ b/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver and https://github.com/NVlabs/edm import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -206,7 +206,7 @@ def precondition_outputs(self, sample, model_output, sigma): return denoised # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.scale_model_input - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. @@ -230,7 +230,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T self.is_scale_input_called = True return sample - def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int = None, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -431,7 +431,7 @@ def dpm_solver_first_order_update( def multistep_dpm_solver_second_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], sample: torch.Tensor = None, noise: Optional[torch.Tensor] = None, ) -> torch.Tensor: @@ -439,7 +439,7 @@ def multistep_dpm_solver_second_order_update( One step for the second-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by the diffusion process. @@ -502,14 +502,14 @@ def multistep_dpm_solver_second_order_update( def multistep_dpm_solver_third_order_update( self, - model_output_list: List[torch.Tensor], + model_output_list: list[torch.Tensor], sample: torch.Tensor = None, ) -> torch.Tensor: """ One step for the third-order multistep DPMSolver. Args: - model_output_list (`List[torch.Tensor]`): + model_output_list (`list[torch.Tensor]`): The direct outputs from learned diffusion model at current and latter timesteps. sample (`torch.Tensor`): A current instance of a sample created by diffusion process. @@ -590,11 +590,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep DPMSolver. diff --git a/src/diffusers/schedulers/scheduling_edm_euler.py b/src/diffusers/schedulers/scheduling_edm_euler.py index dbeff3de5652..f5402e3cc97b 100644 --- a/src/diffusers/schedulers/scheduling_edm_euler.py +++ b/src/diffusers/schedulers/scheduling_edm_euler.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import torch @@ -188,7 +188,7 @@ def precondition_outputs(self, sample, model_output, sigma): return denoised - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. @@ -215,8 +215,8 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T def set_timesteps( self, num_inference_steps: int = None, - device: Union[str, torch.device] = None, - sigmas: Optional[Union[torch.Tensor, List[float]]] = None, + device: str | torch.device = None, + sigmas: Optional[torch.Tensor | list[float]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -226,7 +226,7 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - sigmas (`Union[torch.Tensor, List[float]]`, *optional*): + sigmas (`Union[torch.Tensor, list[float]]`, *optional*): Custom sigmas to use for the denoising process. If not defined, the default behavior when `num_inference_steps` is passed will be used. """ @@ -310,7 +310,7 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, s_churn: float = 0.0, s_tmin: float = 0.0, @@ -319,7 +319,7 @@ def step( generator: Optional[torch.Generator] = None, return_dict: bool = True, pred_original_sample: Optional[torch.Tensor] = None, - ) -> Union[EDMEulerSchedulerOutput, Tuple]: + ) -> EDMEulerSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py b/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py index 9cdaa2c5e101..7a54fe752859 100644 --- a/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py +++ b/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -173,7 +173,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, prediction_type: str = "epsilon", timestep_spacing: str = "linspace", steps_offset: int = 0, @@ -250,7 +250,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. @@ -274,7 +274,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T self.is_scale_input_called = True return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -345,11 +345,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[EulerAncestralDiscreteSchedulerOutput, Tuple]: + ) -> EulerAncestralDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_euler_discrete.py b/src/diffusers/schedulers/scheduling_euler_discrete.py index f58d918dbfbe..cafccf92cd1f 100644 --- a/src/diffusers/schedulers/scheduling_euler_discrete.py +++ b/src/diffusers/schedulers/scheduling_euler_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -190,7 +190,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, prediction_type: str = "epsilon", interpolation_type: str = "linear", use_karras_sigmas: Optional[bool] = False, @@ -292,7 +292,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. @@ -319,9 +319,9 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.T def set_timesteps( self, num_inference_steps: int = None, - device: Union[str, torch.device] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, + device: str | torch.device = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -331,11 +331,11 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary timesteps schedule. If `None`, timesteps will be generated based on the `timestep_spacing` attribute. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`, and `timestep_spacing` attribute will be ignored. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom sigmas used to support arbitrary timesteps schedule schedule. If `None`, timesteps and sigmas will be generated based on the relevant scheduler attributes. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`, and the timesteps will be generated based on the @@ -576,7 +576,7 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, s_churn: float = 0.0, s_tmin: float = 0.0, @@ -584,7 +584,7 @@ def step( s_noise: float = 1.0, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[EulerDiscreteSchedulerOutput, Tuple]: + ) -> EulerDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_euler_discrete_flax.py b/src/diffusers/schedulers/scheduling_euler_discrete_flax.py index dae01302ac6a..ef9e6c3940ec 100644 --- a/src/diffusers/schedulers/scheduling_euler_discrete_flax.py +++ b/src/diffusers/schedulers/scheduling_euler_discrete_flax.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax.numpy as jnp @@ -146,7 +146,7 @@ def scale_model_input(self, state: EulerDiscreteSchedulerState, sample: jnp.ndar return sample def set_timesteps( - self, state: EulerDiscreteSchedulerState, num_inference_steps: int, shape: Tuple = () + self, state: EulerDiscreteSchedulerState, num_inference_steps: int, shape: tuple = () ) -> EulerDiscreteSchedulerState: """ Sets the timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -193,7 +193,7 @@ def step( timestep: int, sample: jnp.ndarray, return_dict: bool = True, - ) -> Union[FlaxEulerDiscreteSchedulerOutput, Tuple]: + ) -> FlaxEulerDiscreteSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_flow_match_euler_discrete.py b/src/diffusers/schedulers/scheduling_flow_match_euler_discrete.py index 1a4f12ddfa53..44c34f09e45b 100644 --- a/src/diffusers/schedulers/scheduling_flow_match_euler_discrete.py +++ b/src/diffusers/schedulers/scheduling_flow_match_euler_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -171,7 +171,7 @@ def set_shift(self, shift: float): def scale_noise( self, sample: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, noise: Optional[torch.FloatTensor] = None, ) -> torch.FloatTensor: """ @@ -249,10 +249,10 @@ def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor: def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, - sigmas: Optional[List[float]] = None, + device: str | torch.device = None, + sigmas: Optional[list[float]] = None, mu: Optional[float] = None, - timesteps: Optional[List[float]] = None, + timesteps: Optional[list[float]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -262,13 +262,13 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed automatically. mu (`float`, *optional*): Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep shifting. - timesteps (`List[float]`, *optional*): + timesteps (`list[float]`, *optional*): Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed automatically. """ @@ -373,7 +373,7 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, sample: torch.FloatTensor, s_churn: float = 0.0, s_tmin: float = 0.0, @@ -382,7 +382,7 @@ def step( generator: Optional[torch.Generator] = None, per_token_timesteps: Optional[torch.Tensor] = None, return_dict: bool = True, - ) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]: + ) -> FlowMatchEulerDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_flow_match_heun_discrete.py b/src/diffusers/schedulers/scheduling_flow_match_heun_discrete.py index 38e5f1ba77a8..4b0fb6f96891 100644 --- a/src/diffusers/schedulers/scheduling_flow_match_heun_discrete.py +++ b/src/diffusers/schedulers/scheduling_flow_match_heun_discrete.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -110,7 +110,7 @@ def set_begin_index(self, begin_index: int = 0): def scale_noise( self, sample: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, noise: Optional[torch.FloatTensor] = None, ) -> torch.FloatTensor: """ @@ -137,7 +137,7 @@ def scale_noise( def _sigma_to_t(self, sigma): return sigma * self.config.num_train_timesteps - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -200,7 +200,7 @@ def state_in_first_order(self): def step( self, model_output: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, sample: torch.FloatTensor, s_churn: float = 0.0, s_tmin: float = 0.0, @@ -208,7 +208,7 @@ def step( s_noise: float = 1.0, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[FlowMatchHeunDiscreteSchedulerOutput, Tuple]: + ) -> FlowMatchHeunDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_flow_match_lcm.py b/src/diffusers/schedulers/scheduling_flow_match_lcm.py index 933bb1cf8e3d..3c3a318ed81a 100644 --- a/src/diffusers/schedulers/scheduling_flow_match_lcm.py +++ b/src/diffusers/schedulers/scheduling_flow_match_lcm.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -106,7 +106,7 @@ def __init__( use_exponential_sigmas: Optional[bool] = False, use_beta_sigmas: Optional[bool] = False, time_shift_type: str = "exponential", - scale_factors: Optional[List[float]] = None, + scale_factors: Optional[list[float]] = None, upscale_mode: Optional[str] = "bicubic", ): if self.config.use_beta_sigmas and not is_scipy_available(): @@ -192,7 +192,7 @@ def set_scale_factors(self, scale_factors: list, upscale_mode): def scale_noise( self, sample: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, noise: Optional[torch.FloatTensor] = None, ) -> torch.FloatTensor: """ @@ -270,10 +270,10 @@ def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor: def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, - sigmas: Optional[List[float]] = None, + device: str | torch.device = None, + sigmas: Optional[list[float]] = None, mu: Optional[float] = None, - timesteps: Optional[List[float]] = None, + timesteps: Optional[list[float]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -283,13 +283,13 @@ def set_timesteps( The number of diffusion steps used when generating samples with a pre-trained model. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - sigmas (`List[float]`, *optional*): + sigmas (`list[float]`, *optional*): Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed automatically. mu (`float`, *optional*): Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep shifting. - timesteps (`List[float]`, *optional*): + timesteps (`list[float]`, *optional*): Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed automatically. """ @@ -394,11 +394,11 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], + timestep: float | torch.FloatTensor, sample: torch.FloatTensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[FlowMatchLCMSchedulerOutput, Tuple]: + ) -> FlowMatchLCMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_heun_discrete.py b/src/diffusers/schedulers/scheduling_heun_discrete.py index bd1239cfaec7..eb98324f3ea6 100644 --- a/src/diffusers/schedulers/scheduling_heun_discrete.py +++ b/src/diffusers/schedulers/scheduling_heun_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -144,7 +144,7 @@ def __init__( beta_start: float = 0.00085, # sensible defaults beta_end: float = 0.012, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, prediction_type: str = "epsilon", use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, @@ -237,7 +237,7 @@ def set_begin_index(self, begin_index: int = 0): def scale_model_input( self, sample: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, ) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the @@ -263,9 +263,9 @@ def scale_model_input( def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, num_train_timesteps: Optional[int] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -278,7 +278,7 @@ def set_timesteps( num_train_timesteps (`int`, *optional*): The number of diffusion steps used when training the model. If `None`, the default `num_train_timesteps` attribute is used. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, timesteps will be generated based on the `timestep_spacing` attribute. If `timesteps` is passed, `num_inference_steps` must be `None`, and `timestep_spacing` attribute will be ignored. @@ -471,11 +471,11 @@ def _init_step_index(self, timestep): def step( self, - model_output: Union[torch.Tensor, np.ndarray], - timestep: Union[float, torch.Tensor], - sample: Union[torch.Tensor, np.ndarray], + model_output: torch.Tensor | np.ndarray, + timestep: float | torch.Tensor, + sample: torch.Tensor | np.ndarray, return_dict: bool = True, - ) -> Union[HeunDiscreteSchedulerOutput, Tuple]: + ) -> HeunDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_ipndm.py b/src/diffusers/schedulers/scheduling_ipndm.py index 23bc21f10ca4..dba2ba0b29e2 100644 --- a/src/diffusers/schedulers/scheduling_ipndm.py +++ b/src/diffusers/schedulers/scheduling_ipndm.py @@ -13,7 +13,7 @@ # limitations under the License. import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -39,9 +39,7 @@ class IPNDMScheduler(SchedulerMixin, ConfigMixin): order = 1 @register_to_config - def __init__( - self, num_train_timesteps: int = 1000, trained_betas: Optional[Union[np.ndarray, List[float]]] = None - ): + def __init__(self, num_train_timesteps: int = 1000, trained_betas: Optional[np.ndarray | list[float]] = None): # set `betas`, `alphas`, `timesteps` self.set_timesteps(num_train_timesteps) @@ -83,7 +81,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -138,10 +136,10 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the linear multistep method. It performs one forward pass multiple times to approximate the solution. diff --git a/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py b/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py index 6588464073a1..d3b440ae3845 100644 --- a/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +++ b/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -142,7 +142,7 @@ def __init__( beta_start: float = 0.00085, # sensible defaults beta_end: float = 0.012, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, use_beta_sigmas: Optional[bool] = False, @@ -214,7 +214,7 @@ def set_begin_index(self, begin_index: int = 0): def scale_model_input( self, sample: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, ) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the @@ -244,7 +244,7 @@ def scale_model_input( def set_timesteps( self, num_inference_steps: int, - device: Union[str, torch.device] = None, + device: str | torch.device = None, num_train_timesteps: Optional[int] = None, ): """ @@ -474,12 +474,12 @@ def _init_step_index(self, timestep): def step( self, - model_output: Union[torch.Tensor, np.ndarray], - timestep: Union[float, torch.Tensor], - sample: Union[torch.Tensor, np.ndarray], + model_output: torch.Tensor | np.ndarray, + timestep: float | torch.Tensor, + sample: torch.Tensor | np.ndarray, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[KDPM2AncestralDiscreteSchedulerOutput, Tuple]: + ) -> KDPM2AncestralDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py b/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py index 9b4cd4e204d6..16992f772ce0 100644 --- a/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py +++ b/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -141,7 +141,7 @@ def __init__( beta_start: float = 0.00085, # sensible defaults beta_end: float = 0.012, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, use_beta_sigmas: Optional[bool] = False, @@ -214,7 +214,7 @@ def set_begin_index(self, begin_index: int = 0): def scale_model_input( self, sample: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, ) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the @@ -244,7 +244,7 @@ def scale_model_input( def set_timesteps( self, num_inference_steps: int, - device: Union[str, torch.device] = None, + device: str | torch.device = None, num_train_timesteps: Optional[int] = None, ): """ @@ -459,11 +459,11 @@ def _convert_to_beta( def step( self, - model_output: Union[torch.Tensor, np.ndarray], - timestep: Union[float, torch.Tensor], - sample: Union[torch.Tensor, np.ndarray], + model_output: torch.Tensor | np.ndarray, + timestep: float | torch.Tensor, + sample: torch.Tensor | np.ndarray, return_dict: bool = True, - ) -> Union[KDPM2DiscreteSchedulerOutput, Tuple]: + ) -> KDPM2DiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_karras_ve_flax.py b/src/diffusers/schedulers/scheduling_karras_ve_flax.py index bacfbd61006d..b56f2a8b0402 100644 --- a/src/diffusers/schedulers/scheduling_karras_ve_flax.py +++ b/src/diffusers/schedulers/scheduling_karras_ve_flax.py @@ -14,7 +14,7 @@ from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax @@ -108,7 +108,7 @@ def create_state(self): return KarrasVeSchedulerState.create() def set_timesteps( - self, state: KarrasVeSchedulerState, num_inference_steps: int, shape: Tuple = () + self, state: KarrasVeSchedulerState, num_inference_steps: int, shape: tuple = () ) -> KarrasVeSchedulerState: """ Sets the continuous timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -141,7 +141,7 @@ def add_noise_to_input( sample: jnp.ndarray, sigma: float, key: jax.Array, - ) -> Tuple[jnp.ndarray, float]: + ) -> tuple[jnp.ndarray, float]: """ Explicit Langevin-like "churn" step of adding noise to the sample according to a factor gamma_i ≥ 0 to reach a higher noise level sigma_hat = sigma_i + gamma_i*sigma_i. @@ -169,7 +169,7 @@ def step( sigma_prev: float, sample_hat: jnp.ndarray, return_dict: bool = True, - ) -> Union[FlaxKarrasVeOutput, Tuple]: + ) -> FlaxKarrasVeOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). @@ -207,7 +207,7 @@ def step_correct( sample_prev: jnp.ndarray, derivative: jnp.ndarray, return_dict: bool = True, - ) -> Union[FlaxKarrasVeOutput, Tuple]: + ) -> FlaxKarrasVeOutput | tuple: """ Correct the predicted sample based on the output model_output of the network. TODO complete description diff --git a/src/diffusers/schedulers/scheduling_lcm.py b/src/diffusers/schedulers/scheduling_lcm.py index cd7a29fe675f..bdabfed9b7c3 100644 --- a/src/diffusers/schedulers/scheduling_lcm.py +++ b/src/diffusers/schedulers/scheduling_lcm.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -199,7 +199,7 @@ def __init__( beta_start: float = 0.00085, beta_end: float = 0.012, beta_schedule: str = "scaled_linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, original_inference_steps: int = 50, clip_sample: bool = False, clip_sample_range: float = 1.0, @@ -349,9 +349,9 @@ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, original_inference_steps: Optional[int] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, strength: int = 1.0, ): """ @@ -368,7 +368,7 @@ def set_timesteps( schedule (which is different from the standard `diffusers` implementation). We will then take `num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps on the training/distillation timestep schedule is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -501,7 +501,7 @@ def step( sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[LCMSchedulerOutput, Tuple]: + ) -> LCMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_lms_discrete.py b/src/diffusers/schedulers/scheduling_lms_discrete.py index c2450204aa8f..09e5281a1e75 100644 --- a/src/diffusers/schedulers/scheduling_lms_discrete.py +++ b/src/diffusers/schedulers/scheduling_lms_discrete.py @@ -14,7 +14,7 @@ import math import warnings from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import scipy.stats @@ -138,7 +138,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, use_beta_sigmas: Optional[bool] = False, @@ -214,7 +214,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + def scale_model_input(self, sample: torch.Tensor, timestep: float | torch.Tensor) -> torch.Tensor: """ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the current timestep. @@ -260,7 +260,7 @@ def lms_derivative(tau): return integrated_coeff - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -437,11 +437,11 @@ def _convert_to_beta( def step( self, model_output: torch.Tensor, - timestep: Union[float, torch.Tensor], + timestep: float | torch.Tensor, sample: torch.Tensor, order: int = 4, return_dict: bool = True, - ) -> Union[LMSDiscreteSchedulerOutput, Tuple]: + ) -> LMSDiscreteSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_lms_discrete_flax.py b/src/diffusers/schedulers/scheduling_lms_discrete_flax.py index b8e08ff9e134..25232d97d890 100644 --- a/src/diffusers/schedulers/scheduling_lms_discrete_flax.py +++ b/src/diffusers/schedulers/scheduling_lms_discrete_flax.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax.numpy as jnp @@ -165,7 +165,7 @@ def lms_derivative(tau): return integrated_coeff def set_timesteps( - self, state: LMSDiscreteSchedulerState, num_inference_steps: int, shape: Tuple = () + self, state: LMSDiscreteSchedulerState, num_inference_steps: int, shape: tuple = () ) -> LMSDiscreteSchedulerState: """ Sets the timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -208,7 +208,7 @@ def step( sample: jnp.ndarray, order: int = 4, return_dict: bool = True, - ) -> Union[FlaxLMSSchedulerOutput, Tuple]: + ) -> FlaxLMSSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_pndm.py b/src/diffusers/schedulers/scheduling_pndm.py index c07621179e2b..5f2734d7660b 100644 --- a/src/diffusers/schedulers/scheduling_pndm.py +++ b/src/diffusers/schedulers/scheduling_pndm.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -117,7 +117,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, skip_prk_steps: bool = False, set_alpha_to_one: bool = False, prediction_type: str = "epsilon", @@ -163,7 +163,7 @@ def __init__( self.plms_timesteps = None self.timesteps = None - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -229,7 +229,7 @@ def step( timestep: int, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise), and calls [`~PNDMScheduler.step_prk`] @@ -262,7 +262,7 @@ def step_prk( timestep: int, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the Runge-Kutta method. It performs four forward passes to approximate the solution to the differential @@ -322,7 +322,7 @@ def step_plms( timestep: int, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the linear multistep method. It performs one forward pass multiple times to approximate the solution. diff --git a/src/diffusers/schedulers/scheduling_pndm_flax.py b/src/diffusers/schedulers/scheduling_pndm_flax.py index 12e22005afaf..22c879cac445 100644 --- a/src/diffusers/schedulers/scheduling_pndm_flax.py +++ b/src/diffusers/schedulers/scheduling_pndm_flax.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax @@ -162,7 +162,7 @@ def create_state(self, common: Optional[CommonSchedulerState] = None) -> PNDMSch timesteps=timesteps, ) - def set_timesteps(self, state: PNDMSchedulerState, num_inference_steps: int, shape: Tuple) -> PNDMSchedulerState: + def set_timesteps(self, state: PNDMSchedulerState, num_inference_steps: int, shape: tuple) -> PNDMSchedulerState: """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -171,7 +171,7 @@ def set_timesteps(self, state: PNDMSchedulerState, num_inference_steps: int, sha the `FlaxPNDMScheduler` state data class instance. num_inference_steps (`int`): the number of diffusion steps used when generating samples with a pre-trained model. - shape (`Tuple`): + shape (`tuple`): the shape of the samples to be generated. """ @@ -241,7 +241,7 @@ def step( timestep: int, sample: jnp.ndarray, return_dict: bool = True, - ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + ) -> FlaxPNDMSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). @@ -295,7 +295,7 @@ def step_prk( model_output: jnp.ndarray, timestep: int, sample: jnp.ndarray, - ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + ) -> FlaxPNDMSchedulerOutput | tuple: """ Step function propagating the sample with the Runge-Kutta method. RK takes 4 forward passes to approximate the solution to the differential equation. @@ -363,7 +363,7 @@ def step_plms( model_output: jnp.ndarray, timestep: int, sample: jnp.ndarray, - ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + ) -> FlaxPNDMSchedulerOutput | tuple: """ Step function propagating the sample with the linear multi-step method. This has one forward pass with multiple times to approximate the solution. diff --git a/src/diffusers/schedulers/scheduling_repaint.py b/src/diffusers/schedulers/scheduling_repaint.py index 6530c5af9e5b..de0a22f73431 100644 --- a/src/diffusers/schedulers/scheduling_repaint.py +++ b/src/diffusers/schedulers/scheduling_repaint.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -182,7 +182,7 @@ def set_timesteps( num_inference_steps: int, jump_length: int = 10, jump_n_sample: int = 10, - device: Union[str, torch.device] = None, + device: str | torch.device = None, ): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -252,7 +252,7 @@ def step( mask: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[RePaintSchedulerOutput, Tuple]: + ) -> RePaintSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_sasolver.py b/src/diffusers/schedulers/scheduling_sasolver.py index 2979ce193a36..9fadff6850e9 100644 --- a/src/diffusers/schedulers/scheduling_sasolver.py +++ b/src/diffusers/schedulers/scheduling_sasolver.py @@ -16,7 +16,7 @@ # The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py import math -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional import numpy as np import torch @@ -154,7 +154,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, predictor_order: int = 2, corrector_order: int = 2, prediction_type: str = "epsilon", @@ -258,7 +258,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int = None, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -1076,7 +1076,7 @@ def step( sample: torch.Tensor, generator=None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the SA-Solver. diff --git a/src/diffusers/schedulers/scheduling_scm.py b/src/diffusers/schedulers/scheduling_scm.py index 63b4a109ff9b..8e62671aeb3f 100644 --- a/src/diffusers/schedulers/scheduling_scm.py +++ b/src/diffusers/schedulers/scheduling_scm.py @@ -16,7 +16,7 @@ # and https://github.com/hojonathanho/diffusion from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -118,7 +118,7 @@ def set_timesteps( self, num_inference_steps: int, timesteps: torch.Tensor = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, max_timesteps: float = 1.57080, intermediate_timesteps: float = 1.3, ): @@ -203,7 +203,7 @@ def step( sample: torch.FloatTensor, generator: torch.Generator = None, return_dict: bool = True, - ) -> Union[SCMSchedulerOutput, Tuple]: + ) -> SCMSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_sde_ve.py b/src/diffusers/schedulers/scheduling_sde_ve.py index 1bfc08cce5e9..afe8fbfea087 100644 --- a/src/diffusers/schedulers/scheduling_sde_ve.py +++ b/src/diffusers/schedulers/scheduling_sde_ve.py @@ -16,7 +16,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import torch @@ -103,9 +103,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps( - self, num_inference_steps: int, sampling_eps: float = None, device: Union[str, torch.device] = None - ): + def set_timesteps(self, num_inference_steps: int, sampling_eps: float = None, device: str | torch.device = None): """ Sets the continuous timesteps used for the diffusion chain (to be run before inference). @@ -164,7 +162,7 @@ def step_pred( sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[SdeVeOutput, Tuple]: + ) -> SdeVeOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). @@ -231,7 +229,7 @@ def step_correct( sample: torch.Tensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Correct the predicted sample based on the `model_output` of the network. This is often run repeatedly after making the prediction for the previous timestep. diff --git a/src/diffusers/schedulers/scheduling_sde_ve_flax.py b/src/diffusers/schedulers/scheduling_sde_ve_flax.py index 09cd081462b3..9fdcf78302bf 100644 --- a/src/diffusers/schedulers/scheduling_sde_ve_flax.py +++ b/src/diffusers/schedulers/scheduling_sde_ve_flax.py @@ -15,7 +15,7 @@ # DISCLAIMER: This file is strongly influenced by https://github.com/yang-song/score_sde_pytorch from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax @@ -108,7 +108,7 @@ def create_state(self): ) def set_timesteps( - self, state: ScoreSdeVeSchedulerState, num_inference_steps: int, shape: Tuple = (), sampling_eps: float = None + self, state: ScoreSdeVeSchedulerState, num_inference_steps: int, shape: tuple = (), sampling_eps: float = None ) -> ScoreSdeVeSchedulerState: """ Sets the continuous timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -172,7 +172,7 @@ def step_pred( sample: jnp.ndarray, key: jax.Array, return_dict: bool = True, - ) -> Union[FlaxSdeVeOutput, Tuple]: + ) -> FlaxSdeVeOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). @@ -231,7 +231,7 @@ def step_correct( sample: jnp.ndarray, key: jax.Array, return_dict: bool = True, - ) -> Union[FlaxSdeVeOutput, Tuple]: + ) -> FlaxSdeVeOutput | tuple: """ Correct the predicted sample based on the output model_output of the network. This is often run repeatedly after making the prediction for the previous timestep. diff --git a/src/diffusers/schedulers/scheduling_tcd.py b/src/diffusers/schedulers/scheduling_tcd.py index 3fd5c341eca9..6639a11a3f08 100644 --- a/src/diffusers/schedulers/scheduling_tcd.py +++ b/src/diffusers/schedulers/scheduling_tcd.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -200,7 +200,7 @@ def __init__( beta_start: float = 0.00085, beta_end: float = 0.012, beta_schedule: str = "scaled_linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, original_inference_steps: int = 50, clip_sample: bool = False, clip_sample_range: float = 1.0, @@ -362,9 +362,9 @@ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: def set_timesteps( self, num_inference_steps: Optional[int] = None, - device: Union[str, torch.device] = None, + device: str | torch.device = None, original_inference_steps: Optional[int] = None, - timesteps: Optional[List[int]] = None, + timesteps: Optional[list[int]] = None, strength: float = 1.0, ): """ @@ -381,7 +381,7 @@ def set_timesteps( schedule (which is different from the standard `diffusers` implementation). We will then take `num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute. - timesteps (`List[int]`, *optional*): + timesteps (`list[int]`, *optional*): Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default timestep spacing strategy of equal spacing between timesteps on the training/distillation timestep schedule is used. If `timesteps` is passed, `num_inference_steps` must be `None`. @@ -529,7 +529,7 @@ def step( eta: float = 0.3, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[TCDSchedulerOutput, Tuple]: + ) -> TCDSchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_unclip.py b/src/diffusers/schedulers/scheduling_unclip.py index d78efabfbc57..d4e161267238 100644 --- a/src/diffusers/schedulers/scheduling_unclip.py +++ b/src/diffusers/schedulers/scheduling_unclip.py @@ -14,7 +14,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -160,7 +160,7 @@ def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None """ return sample - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. @@ -221,7 +221,7 @@ def step( prev_timestep: Optional[int] = None, generator=None, return_dict: bool = True, - ) -> Union[UnCLIPSchedulerOutput, Tuple]: + ) -> UnCLIPSchedulerOutput | tuple: """ Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process from the learned model outputs (most often the predicted noise). diff --git a/src/diffusers/schedulers/scheduling_unipc_multistep.py b/src/diffusers/schedulers/scheduling_unipc_multistep.py index 162a34bd2774..ba541e02ca27 100644 --- a/src/diffusers/schedulers/scheduling_unipc_multistep.py +++ b/src/diffusers/schedulers/scheduling_unipc_multistep.py @@ -16,7 +16,7 @@ # The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py import math -from typing import List, Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -194,7 +194,7 @@ def __init__( beta_start: float = 0.0001, beta_end: float = 0.02, beta_schedule: str = "linear", - trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + trained_betas: Optional[np.ndarray | list[float]] = None, solver_order: int = 2, prediction_type: str = "epsilon", thresholding: bool = False, @@ -203,7 +203,7 @@ def __init__( predict_x0: bool = True, solver_type: str = "bh2", lower_order_final: bool = True, - disable_corrector: List[int] = [], + disable_corrector: list[int] = [], solver_p: SchedulerMixin = None, use_karras_sigmas: Optional[bool] = False, use_exponential_sigmas: Optional[bool] = False, @@ -302,9 +302,7 @@ def set_begin_index(self, begin_index: int = 0): """ self._begin_index = begin_index - def set_timesteps( - self, num_inference_steps: int, device: Union[str, torch.device] = None, mu: Optional[float] = None - ): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None, mu: Optional[float] = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -953,10 +951,10 @@ def _init_step_index(self, timestep): def step( self, model_output: torch.Tensor, - timestep: Union[int, torch.Tensor], + timestep: int | torch.Tensor, sample: torch.Tensor, return_dict: bool = True, - ) -> Union[SchedulerOutput, Tuple]: + ) -> SchedulerOutput | tuple: """ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with the multistep UniPC. diff --git a/src/diffusers/schedulers/scheduling_utils.py b/src/diffusers/schedulers/scheduling_utils.py index a355c7bb1a51..f90f5e7c2ac0 100644 --- a/src/diffusers/schedulers/scheduling_utils.py +++ b/src/diffusers/schedulers/scheduling_utils.py @@ -15,7 +15,7 @@ import os from dataclasses import dataclass from enum import Enum -from typing import Optional, Union +from typing import Optional import torch from huggingface_hub.utils import validate_hf_hub_args @@ -83,7 +83,7 @@ class SchedulerMixin(PushToHubMixin): the scheduler's `__init__` function, and the attributes can be accessed by `scheduler.config.num_train_timesteps`. Class attributes: - - **_compatibles** (`List[str]`) -- A list of scheduler classes that are compatible with the parent scheduler + - **_compatibles** (`list[str]`) -- A list of scheduler classes that are compatible with the parent scheduler class. Use [`~ConfigMixin.from_config`] to load a different compatible scheduler class (should be overridden by parent class). """ @@ -96,7 +96,7 @@ class SchedulerMixin(PushToHubMixin): @validate_hf_hub_args def from_pretrained( cls, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: Optional[str | os.PathLike] = None, subfolder: Optional[str] = None, return_unused_kwargs=False, **kwargs, @@ -123,7 +123,7 @@ def from_pretrained( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -153,7 +153,7 @@ def from_pretrained( ) return cls.from_config(config, return_unused_kwargs=return_unused_kwargs, **kwargs) - def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save a scheduler configuration object to a directory so that it can be reloaded using the [`~SchedulerMixin.from_pretrained`] class method. @@ -165,7 +165,7 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ self.save_config(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) @@ -176,7 +176,7 @@ def compatibles(self): Returns all schedulers that are compatible with this scheduler Returns: - `List[SchedulerMixin]`: List of compatible schedulers + `list[SchedulerMixin]`: list of compatible schedulers """ return self._get_compatibles() diff --git a/src/diffusers/schedulers/scheduling_utils_flax.py b/src/diffusers/schedulers/scheduling_utils_flax.py index 0534e47d8a30..ec49e6adb6b1 100644 --- a/src/diffusers/schedulers/scheduling_utils_flax.py +++ b/src/diffusers/schedulers/scheduling_utils_flax.py @@ -16,7 +16,7 @@ import os from dataclasses import dataclass from enum import Enum -from typing import Optional, Tuple, Union +from typing import Optional import flax import jax.numpy as jnp @@ -62,7 +62,7 @@ class FlaxSchedulerMixin(PushToHubMixin): Mixin containing common functions for the schedulers. Class attributes: - - **_compatibles** (`List[str]`) -- A list of classes that are compatible with the parent class, so that + - **_compatibles** (`list[str]`) -- A list of classes that are compatible with the parent class, so that `from_config` can be used from a class different than the one used to save the config (should be overridden by parent class). """ @@ -76,7 +76,7 @@ class FlaxSchedulerMixin(PushToHubMixin): @validate_hf_hub_args def from_pretrained( cls, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: Optional[str | os.PathLike] = None, subfolder: Optional[str] = None, return_unused_kwargs=False, **kwargs, @@ -105,7 +105,7 @@ def from_pretrained( Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): @@ -148,7 +148,7 @@ def from_pretrained( return scheduler, state - def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs): """ Save a scheduler configuration object to the directory `save_directory`, so that it can be re-loaded using the [`~FlaxSchedulerMixin.from_pretrained`] class method. @@ -160,7 +160,7 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). - kwargs (`Dict[str, Any]`, *optional*): + kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. """ self.save_config(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) @@ -171,7 +171,7 @@ def compatibles(self): Returns all schedulers that are compatible with this scheduler Returns: - `List[SchedulerMixin]`: List of compatible schedulers + `list[SchedulerMixin]`: list of compatible schedulers """ return self._get_compatibles() @@ -185,7 +185,7 @@ def _get_compatibles(cls): return compatible_classes -def broadcast_to_shape_from_left(x: jnp.ndarray, shape: Tuple[int]) -> jnp.ndarray: +def broadcast_to_shape_from_left(x: jnp.ndarray, shape: tuple[int]) -> jnp.ndarray: assert len(shape) >= x.ndim return jnp.broadcast_to(x.reshape(x.shape + (1,) * (len(shape) - x.ndim)), shape) diff --git a/src/diffusers/schedulers/scheduling_vq_diffusion.py b/src/diffusers/schedulers/scheduling_vq_diffusion.py index 57306301d023..c4983f72f976 100644 --- a/src/diffusers/schedulers/scheduling_vq_diffusion.py +++ b/src/diffusers/schedulers/scheduling_vq_diffusion.py @@ -13,7 +13,7 @@ # limitations under the License. from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional import numpy as np import torch @@ -175,7 +175,7 @@ def __init__( self.num_inference_steps = None self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) - def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + def set_timesteps(self, num_inference_steps: int, device: str | torch.device = None): """ Sets the discrete timesteps used for the diffusion chain (to be run before inference). @@ -204,7 +204,7 @@ def step( sample: torch.LongTensor, generator: Optional[torch.Generator] = None, return_dict: bool = True, - ) -> Union[VQDiffusionSchedulerOutput, Tuple]: + ) -> VQDiffusionSchedulerOutput | tuple: """ Predict the sample from the previous timestep by the reverse transition distribution. See [`~VQDiffusionScheduler.q_posterior`] for more details about how the distribution is computer. diff --git a/src/diffusers/training_utils.py b/src/diffusers/training_utils.py index 7a98fa3da14a..8fb549e308da 100644 --- a/src/diffusers/training_utils.py +++ b/src/diffusers/training_utils.py @@ -6,7 +6,7 @@ import re import warnings from contextlib import contextmanager -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Iterable, Optional import numpy as np import torch @@ -151,7 +151,7 @@ def compute_dream_and_update_latents( target: torch.Tensor, encoder_hidden_states: torch.Tensor, dream_detail_preservation: float = 1.0, -) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: +) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: """ Implements "DREAM (Diffusion Rectification and Estimation-Adaptive Models)" from https://huggingface.co/papers/2312.00210. DREAM helps align training with sampling to help training be more @@ -196,7 +196,7 @@ def compute_dream_and_update_latents( return _noisy_latents, _target -def unet_lora_state_dict(unet: UNet2DConditionModel) -> Dict[str, torch.Tensor]: +def unet_lora_state_dict(unet: UNet2DConditionModel) -> dict[str, torch.Tensor]: r""" Returns: A state dict containing just the LoRA parameters. @@ -215,7 +215,7 @@ def unet_lora_state_dict(unet: UNet2DConditionModel) -> Dict[str, torch.Tensor]: return lora_state_dict -def cast_training_params(model: Union[torch.nn.Module, List[torch.nn.Module]], dtype=torch.float32): +def cast_training_params(model: torch.nn.Module | list[torch.nn.Module], dtype=torch.float32): """ Casts the training parameters of the model to the specified data type. @@ -233,7 +233,7 @@ def cast_training_params(model: Union[torch.nn.Module, List[torch.nn.Module]], d def _set_state_dict_into_text_encoder( - lora_state_dict: Dict[str, torch.Tensor], prefix: str, text_encoder: torch.nn.Module + lora_state_dict: dict[str, torch.Tensor], prefix: str, text_encoder: torch.nn.Module ): """ Sets the `lora_state_dict` into `text_encoder` coming from `transformers`. @@ -251,7 +251,7 @@ def _set_state_dict_into_text_encoder( set_peft_model_state_dict(text_encoder, text_encoder_state_dict, adapter_name="default") -def _collate_lora_metadata(modules_to_save: Dict[str, torch.nn.Module]) -> Dict[str, Any]: +def _collate_lora_metadata(modules_to_save: dict[str, torch.nn.Module]) -> dict[str, Any]: metadatas = {} for module_name, module in modules_to_save.items(): if module is not None: @@ -265,7 +265,7 @@ def compute_density_for_timestep_sampling( logit_mean: float = None, logit_std: float = None, mode_scale: float = None, - device: Union[torch.device, str] = "cpu", + device: torch.device | str = "cpu", generator: Optional[torch.Generator] = None, ): """ @@ -321,9 +321,7 @@ def free_memory(): @contextmanager -def offload_models( - *modules: Union[torch.nn.Module, DiffusionPipeline], device: Union[str, torch.device], offload: bool = True -): +def offload_models(*modules: torch.nn.Module | DiffusionPipeline, device: str | torch.device, offload: bool = True): """ Context manager that, if offload=True, moves each module to `device` on enter, then moves it back to its original device on exit. @@ -407,11 +405,11 @@ def __init__( min_decay: float = 0.0, update_after_step: int = 0, use_ema_warmup: bool = False, - inv_gamma: Union[float, int] = 1.0, - power: Union[float, int] = 2 / 3, + inv_gamma: float | int = 1.0, + power: float | int = 2 / 3, foreach: bool = False, model_cls: Optional[Any] = None, - model_config: Dict[str, Any] = None, + model_config: dict[str, Any] = None, **kwargs, ): """ diff --git a/src/diffusers/utils/deprecation_utils.py b/src/diffusers/utils/deprecation_utils.py index 4f001b3047d6..71000805ec41 100644 --- a/src/diffusers/utils/deprecation_utils.py +++ b/src/diffusers/utils/deprecation_utils.py @@ -1,11 +1,11 @@ import inspect import warnings -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional from packaging import version -def deprecate(*args, take_from: Optional[Union[Dict, Any]] = None, standard_warn=True, stacklevel=2): +def deprecate(*args, take_from: Optional[Dict | Any] = None, standard_warn=True, stacklevel=2): from .. import __version__ deprecated_kwargs = take_from diff --git a/src/diffusers/utils/dynamic_modules_utils.py b/src/diffusers/utils/dynamic_modules_utils.py index 627b1e0604dc..6b97bcbcd230 100644 --- a/src/diffusers/utils/dynamic_modules_utils.py +++ b/src/diffusers/utils/dynamic_modules_utils.py @@ -24,7 +24,7 @@ import threading from pathlib import Path from types import ModuleType -from typing import Dict, Optional, Union +from typing import Optional from urllib import request from huggingface_hub import hf_hub_download, model_info @@ -65,7 +65,7 @@ def init_hf_modules(): init_path.touch() -def create_dynamic_module(name: Union[str, os.PathLike]): +def create_dynamic_module(name: str | os.PathLike): """ Creates a dynamic module in the cache directory for modules. """ @@ -245,13 +245,13 @@ def find_pipeline_class(loaded_module): @validate_hf_hub_args def get_cached_module_file( - pretrained_model_name_or_path: Union[str, os.PathLike], + pretrained_model_name_or_path: str | os.PathLike, module_file: str, subfolder: Optional[str] = None, - cache_dir: Optional[Union[str, os.PathLike]] = None, + cache_dir: Optional[str | os.PathLike] = None, force_download: bool = False, - proxies: Optional[Dict[str, str]] = None, - token: Optional[Union[bool, str]] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[bool | str] = None, revision: Optional[str] = None, local_files_only: bool = False, ): @@ -277,7 +277,7 @@ def get_cached_module_file( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or *bool*, *optional*): @@ -421,14 +421,14 @@ def get_cached_module_file( @validate_hf_hub_args def get_class_from_dynamic_module( - pretrained_model_name_or_path: Union[str, os.PathLike], + pretrained_model_name_or_path: str | os.PathLike, module_file: str, subfolder: Optional[str] = None, class_name: Optional[str] = None, - cache_dir: Optional[Union[str, os.PathLike]] = None, + cache_dir: Optional[str | os.PathLike] = None, force_download: bool = False, - proxies: Optional[Dict[str, str]] = None, - token: Optional[Union[bool, str]] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[bool | str] = None, revision: Optional[str] = None, local_files_only: bool = False, **kwargs, @@ -459,7 +459,7 @@ def get_class_from_dynamic_module( force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): + proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or `bool`, *optional*): diff --git a/src/diffusers/utils/export_utils.py b/src/diffusers/utils/export_utils.py index 07cf46928a44..9bf3159d92e3 100644 --- a/src/diffusers/utils/export_utils.py +++ b/src/diffusers/utils/export_utils.py @@ -3,7 +3,7 @@ import struct import tempfile from contextlib import contextmanager -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL.Image @@ -25,7 +25,7 @@ def buffered_writer(raw_f): f.flush() -def export_to_gif(image: List[PIL.Image.Image], output_gif_path: str = None, fps: int = 10) -> str: +def export_to_gif(image: list[PIL.Image.Image], output_gif_path: str = None, fps: int = 10) -> str: if output_gif_path is None: output_gif_path = tempfile.NamedTemporaryFile(suffix=".gif").name @@ -113,7 +113,7 @@ def export_to_obj(mesh, output_obj_path: str = None): def _legacy_export_to_video( - video_frames: Union[List[np.ndarray], List[PIL.Image.Image]], output_video_path: str = None, fps: int = 10 + video_frames: list[np.ndarray] | list[PIL.Image.Image], output_video_path: str = None, fps: int = 10 ): if is_opencv_available(): import cv2 @@ -139,7 +139,7 @@ def _legacy_export_to_video( def export_to_video( - video_frames: Union[List[np.ndarray], List[PIL.Image.Image]], + video_frames: list[np.ndarray] | list[PIL.Image.Image], output_video_path: str = None, fps: int = 10, quality: float = 5.0, diff --git a/src/diffusers/utils/hub_utils.py b/src/diffusers/utils/hub_utils.py index d0b05c7d9541..cbc65bd3f3a1 100644 --- a/src/diffusers/utils/hub_utils.py +++ b/src/diffusers/utils/hub_utils.py @@ -21,7 +21,7 @@ import tempfile import warnings from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Optional from uuid import uuid4 from huggingface_hub import ( @@ -72,7 +72,7 @@ SESSION_ID = uuid4().hex -def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: +def http_user_agent(user_agent: Dict | str | None = None) -> str: """ Formats a user-agent string with basic info about a request. """ @@ -105,7 +105,7 @@ def load_or_create_model_card( base_model: str = None, prompt: Optional[str] = None, license: Optional[str] = None, - widget: Optional[List[dict]] = None, + widget: Optional[list[dict]] = None, inference: Optional[bool] = None, ) -> ModelCard: """ @@ -128,7 +128,7 @@ def load_or_create_model_card( prompt (`str`, *optional*): Prompt used for training. Useful for DreamBooth-like training. license: (`str`, *optional*): License of the output artifact. Helpful when using `load_or_create_model_card` from a training script. - widget (`List[dict]`, *optional*): Widget to accompany a gallery template. + widget (`list[dict]`, *optional*): Widget to accompany a gallery template. inference: (`bool`, optional): Whether to turn on inference widget. Helpful when using `load_or_create_model_card` from a training script. """ @@ -167,7 +167,7 @@ def load_or_create_model_card( return model_card -def populate_model_card(model_card: ModelCard, tags: Union[str, List[str]] = None) -> ModelCard: +def populate_model_card(model_card: ModelCard, tags: str | list[str] = None) -> ModelCard: """Populates the `model_card` with library name and optional tags.""" if model_card.data.library_name is None: model_card.data.library_name = "diffusers" @@ -208,7 +208,7 @@ def _add_variant(weights_name: str, variant: Optional[str] = None) -> str: @validate_hf_hub_args def _get_model_file( - pretrained_model_name_or_path: Union[str, Path], + pretrained_model_name_or_path: str | Path, *, weights_name: str, subfolder: Optional[str] = None, @@ -217,10 +217,10 @@ def _get_model_file( proxies: Optional[Dict] = None, local_files_only: bool = False, token: Optional[str] = None, - user_agent: Optional[Union[Dict, str]] = None, + user_agent: Optional[Dict | str] = None, revision: Optional[str] = None, commit_hash: Optional[str] = None, - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, ): pretrained_model_name_or_path = str(pretrained_model_name_or_path) @@ -348,7 +348,7 @@ def _get_checkpoint_shard_files( user_agent=None, revision=None, subfolder="", - dduf_entries: Optional[Dict[str, DDUFEntry]] = None, + dduf_entries: Optional[dict[str, DDUFEntry]] = None, ): """ For a given model: @@ -449,7 +449,7 @@ def _get_checkpoint_shard_files( return cached_filenames, sharded_metadata -def _check_legacy_sharding_variant_format(folder: str = None, filenames: List[str] = None, variant: str = None): +def _check_legacy_sharding_variant_format(folder: str = None, filenames: list[str] = None, variant: str = None): if filenames and folder: raise ValueError("Both `filenames` and `folder` cannot be provided.") if not filenames: @@ -469,7 +469,7 @@ class PushToHubMixin: def _upload_folder( self, - working_dir: Union[str, os.PathLike], + working_dir: str | os.PathLike, repo_id: str, token: Optional[str] = None, commit_message: Optional[str] = None, diff --git a/src/diffusers/utils/import_utils.py b/src/diffusers/utils/import_utils.py index 97065267b004..d3684955e270 100644 --- a/src/diffusers/utils/import_utils.py +++ b/src/diffusers/utils/import_utils.py @@ -24,7 +24,7 @@ from functools import lru_cache as cache from itertools import chain from types import ModuleType -from typing import Any, Tuple, Union +from typing import Any from huggingface_hub.utils import is_jinja_available # noqa: F401 from packaging.version import Version, parse @@ -59,7 +59,7 @@ _is_google_colab = "google.colab" in sys.modules or any(k.startswith("COLAB_") for k in os.environ) -def _is_package_available(pkg_name: str, get_dist_name: bool = False) -> Tuple[bool, str]: +def _is_package_available(pkg_name: str, get_dist_name: bool = False) -> tuple[bool, str]: global _package_map pkg_exists = importlib.util.find_spec(pkg_name) is not None pkg_version = "N/A" @@ -653,7 +653,7 @@ def __getattr__(cls, key): # This function was copied from: https://github.com/huggingface/accelerate/blob/874c4967d94badd24f893064cc3bef45f57cadf7/src/accelerate/utils/versions.py#L319 -def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str): +def compare_versions(library_or_version: str | Version, operation: str, requirement_version: str): """ Compares a library version to some requirement using a given operation. diff --git a/src/diffusers/utils/loading_utils.py b/src/diffusers/utils/loading_utils.py index dd23ae73c861..d9be0d3c9884 100644 --- a/src/diffusers/utils/loading_utils.py +++ b/src/diffusers/utils/loading_utils.py @@ -1,6 +1,6 @@ import os import tempfile -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, Optional from urllib.parse import unquote, urlparse import PIL.Image @@ -12,7 +12,7 @@ def load_image( - image: Union[str, PIL.Image.Image], convert_method: Optional[Callable[[PIL.Image.Image], PIL.Image.Image]] = None + image: str | PIL.Image.Image, convert_method: Optional[Callable[[PIL.Image.Image], PIL.Image.Image]] = None ) -> PIL.Image.Image: """ Loads `image` to a PIL Image. @@ -56,20 +56,20 @@ def load_image( def load_video( video: str, - convert_method: Optional[Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]]] = None, -) -> List[PIL.Image.Image]: + convert_method: Optional[Callable[[list[PIL.Image.Image]], list[PIL.Image.Image]]] = None, +) -> list[PIL.Image.Image]: """ Loads `video` to a list of PIL Image. Args: video (`str`): A URL or Path to a video to convert to a list of PIL Image format. - convert_method (Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]], *optional*): + convert_method (Callable[[list[PIL.Image.Image]], list[PIL.Image.Image]], *optional*): A conversion method to apply to the video after loading it. When set to `None` the images will be converted to "RGB". Returns: - `List[PIL.Image.Image]`: + `list[PIL.Image.Image]`: The video as a list of PIL images. """ is_url = video.startswith("http://") or video.startswith("https://") @@ -139,7 +139,7 @@ def load_video( # Taken from `transformers`. -def get_module_from_name(module, tensor_name: str) -> Tuple[Any, str]: +def get_module_from_name(module, tensor_name: str) -> tuple[Any, str]: if "." in tensor_name: splits = tensor_name.split(".") for split in splits[:-1]: diff --git a/src/diffusers/utils/logging.py b/src/diffusers/utils/logging.py index 2ad6d3a47607..561b97ef5e8b 100644 --- a/src/diffusers/utils/logging.py +++ b/src/diffusers/utils/logging.py @@ -28,7 +28,7 @@ WARN, # NOQA WARNING, # NOQA ) -from typing import Dict, Optional +from typing import Optional from tqdm import auto as tqdm_lib @@ -105,7 +105,7 @@ def _reset_library_root_logger() -> None: _default_handler = None -def get_log_levels_dict() -> Dict[str, int]: +def get_log_levels_dict() -> dict[str, int]: return log_levels diff --git a/src/diffusers/utils/outputs.py b/src/diffusers/utils/outputs.py index 2b20f6120ce3..b7d9a29ccfce 100644 --- a/src/diffusers/utils/outputs.py +++ b/src/diffusers/utils/outputs.py @@ -17,7 +17,7 @@ from collections import OrderedDict from dataclasses import fields, is_dataclass -from typing import Any, Tuple +from typing import Any import numpy as np @@ -127,7 +127,7 @@ def __reduce__(self): args = tuple(getattr(self, field.name) for field in fields(self)) return callable, args, *remaining - def to_tuple(self) -> Tuple[Any, ...]: + def to_tuple(self) -> tuple[Any, ...]: """ Convert self to a tuple containing all the attributes/keys that are not `None`. """ diff --git a/src/diffusers/utils/pil_utils.py b/src/diffusers/utils/pil_utils.py index 76678070b697..72d4704fa945 100644 --- a/src/diffusers/utils/pil_utils.py +++ b/src/diffusers/utils/pil_utils.py @@ -1,5 +1,3 @@ -from typing import List - import PIL.Image import PIL.ImageOps from packaging import version @@ -50,7 +48,7 @@ def numpy_to_pil(images): return pil_images -def make_image_grid(images: List[PIL.Image.Image], rows: int, cols: int, resize: int = None) -> PIL.Image.Image: +def make_image_grid(images: list[PIL.Image.Image], rows: int, cols: int, resize: int = None) -> PIL.Image.Image: """ Prepares a single grid of images. Useful for visualization purposes. """ diff --git a/src/diffusers/utils/remote_utils.py b/src/diffusers/utils/remote_utils.py index 6494dc14171a..29c6449729b7 100644 --- a/src/diffusers/utils/remote_utils.py +++ b/src/diffusers/utils/remote_utils.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import io import json -from typing import List, Literal, Optional, Union, cast +from typing import Literal, Optional, cast import requests @@ -58,7 +60,7 @@ def detect_image_type(data: bytes) -> str: def check_inputs_decode( endpoint: str, tensor: "torch.Tensor", - processor: Optional[Union["VaeImageProcessor", "VideoProcessor"]] = None, + processor: Optional["VaeImageProcessor" | "VideoProcessor"] = None, do_scaling: bool = True, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, @@ -91,7 +93,7 @@ def check_inputs_decode( def postprocess_decode( response: requests.Response, - processor: Optional[Union["VaeImageProcessor", "VideoProcessor"]] = None, + processor: Optional["VaeImageProcessor" | "VideoProcessor"] = None, output_type: Literal["mp4", "pil", "pt"] = "pil", return_type: Literal["mp4", "pil", "pt"] = "pil", partial_postprocess: bool = False, @@ -117,7 +119,7 @@ def postprocess_decode( else: if isinstance(processor, VideoProcessor): output = cast( - List[Image.Image], + list[Image.Image], processor.postprocess_video(output_tensor, output_type="pil")[0], ) else: @@ -144,7 +146,7 @@ def postprocess_decode( def prepare_decode( tensor: "torch.Tensor", - processor: Optional[Union["VaeImageProcessor", "VideoProcessor"]] = None, + processor: Optional["VaeImageProcessor" | "VideoProcessor"] = None, do_scaling: bool = True, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, @@ -188,7 +190,7 @@ def prepare_decode( def remote_decode( endpoint: str, tensor: "torch.Tensor", - processor: Optional[Union["VaeImageProcessor", "VideoProcessor"]] = None, + processor: Optional["VaeImageProcessor" | "VideoProcessor"] = None, do_scaling: bool = True, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, @@ -200,7 +202,7 @@ def remote_decode( output_tensor_type: Literal["binary"] = "binary", height: Optional[int] = None, width: Optional[int] = None, -) -> Union[Image.Image, List[Image.Image], bytes, "torch.Tensor"]: +) -> Image.Image | list[Image.Image] | bytes | "torch.Tensor": """ Hugging Face Hybrid Inference that allow running VAE decode remotely. @@ -275,7 +277,7 @@ def remote_decode( Required for `"packed"` latents. Returns: - output (`Image.Image` or `List[Image.Image]` or `bytes` or `torch.Tensor`). + output (`Image.Image` or `list[Image.Image]` or `bytes` or `torch.Tensor`). """ if input_tensor_type == "base64": deprecate( @@ -336,7 +338,7 @@ def remote_decode( def check_inputs_encode( endpoint: str, - image: Union["torch.Tensor", Image.Image], + image: "torch.Tensor" | Image.Image, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, ): @@ -356,7 +358,7 @@ def postprocess_encode( def prepare_encode( - image: Union["torch.Tensor", Image.Image], + image: "torch.Tensor" | Image.Image, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, ): @@ -379,7 +381,7 @@ def prepare_encode( def remote_encode( endpoint: str, - image: Union["torch.Tensor", Image.Image], + image: "torch.Tensor" | Image.Image, scaling_factor: Optional[float] = None, shift_factor: Optional[float] = None, ) -> "torch.Tensor": diff --git a/src/diffusers/utils/source_code_parsing_utils.py b/src/diffusers/utils/source_code_parsing_utils.py index 5f94711c21d8..c69b40d11b49 100644 --- a/src/diffusers/utils/source_code_parsing_utils.py +++ b/src/diffusers/utils/source_code_parsing_utils.py @@ -12,7 +12,7 @@ def __init__(self): def visit_Return(self, node): # Check if the return value is a tuple. - if isinstance(node.value, ast.Tuple): + if isinstance(node.value, ast.tuple): for elt in node.value.elts: if isinstance(elt, ast.Name): self.return_names.append(elt.id) diff --git a/src/diffusers/utils/testing_utils.py b/src/diffusers/utils/testing_utils.py index 3297bb5fdcd6..1065b34a0cb4 100644 --- a/src/diffusers/utils/testing_utils.py +++ b/src/diffusers/utils/testing_utils.py @@ -19,7 +19,7 @@ from contextlib import contextmanager from io import BytesIO, StringIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Set import numpy as np import PIL.Image @@ -676,7 +676,7 @@ def get_python_version(): return major, minor -def load_numpy(arry: Union[str, np.ndarray], local_path: Optional[str] = None) -> np.ndarray: +def load_numpy(arry: str | np.ndarray, local_path: Optional[str] = None) -> np.ndarray: if isinstance(arry, str): if local_path is not None: # local_path can be passed to correct images of tests @@ -709,7 +709,7 @@ def load_pt(url: str, map_location: Optional[str] = None, weights_only: Optional return arry -def load_image(image: Union[str, PIL.Image.Image]) -> PIL.Image.Image: +def load_image(image: str | PIL.Image.Image) -> PIL.Image.Image: """ Loads `image` to a PIL Image. @@ -750,7 +750,7 @@ def preprocess_image(image: PIL.Image, batch_size: int): return 2.0 * image - 1.0 -def export_to_gif(image: List[PIL.Image.Image], output_gif_path: str = None) -> str: +def export_to_gif(image: list[PIL.Image.Image], output_gif_path: str = None) -> str: if output_gif_path is None: output_gif_path = tempfile.NamedTemporaryFile(suffix=".gif").name @@ -843,7 +843,7 @@ def export_to_obj(mesh, output_obj_path: str = None): f.writelines("\n".join(combined_data)) -def export_to_video(video_frames: List[np.ndarray], output_video_path: str = None) -> str: +def export_to_video(video_frames: list[np.ndarray], output_video_path: str = None) -> str: if is_opencv_available(): import cv2 else: @@ -1281,7 +1281,7 @@ def _is_torch_fp64_available(device): # This dispatches a defined function according to the accelerator from the function definitions. -def _device_agnostic_dispatch(device: str, dispatch_table: Dict[str, Callable], *args, **kwargs): +def _device_agnostic_dispatch(device: str, dispatch_table: dict[str, Callable], *args, **kwargs): if device not in dispatch_table: return dispatch_table["default"](*args, **kwargs) @@ -1381,7 +1381,7 @@ def backend_supports_training(device: str): # Guard for when Torch is not available if is_torch_available(): # Update device function dict mapping - def update_mapping_from_spec(device_fn_dict: Dict[str, Callable], attribute_name: str): + def update_mapping_from_spec(device_fn_dict: dict[str, Callable], attribute_name: str): try: # Try to import the function directly spec_fn = getattr(device_spec_module, attribute_name) @@ -1430,7 +1430,7 @@ def update_mapping_from_spec(device_fn_dict: Dict[str, Callable], attribute_name # Modified from https://github.com/huggingface/transformers/blob/cdfb018d0300fef3b07d9220f3efe9c2a9974662/src/transformers/testing_utils.py#L3090 # Type definition of key used in `Expectations` class. -DeviceProperties = Tuple[Union[str, None], Union[int, None]] +DeviceProperties = tuple[str | None, int | None] @functools.lru_cache diff --git a/src/diffusers/utils/torch_utils.py b/src/diffusers/utils/torch_utils.py index a1ab8cda431f..a221ec736fa7 100644 --- a/src/diffusers/utils/torch_utils.py +++ b/src/diffusers/utils/torch_utils.py @@ -15,9 +15,11 @@ PyTorch utilities: Utilities related to PyTorch """ +from __future__ import annotations + import functools import os -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Optional from . import logging from .import_utils import is_torch_available, is_torch_npu_available, is_torch_version @@ -88,7 +90,7 @@ def maybe_allow_in_graph(cls): # This dispatches a defined function according to the accelerator from the function definitions. -def _device_agnostic_dispatch(device: str, dispatch_table: Dict[str, Callable], *args, **kwargs): +def _device_agnostic_dispatch(device: str, dispatch_table: dict[str, Callable], *args, **kwargs): if device not in dispatch_table: return dispatch_table["default"](*args, **kwargs) @@ -144,9 +146,9 @@ def backend_supports_training(device: str): def randn_tensor( - shape: Union[Tuple, List], - generator: Optional[Union[List["torch.Generator"], "torch.Generator"]] = None, - device: Optional[Union[str, "torch.device"]] = None, + shape: tuple | list, + generator: Optional[list["torch.Generator"] | "torch.Generator"] = None, + device: Optional[str | "torch.device"] = None, dtype: Optional["torch.dtype"] = None, layout: Optional["torch.layout"] = None, ): @@ -241,7 +243,7 @@ def fourier_filter(x_in: "torch.Tensor", threshold: int, scale: int) -> "torch.T def apply_freeu( resolution_idx: int, hidden_states: "torch.Tensor", res_hidden_states: "torch.Tensor", **freeu_kwargs -) -> Tuple["torch.Tensor", "torch.Tensor"]: +) -> tuple["torch.Tensor", "torch.Tensor"]: """Applies the FreeU mechanism as introduced in https: //arxiv.org/abs/2309.11497. Adapted from the official code repository: https://github.com/ChenyangSi/FreeU. diff --git a/src/diffusers/utils/typing_utils.py b/src/diffusers/utils/typing_utils.py index 2b5b1a4f5ab5..00853608b447 100644 --- a/src/diffusers/utils/typing_utils.py +++ b/src/diffusers/utils/typing_utils.py @@ -15,10 +15,10 @@ Typing utilities: Utilities related to type checking and validation """ -from typing import Any, Dict, List, Set, Tuple, Type, Union, get_args, get_origin +from typing import Any, Set, Type, Union, get_args, get_origin -def _is_valid_type(obj: Any, class_or_tuple: Union[Type, Tuple[Type, ...]]) -> bool: +def _is_valid_type(obj: Any, class_or_tuple: Type | tuple[Type, ...]) -> bool: """ Checks if an object is an instance of any of the provided types. For collections, it checks if every element is of the correct type as well. @@ -43,7 +43,7 @@ def _is_valid_type(obj: Any, class_or_tuple: Union[Type, Tuple[Type, ...]]) -> b class_or_tuple = {t for t in class_or_tuple if isinstance(obj, get_origin(t) or t)} # Singular types (e.g. int, ControlNet, ...) - # Untyped collections (e.g. List, but not List[int]) + # Untyped collections (e.g. list, but not list[int]) elem_class_or_tuple = {get_args(t) for t in class_or_tuple} if () in elem_class_or_tuple: return True @@ -53,10 +53,10 @@ def _is_valid_type(obj: Any, class_or_tuple: Union[Type, Tuple[Type, ...]]) -> b # Typed tuples elif obj_type is tuple: return any( - # Tuples with any length and single type (e.g. Tuple[int, ...]) + # tuples with any length and single type (e.g. tuple[int, ...]) (len(t) == 2 and t[-1] is Ellipsis and all(_is_valid_type(x, t[0]) for x in obj)) or - # Tuples with fixed length and any types (e.g. Tuple[int, str]) + # tuples with fixed length and any types (e.g. tuple[int, str]) (len(obj) == len(t) and all(_is_valid_type(x, tt) for x, tt in zip(obj, t))) for t in elem_class_or_tuple ) @@ -78,14 +78,14 @@ def _get_detailed_type(obj: Any) -> Type: obj_type = type(obj) if obj_type in (list, set): - obj_origin_type = List if obj_type is list else Set - elems_type = Union[tuple({_get_detailed_type(x) for x in obj})] + obj_origin_type = list if obj_type is list else Set + elems_type = tuple({_get_detailed_type(x) for x in obj}) return obj_origin_type[elems_type] elif obj_type is tuple: - return Tuple[tuple(_get_detailed_type(x) for x in obj)] + return tuple[tuple(_get_detailed_type(x) for x in obj)] elif obj_type is dict: - keys_type = Union[tuple({_get_detailed_type(k) for k in obj.keys()})] - values_type = Union[tuple({_get_detailed_type(k) for k in obj.values()})] - return Dict[keys_type, values_type] + keys_type = tuple({_get_detailed_type(k) for k in obj.keys()}) + values_type = tuple({_get_detailed_type(k) for k in obj.values()}) + return dict[keys_type, values_type] else: return obj_type diff --git a/src/diffusers/video_processor.py b/src/diffusers/video_processor.py index 59b59b47d2c7..44b45fcc258a 100644 --- a/src/diffusers/video_processor.py +++ b/src/diffusers/video_processor.py @@ -13,7 +13,7 @@ # limitations under the License. import warnings -from typing import List, Optional, Union +from typing import Optional import numpy as np import PIL @@ -30,15 +30,15 @@ def preprocess_video(self, video, height: Optional[int] = None, width: Optional[ Preprocesses input video(s). Args: - video (`List[PIL.Image]`, `List[List[PIL.Image]]`, `torch.Tensor`, `np.array`, `List[torch.Tensor]`, `List[np.array]`): + video (`list[PIL.Image]`, `list[list[PIL.Image]]`, `torch.Tensor`, `np.array`, `list[torch.Tensor]`, `list[np.array]`): The input video. It can be one of the following: - * List of the PIL images. - * List of list of PIL images. + * list of the PIL images. + * list of list of PIL images. * 4D Torch tensors (expected shape for each tensor `(num_frames, num_channels, height, width)`). * 4D NumPy arrays (expected shape for each array `(num_frames, height, width, num_channels)`). - * List of 4D Torch tensors (expected shape for each tensor `(num_frames, num_channels, height, + * list of 4D Torch tensors (expected shape for each tensor `(num_frames, num_channels, height, width)`). - * List of 4D NumPy arrays (expected shape for each array `(num_frames, height, width, num_channels)`). + * list of 4D NumPy arrays (expected shape for each array `(num_frames, height, width, num_channels)`). * 5D NumPy arrays: expected shape for each array `(batch_size, num_frames, height, width, num_channels)`. * 5D Torch tensors: expected shape for each array `(batch_size, num_frames, num_channels, height, @@ -88,7 +88,7 @@ def preprocess_video(self, video, height: Optional[int] = None, width: Optional[ def postprocess_video( self, video: torch.Tensor, output_type: str = "np" - ) -> Union[np.ndarray, torch.Tensor, List[PIL.Image.Image]]: + ) -> np.ndarray | torch.Tensor | list[PIL.Image.Image]: r""" Converts a video tensor to a list of frames for export.