From a560f52bf0f850e3ccb67c1a768cd32f64bce3d6 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 19 May 2025 22:43:46 +0200 Subject: [PATCH 1/3] update to run on the latest branch --- __init__.py | 258 +++--------- mellon-modular-diffusers.py | 790 +++++++++++++++--------------------- 2 files changed, 378 insertions(+), 670 deletions(-) diff --git a/__init__.py b/__init__.py index 7c7c219..0c885c9 100644 --- a/__init__.py +++ b/__init__.py @@ -4,123 +4,27 @@ from .schedulers import load_all_schedulers MODULE_MAP = { - "ControlnetModelLoader": { - "label": "Load Controlnet Model", - "category": "Modular Diffusers", - "params": { - "model_id": { - "label": "Model ID", - "type": "string", - "default": "xinsir/controlnet-depth-sdxl-1.0", - }, - "variant": { - "label": "Variant", - "type": "string", - "options": ["", "fp16"], - "default": "", - "display": "autocomplete", - "no_validation": True, - }, - "dtype": { - "label": "dtype", - "options": ["auto", "float32", "bfloat16", "float16"], - "default": "float16", - "postProcess": str_to_dtype, - }, - "controlnet_model": { - "label": "Controlnet Model", - "display": "output", - "type": "controlnet_model", - }, - }, - }, - "ControlnetUnionModelLoader": { - "label": "Load Controlnet Union Model", + + "AutoModelLoader": { + "label": "Auto Model Loader", "category": "Modular Diffusers", "params": { - "model_id": { - "label": "Model ID", - "type": "string", - "default": "OzzyGT/controlnet-union-promax-sdxl-1.0", - }, - "variant": { - "label": "Variant", + "name": { + "label": "Name", "type": "string", - "options": ["", "fp16"], - "default": "", - "display": "autocomplete", - "no_validation": True, - }, - "dtype": { - "label": "dtype", - "options": ["auto", "float32", "bfloat16", "float16"], - "default": "float16", - "postProcess": str_to_dtype, }, - "controlnet_union_model": { - "label": "Controlnet Union Model", - "display": "output", - "type": "controlnet_union_model", - }, - }, - }, - "UnetLoader": { - "label": "Unet Loader", - "category": "Modular Diffusers", - "params": { "model_id": { "label": "Model ID", - "options": list_local_models(), - "display": "autocomplete", - "no_validation": True, - "default": "stabilityai/stable-diffusion-xl-base-1.0", - }, - "subfolder": { - "label": "Subfolder", "type": "string", - "default": "unet", - }, - "variant": { - "label": "Variant", - "type": "string", - "options": ["", "fp16"], - "default": "", - "display": "autocomplete", - "no_validation": True, - }, - "dtype": { - "label": "dtype", - "options": ["auto", "float32", "bfloat16", "float16"], - "default": "float16", - "postProcess": str_to_dtype, - }, - "unet": { - "label": "Unet", - "display": "output", - "type": "diffusers_unet", - }, - }, - }, - "VAELoader": { - "label": "VAE Loader", - "category": "Modular Diffusers", - "params": { - "model_id": { - "label": "Model ID", - "options": list_local_models(), - "display": "autocomplete", - "no_validation": True, - "default": "stabilityai/stable-diffusion-xl-base-1.0", }, "subfolder": { "label": "Subfolder", "type": "string", - "default": "vae", }, "variant": { "label": "Variant", "type": "string", - "options": ["", "fp16"], + "options": ["", "fp16", "bf16"], "default": "", "display": "autocomplete", "no_validation": True, @@ -131,13 +35,14 @@ "default": "float16", "postProcess": str_to_dtype, }, - "vae": { - "label": "VAE", + "model": { + "label": "Model", "display": "output", - "type": "diffusers_vae", + "type": "diffusers_auto_model", }, }, }, + "ModelsLoader": { "label": "Diffusers Model Loader", "category": "Modular Diffusers", @@ -147,7 +52,7 @@ "options": list_local_models(), "display": "autocomplete", "no_validation": True, - "default": "stabilityai/stable-diffusion-xl-base-1.0", + "default": "YiYiXu/modular-demo-auto", }, "variant": { "label": "Variant", @@ -172,12 +77,12 @@ "unet": { "label": "Unet", "display": "input", - "type": "diffusers_unet", + "type": "diffusers_auto_model", }, "vae": { "label": "VAE", "display": "input", - "type": "diffusers_vae", + "type": "diffusers_auto_model", }, "lora_list": { "label": "Lora", @@ -192,12 +97,12 @@ "unet_out": { "label": "UNet", "display": "output", - "type": "diffusers_unet", + "type": "diffusers_auto_model", }, "vae_out": { "label": "VAE", "display": "output", - "type": "diffusers_vae", + "type": "diffusers_auto_model", }, "scheduler": { "label": "Scheduler", @@ -215,14 +120,21 @@ "display": "input", "type": "diffusers_text_encoders", }, + "guider": { + "label": "Guider", + "display": "input", + "type": "guider", + }, "prompt": { "label": "Prompt", "type": "string", + "default": "a bear sitting in a chair drinking a milkshake", "display": "textarea", }, "negative_prompt": { "label": "Negative Prompt", "type": "string", + "default": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality", "display": "textarea", }, "embeddings": { @@ -230,6 +142,11 @@ "display": "output", "type": "prompt_embeddings", }, + "guider_out": { + "label": "Guider", + "display": "output", + "type": "guider", + }, }, }, "Scheduler": { @@ -255,7 +172,7 @@ "unet": { "label": "Unet", "display": "input", - "type": "diffusers_unet", + "type": "diffusers_auto_model", }, "scheduler": { "label": "Scheduler", @@ -274,14 +191,6 @@ "min": 1, "max": 1000, }, - "cfg": { - "label": "Guidance", - "type": "float", - "display": "slider", - "default": 7.0, - "min": 0, - "max": 20, - }, "seed": { "label": "Seed", "type": "int", @@ -310,7 +219,7 @@ "group": "dimensions", }, "guider": { - "label": "Optional Guider", + "label": "Guider", "type": "guider", "display": "input", }, @@ -338,7 +247,7 @@ "vae": { "label": "VAE", "display": "input", - "type": "diffusers_vae", + "type": "diffusers_auto_model", }, "latents": { "label": "Latents", @@ -402,47 +311,21 @@ "label": "Perturbed Attention Guidance", "category": "Modular Diffusers", "params": { - "pag_scale": { - "label": "Scale", + "guidance_scale": { + "label": "Guidance Scale", "type": "float", "display": "slider", - "default": 3, + "default": 5.0, "min": 0, - "max": 5, - }, - "pag_layers": { - "label": "PAG Layers", - "type": "string", - "default": "mid", - }, - "guider": { - "label": "Guider", - "display": "output", - "type": "guider", + "max": 10, }, - }, - }, - "APGOptionalGuider": { - "label": "Adaptive Projected Guidance", - "category": "Modular Diffusers", - "params": { - "rescale_factor": { - "label": "Rescale Factor", + "skip_layer_guidance_scale": { + "label": "Scale", "type": "float", "display": "slider", - "default": 15, + "default": 3, "min": 0, - "max": 20, - "step": 0.1, - }, - "momentum": { - "label": "Momentum", - "type": "float", - "display": "slider", - "default": -0.5, - "min": -5, "max": 5, - "step": 0.1, }, "guider": { "label": "Guider", @@ -486,7 +369,7 @@ }, "controlnet_model": { "label": "Controlnet Model", - "type": "controlnet_model", + "type": "diffusers_auto_model", "display": "input", }, "controlnet": { @@ -566,7 +449,7 @@ }, "controlnet_model": { "label": "Controlnet Union Model", - "type": "controlnet_union_model", + "type": "diffusers_auto_model", "display": "input", }, "controlnet": { @@ -638,15 +521,20 @@ }, }, }, - "IPAdapter": { - "label": "IP-Adapter", - "description": "Process images with IP-Adapter", + "IPAdapterLoader": { + "label": "IP-Adapter Loader", + "description": "IP-Adapter Loader", "category": "Modular Diffusers", "params": { "unet": { "label": "UNet", "display": "input", - "type": "diffusers_unet", + "type": "diffusers_auto_model", + }, + "guider": { + "label": "Guider", + "display": "input", + "type": "guider", }, "ip_adapter_inputs": { "label": "IP-Adapter Inputs", @@ -662,54 +550,12 @@ "unet_out": { "label": "UNet", "display": "output", - "type": "diffusers_unet", + "type": "diffusers_auto_model", }, - }, - }, - "Ghiblify": { - "label": "Ghiblify Image", - "category": "Modular Diffusers", - "params": { - "repo_id": { - "label": "Repository ID", - "options": list_local_models(), - "display": "autocomplete", - "no_validation": True, - "default": "stabilityai/stable-diffusion-xl-base-1.0", - }, - "device": { - "label": "Device", - "type": "string", - "options": device_list, - "default": default_device, - }, - "dtype": { - "label": "dtype", - "options": ["auto", "float32", "bfloat16", "float16"], - "default": "float16", - "postProcess": str_to_dtype, - }, - "seed": { - "label": "Seed", - "type": "int", - "default": 0, - "min": 0, - "display": "random", - }, - "prompt": { - "label": "Prompt", - "type": "string", - "display": "textarea", - }, - "image": { - "label": "Source image", - "type": "image", - "display": "input", - }, - "images": { - "label": "Images", - "type": "image", + "guider_out": { + "label": "Guider", "display": "output", + "type": "guider", }, }, }, diff --git a/mellon-modular-diffusers.py b/mellon-modular-diffusers.py index 7fe8cb3..eeb580b 100644 --- a/mellon-modular-diffusers.py +++ b/mellon-modular-diffusers.py @@ -1,46 +1,38 @@ -import asyncio import logging import os +import time import torch from diffusers import ( - AutoencoderKL, - ControlNetModel, - ControlNetUnionModel, - DiffusionPipeline, - ModularPipeline, - UNet2DConditionModel, + AutoModel, + AdaptiveProjectedGuidance, + ClassifierFreeGuidance, + SkipLayerGuidance, + LayerSkipConfig, + ComponentsManager, + StableDiffusionXLAutoPipeline, + ComponentSpec, + StableDiffusionXLModularLoader ) -from diffusers.guider import APGGuider, CFGGuider, PAGGuider -from diffusers.pipelines.components_manager import ComponentsManager from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel -from diffusers.pipelines.modular_pipeline import SequentialPipelineBlocks -from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_modular import ( - AUTO_BLOCKS, - StableDiffusionXLAutoDecodeStep, - StableDiffusionXLIPAdapterStep, - StableDiffusionXLLoraStep, - StableDiffusionXLTextEncoderStep, -) -from huggingface_hub import hf_hub_download -from huggingface_hub.errors import EntryNotFoundError -from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection +from transformers import CLIPVisionModelWithProjection from mellon.NodeBase import NodeBase, are_different -from mellon.server import web_server +# Configure logger logger = logging.getLogger("mellon") +logger.setLevel(logging.DEBUG) + -# YiYi TODO: make the SDXLAutoBlocks user can directly import -all_blocks_map = AUTO_BLOCKS.copy() -text_block = all_blocks_map.pop("text_encoder")() -decoder_block = all_blocks_map.pop("decode")() -image_encoder_block = all_blocks_map.pop("image_encoder")() +auto_blocks =StableDiffusionXLAutoPipeline() +text_block = auto_blocks.blocks.pop("text_encoder") +decoder_block = auto_blocks.blocks.pop("decoder") +ip_adapter_block = auto_blocks.blocks.pop("ip_adapter") +image_encoder_block = auto_blocks.blocks.pop("image_encoder") -class SDXLAutoBlocks(SequentialPipelineBlocks): - block_classes = list(all_blocks_map.values()) - block_names = list(all_blocks_map.keys()) +# Initialize components manager +components = ComponentsManager() def _has_changed(old_params, new_params): @@ -195,117 +187,77 @@ def update_lora_adapters(lora_node, lora_list): if scales: lora_node.set_adapters(list(scales.keys()), list(scales.values())) - -components = ComponentsManager() - - -class ControlnetModelLoader(NodeBase): - def __del__(self): - components.remove(f"controlnet_{self.node_id}") - super().__del__() - - def execute(self, model_id, variant, dtype): - # Normalize parameters - variant = None if variant == "" else variant - - controlnet_model = ControlNetModel.from_pretrained( - model_id, variant=variant, torch_dtype=dtype - ) - components.add(f"controlnet_{self.node_id}", controlnet_model) - return { - "controlnet_model": components.get_model_info(f"controlnet_{self.node_id}") - } - - -class ControlnetUnionModelLoader(NodeBase): +# YiYi TODO: add to node wrapper class +# "unet" -> "unet_12344" +def node_get_component_id(node_id=None, manager=None, name=None): + comp_ids = manager._lookup_ids(name=name, collection=node_id) + if len(comp_ids) != 1: + raise ValueError(f"Expected 1 component for {name} for node {node_id}, got {len(comp_ids)}") + return list(comp_ids)[0] + +# "unet" +# -> +# {"model_id": "unet_12344", +# "added_time": 1716239234.0, +# "collection": "node_id", +# "class_name": "UNet2DConditionModel", +# "size_gb": 1.23, +# "adapters": ["lora_1", "lora_2"], +# "ip_adapter": {"", [0.5]} +# } +def node_get_component_info(node_id=None, manager=None, name=None): + comp_id = node_get_component_id(node_id, manager, name) + return manager.get_model_info(comp_id) + + + +# (1) loader nodes: nodes that creates components (load models or create components e.g. guiders, schedulers, etc.) +# AutoModelLoader +# ModelsLoader +# Scheduler +# IPAdapterLoader (it also runs the image_encoder) + +class AutoModelLoader(NodeBase): def __del__(self): - components.remove(f"controlnet_{self.node_id}") + node_comp_ids = components._lookup_ids(collection=self.node_id) + for comp_id in node_comp_ids: + components.remove(comp_id) super().__del__() - def execute(self, model_id, variant, dtype): + def execute(self, name, model_id, subfolder, variant, dtype): + # Debug logging + logger.debug(f"AutoModelLoader ({self.node_id}) received parameters:") + logger.debug(f" name: '{name}'") + logger.debug(f" model_id: '{model_id}'") + logger.debug(f" subfolder: '{subfolder}'") + logger.debug(f" variant: '{variant}'") + logger.debug(f" dtype: '{dtype}'") + # Normalize parameters variant = None if variant == "" else variant + subfolder = None if subfolder == "" else subfolder - controlnet_model = ControlNetUnionModel.from_pretrained( - model_id, variant=variant, torch_dtype=dtype - ) + spec = ComponentSpec(name=name, repo=model_id, subfolder=subfolder, variant=variant) + model = spec.load(torch_dtype=dtype) + comp_id = components.add(name, model, collection=self.node_id) + logger.debug(f" AutoModelLoader: comp_id added: {comp_id}") + logger.debug(f" AutoModelLoader: component manager: {components}") - components.add(f"controlnet_{self.node_id}", controlnet_model) return { - "controlnet_union_model": components.get_model_info( - f"controlnet_{self.node_id}" - ) + "model": components.get_model_info(comp_id) } - -class UnetLoader(NodeBase): - def __del__(self): - components.remove(f"unet_{self.node_id}") - super().__del__() - - def execute(self, model_id, subfolder, variant, dtype): - # Normalize parameters - subfolder = None if subfolder == "" else subfolder - variant = None if variant == "" else variant - - logger.debug( - f" load unet from model_id: {model_id}, subfolder: {subfolder}, variant: {variant}, dtype: {dtype}" - ) - - # for the searching and autocomplete later, we download the model_index.json if it has one - try: - _config_file = hf_hub_download(model_id, filename="model_index.json") - except EntryNotFoundError as _e: - logger.info( - "Unet model doesn't have a main model_index.json, model won't appear in autocomplete" - ) - - unet = UNet2DConditionModel.from_pretrained( - model_id, subfolder=subfolder, variant=variant, torch_dtype=dtype - ) - components.add(f"unet_{self.node_id}", unet) - return {"unet": components.get_model_info(f"unet_{self.node_id}")} - - -class VAELoader(NodeBase): - def __del__(self): - components.remove(f"vae_{self.node_id}") - super().__del__() - - def execute(self, model_id, subfolder, variant, dtype): - # Normalize parameters - subfolder = None if subfolder == "" else subfolder - variant = None if variant == "" else variant - - vae = AutoencoderKL.from_pretrained( - model_id, subfolder=subfolder, variant=variant, torch_dtype=dtype - ) - components.add(f"vae_{self.node_id}", vae) - return {"vae": components.get_model_info(f"vae_{self.node_id}")} - - class ModelsLoader(NodeBase): def __init__(self, node_id=None): super().__init__(node_id) - # lora - lora_step = StableDiffusionXLLoraStep() - self._lora_node = ModularPipeline.from_block(lora_step) - # ip adapter - ip_adapter_block = StableDiffusionXLIPAdapterStep() - self._ip_adapter_node = ModularPipeline.from_block(ip_adapter_block) + self.loader_class = StableDiffusionXLModularLoader + self.loader = None def __del__(self): - components.remove(f"text_encoder_{self.node_id}") - components.remove(f"text_encoder_2_{self.node_id}") - components.remove(f"tokenizer_{self.node_id}") - components.remove(f"tokenizer_2_{self.node_id}") - components.remove(f"scheduler_{self.node_id}") - components.remove(f"unet_{self.node_id}") - components.remove(f"vae_{self.node_id}") - components.remove(f"image_encoder_{self.node_id}") - components.remove(f"feature_extractor_{self.node_id}") - self._lora_node.unload_lora_weights() - self._ip_adapter_node.unload_ip_adapter() + node_comp_ids = components._lookup_ids(collection=self.node_id) + for comp_id in node_comp_ids: + components.remove(comp_id) + self.loader = None super().__del__() def __call__(self, **kwargs): @@ -321,7 +273,6 @@ def execute( unet=None, vae=None, lora_list=None, - ip_adapter_input=None, ): # Normalize parameters variant = None if variant == "" else variant @@ -337,7 +288,7 @@ def _has_changed(old_params, new_params): return False logger.debug(f""" - in SDXLModelsLoader execute: {self.node_id} + ModelsLoader ({self.node_id}) received parameters: old_params: {self._old_params} new params: - repo_id: {repo_id} @@ -346,7 +297,6 @@ def _has_changed(old_params, new_params): - unet: {unet} - vae: {vae} - lora_list: {lora_list} - ip_adapter_input: {ip_adapter_input} """) repo_changed = _has_changed( @@ -355,207 +305,94 @@ def _has_changed(old_params, new_params): unet_input_changed = _has_changed(self._old_params, {"unet": unet}) vae_input_changed = _has_changed(self._old_params, {"vae": vae}) lora_input_changed = _has_changed(self._old_params, {"lora_list": lora_list}) - ip_adapter_input_changed = _has_changed( - self._old_params, {"ip_adapter_input": ip_adapter_input} - ) logger.debug( f"Changes detected - repo: {repo_changed}, unet: {unet_input_changed}, vae: {vae_input_changed}, " - f"lora: {lora_input_changed}, ip_adapter: {ip_adapter_input_changed}" + f"lora: {lora_input_changed}" ) unet_changed = unet_input_changed or (unet is None and repo_changed) vae_changed = vae_input_changed or (vae is None and repo_changed) + if repo_changed: + self.loader = self.loader_class.from_pretrained(repo_id, component_manager=components, collection=self.node_id) + logger.debug(f" ModelsLoader: loader created/updated: {self.loader}") + + # Load and update base models + loaded_components = {} if unet is None: if repo_changed or unet_input_changed: - print( - f" load unet from repo_id: {repo_id}, subfolder: unet, variant: {variant}, dtype: {dtype}" - ) - unet = UNet2DConditionModel.from_pretrained( - repo_id, subfolder="unet", variant=variant, torch_dtype=dtype + logger.debug( + f" ModelsLoader: load unet from repo_id: {repo_id}, subfolder: unet, variant: {variant}, dtype: {dtype}" ) - # Only add to components if we loaded it ourselves - print(f" add unet to components: unet_{self.node_id}") - components.add(f"unet_{self.node_id}", unet) - else: - # Get existing unet from our components - unet = components.get(f"unet_{self.node_id}") - unet_model_id = ( - f"unet_{self.node_id}" # Always use our node_id if input is None - ) + self.loader.load("unet", variant=variant, torch_dtype=dtype) else: # unet is always a model info dict if not None - unet_model_id = unet["model_id"] - unet = components.get(unet_model_id) + unet_id = unet["model_id"] + self.loader.update(unet=components.get_one(unet_id)) if vae is None: if repo_changed or vae_input_changed: logger.debug( - f" load vae from repo_id: {repo_id}, subfolder: vae, variant: {variant}, dtype: {dtype}" + f" ModelsLoader: load vae from repo_id: {repo_id}, subfolder: vae, variant: {variant}, dtype: {dtype}" ) - vae = AutoencoderKL.from_pretrained( - repo_id, subfolder="vae", variant=variant, torch_dtype=dtype - ) - # Only add to components if we loaded it ourselves - logger.debug(f" add vae to components: vae_{self.node_id}") - components.add(f"vae_{self.node_id}", vae) - else: - # Get existing vae from our components - vae = components.get(f"vae_{self.node_id}") - vae_model_id = ( - f"vae_{self.node_id}" # Always use our node_id if input is None - ) + self.loader.load("vae", variant=variant, torch_dtype=dtype) else: # vae is always a model info dict if not None - vae_model_id = vae["model_id"] - vae = components.get(vae_model_id) + vae_id = vae["model_id"] + self.loader.update(vae=components.get_one(vae_id)) # Load text encoders and scheduler if repo_changed: logger.debug( - f" load text encoders and scheduler from pipeline: {repo_id}, variant: {variant}, dtype: {dtype}" - ) - pipe = DiffusionPipeline.from_pretrained( - repo_id, - torch_dtype=dtype, - variant=variant, - vae=None, - unet=None, + f" ModelsLoader: load text encoders and scheduler from: {repo_id}, variant: {variant}, dtype: {dtype}" ) - components.add(f"text_encoder_{self.node_id}", pipe.text_encoder) - components.add(f"text_encoder_2_{self.node_id}", pipe.text_encoder_2) - components.add(f"tokenizer_{self.node_id}", pipe.tokenizer) - components.add(f"tokenizer_2_{self.node_id}", pipe.tokenizer_2) - components.add(f"scheduler_{self.node_id}", pipe.scheduler) + self.loader.load(["text_encoder", "text_encoder_2", "tokenizer", "tokenizer_2", "scheduler"], variant=variant, torch_dtype=dtype) + # Handle LoRA if not lora_list: - logger.debug(f" unload lora from components: lora_{self.node_id}") - self._lora_node.unload_lora_weights() + logger.debug(f" ModelsLoader: unload lora:") + self.loader.unload_lora_weights() elif lora_input_changed or unet_changed: - logger.debug(f" update lora from components: lora_{self.node_id}") - - # Unload first to clean previous model's state - self._lora_node.unload_lora_weights() - - self._lora_node.update_states( - unet=unet, # Use actual model - text_encoder=components.get(f"text_encoder_{self.node_id}"), - text_encoder_2=components.get(f"text_encoder_2_{self.node_id}"), - ) - update_lora_adapters(self._lora_node, lora_list) + logger.debug(f" ModelsLoader: load lora:") - # Handle IP-Adapter - if not ip_adapter_input: - if self._ip_adapter_node is not None: - logger.debug( - f" unload ip_adapter from components: ip_adapter_{self.node_id}" - ) - self._ip_adapter_node.unload_ip_adapter() - elif ip_adapter_input_changed or unet_changed: # Unload first to clean previous model's state - logger.debug(" unload ip_adapter") - self._ip_adapter_node.unload_ip_adapter() - - if ip_adapter_input_changed: - logger.debug( - f" load image_encoder from repo_id: {ip_adapter_input['repo_id']}, subfolder: {ip_adapter_input['image_encoder_path']}, dtype: {dtype}" - ) - image_encoder = CLIPVisionModelWithProjection.from_pretrained( - ip_adapter_input["repo_id"], - subfolder=ip_adapter_input["image_encoder_path"], - torch_dtype=dtype, - ) - feature_extractor = CLIPImageProcessor(size=224, crop_size=224) - logger.debug( - f" add image_encoder to components: image_encoder_{self.node_id}" - ) - components.add(f"image_encoder_{self.node_id}", image_encoder) - logger.debug( - f" add feature_extractor to components: feature_extractor_{self.node_id}" - ) - components.add(f"feature_extractor_{self.node_id}", feature_extractor) - logger.debug( - f" update ip_adapter from components: ip_adapter_{self.node_id}" - ) - self._ip_adapter_node.update_states( - image_encoder=image_encoder, feature_extractor=feature_extractor - ) - - if unet_changed: - logger.debug("update unet") - self._ip_adapter_node.update_states(unet=unet) # Use actual model - - logger.debug(" load ip_adapter") - self._ip_adapter_node.load_ip_adapter( - ip_adapter_input["repo_id"], - subfolder=ip_adapter_input["subfolder"], - weight_name=ip_adapter_input["weight_name"], - ) - # Set the scale after loading - self._ip_adapter_node.set_ip_adapter_scale(ip_adapter_input["scale"]) + self.loader.unload_lora_weights() + update_lora_adapters(self.loader, lora_list) if unet_changed or vae_changed or repo_changed: components.enable_auto_cpu_offload(device=device) # Construct loaded_components at the end after all modifications loaded_components = { - "unet_out": components.get_model_info(unet_model_id), - "vae_out": components.get_model_info(vae_model_id), + "unet_out": node_get_component_info(node_id=self.node_id, manager=components, name="unet"), + "vae_out": node_get_component_info(node_id=self.node_id, manager=components, name="vae"), "text_encoders": { - k: components.get_model_info(v) - for k, v in { - "text_encoder": f"text_encoder_{self.node_id}", - "text_encoder_2": f"text_encoder_2_{self.node_id}", - "tokenizer": f"tokenizer_{self.node_id}", - "tokenizer_2": f"tokenizer_2_{self.node_id}", - }.items() + k: node_get_component_info(node_id=self.node_id, manager=components, name=k) + for k in ["text_encoder", "text_encoder_2", "tokenizer", "tokenizer_2"] }, - "scheduler": components.get_model_info(f"scheduler_{self.node_id}"), - "ip_adapter": { - "image_encoder": components.get_model_info( - f"image_encoder_{self.node_id}" - ), - "feature_extractor": components.get_model_info( - f"feature_extractor_{self.node_id}" - ), - "unet": components.get_model_info(unet_model_id, fields="model_id"), - } - if ip_adapter_input - else None, + "scheduler": node_get_component_info(node_id=self.node_id, manager=components, name="scheduler"), } - logger.debug(f" Final components state: {components}") + logger.debug(f" ModelsLoader: Final component_manager state: {components}") return loaded_components - -class EncodePrompt(NodeBase): - def __init__(self, node_id=None): - super().__init__(node_id) - text_block = StableDiffusionXLTextEncoderStep() - self._text_encoder_node = ModularPipeline.from_block(text_block) - - def execute(self, text_encoders, **kwargs): - text_encoder_components = { - "text_encoder": components.get(text_encoders["text_encoder"]["model_id"]), - "text_encoder_2": components.get( - text_encoders["text_encoder_2"]["model_id"] - ), - "tokenizer": components.get(text_encoders["tokenizer"]["model_id"]), - "tokenizer_2": components.get(text_encoders["tokenizer_2"]["model_id"]), - } - - self._text_encoder_node.update_states(**text_encoder_components) - text_state = self._text_encoder_node(**kwargs) - # Return the intermediates dict instead of the PipelineState - return {"embeddings": text_state.intermediates} - - class Scheduler(NodeBase): + def __del__(self): + node_comp_ids = components._lookup_ids(collection=self.node_id) + for comp_id in node_comp_ids: + components.remove(comp_id) + super().__del__() + def execute(self, input_scheduler, scheduler, **kwargs): - scheduler_component = components.get(input_scheduler["model_id"]) + logger.debug(f" Scheduler ({self.node_id}) received parameters:") + logger.debug(f" - input_scheduler: {input_scheduler}") + logger.debug(f" - scheduler: {scheduler}") + logger.debug(f" - kwargs: {kwargs}") + + scheduler_component = components.get_one(input_scheduler["model_id"]) scheduler_cls = getattr( __import__("diffusers", fromlist=[scheduler]), scheduler @@ -576,46 +413,162 @@ def execute(self, input_scheduler, scheduler, **kwargs): else: scheduler_options[key_parts[1]] = value - new_scheduler = scheduler_cls.from_config( - scheduler_component.config, **scheduler_options + schedule_spec = ComponentSpec( + name="scheduler", + type_hint=scheduler_cls, + config=scheduler_component.config, default_creation_method="from_config") + new_scheduler = schedule_spec.create(**scheduler_options) + comp_id = components.add(name="scheduler", component=new_scheduler, collection=self.node_id) + logger.debug(f" Scheduler: new_scheduler: {new_scheduler}") + + return { + "output_scheduler": components.get_model_info(comp_id) + } + +class IPAdapterLoader(NodeBase): + def __init__(self, node_id=None): + super().__init__(node_id) + self._ip_adapter_node = ip_adapter_block + self._ip_adapter_node.setup_loader(component_manager=components, collection=self.node_id) + self._last_embeddings = None # Store last valid embeddings + + def __del__(self): + self._ip_adapter_node.loader.unload_ip_adapter() + node_comp_ids = components._lookup_ids(collection=self.node_id) + for comp_id in node_comp_ids: + components.remove(comp_id) + super().__del__() + + def __call__(self, **kwargs): + # Convert ip_adapter_inputs list to combined map before tracking params + if "ip_adapter_inputs" in kwargs: + if not isinstance(kwargs["ip_adapter_inputs"], list): + kwargs["ip_adapter_inputs"] = [kwargs["ip_adapter_inputs"]] + kwargs["ip_adapter_inputs"] = combine_multi_inputs( + kwargs["ip_adapter_inputs"] + ) + self._old_params = self.params.copy() + return super().__call__(**kwargs) + + def execute(self, unet, guider, ip_adapter_inputs): + logger.debug(f" IPAdapterLoader ({self.node_id}) received parameters:") + logger.debug(f" - unet: {unet}") + logger.debug(f" - guider: {guider}") + logger.debug(f" - ip_adapter_inputs: {ip_adapter_inputs}") + + new_params = {"unet": unet, "guider": guider, "ip_adapter_inputs": ip_adapter_inputs} + unet_changed = check_nested_changes(self._old_params, new_params, "unet") + + ip_adapter_image_changed = check_nested_changes( + self._old_params, new_params, "ip_adapter_inputs.ip_adapter_image" + ) + ip_adapter_config_changed = check_nested_changes( + self._old_params, new_params, "ip_adapter_inputs.ip_adapter_config" + ) + ip_adapter_scale_changed = check_nested_changes( + self._old_params, new_params, "ip_adapter_inputs.scale" ) - # Using add here overrides the previous scheduler if there's one - components.add(f"scheduler_{self.node_id}", new_scheduler) + if not ip_adapter_inputs: + logger.debug(f" unload ip_adapter from components: ip_adapter_{self.node_id}") + self._ip_adapter_node.unload_ip_adapter() + self._last_embeddings = None + return {"ip_adapter_image_embeddings": None, "unet_out": unet} + + repo_ids = ip_adapter_inputs["ip_adapter_config"]["repo_id"] + subfolders = ip_adapter_inputs["ip_adapter_config"]["subfolder"] + weight_names = ip_adapter_inputs["ip_adapter_config"]["weight_name"] + image_encoder_paths = ip_adapter_inputs["ip_adapter_config"][ + "image_encoder_path" + ] + scale = ip_adapter_inputs["scale"] + image = ip_adapter_inputs["ip_adapter_image"] + + need_reload = True if ip_adapter_config_changed or unet_changed else False + if ip_adapter_config_changed: + # Load the first image encoder (they should all be the same) + logger.debug( + f" load image_encoder from repo_id: {repo_ids[0]}, subfolder: {subfolders[0]}, image_encoder_path: {image_encoder_paths[0]}" + ) + image_encoder_spec = ComponentSpec(name="image_encoder", type_hint=CLIPVisionModelWithProjection, repo=repo_ids[0], subfolder=image_encoder_paths[0]) + image_encoder = image_encoder_spec.load(torch_dtype=torch.float16) + self._ip_adapter_node.loader.update(image_encoder=image_encoder) + + if unet_changed: + logger.debug(" update unet") + self._ip_adapter_node.loader.update( + unet=components.get_one(unet["model_id"]), + ) + if guider is not None: + self._ip_adapter_node.loader.update(guider=guider) + if need_reload: + logger.debug(" load ip_adapter(s)") + self._ip_adapter_node.loader.unload_ip_adapter() + self._ip_adapter_node.loader.load_ip_adapter( + repo_ids, + subfolder=subfolders, + weight_name=weight_names, + ) + + if ip_adapter_scale_changed: + logger.debug(f" set ip_adapter scale: {scale}") + self._ip_adapter_node.loader.set_ip_adapter_scale(scale) + + if ip_adapter_image_changed or need_reload: + logger.debug(" process ip_adapter image") + ip_adapter_state = self._ip_adapter_node.run(ip_adapter_image=image) + self._last_embeddings = ip_adapter_state.intermediates + + logger.debug(f" IPAdapterLoader: final component_manager: {components}") return { - "output_scheduler": components.get_model_info(f"scheduler_{self.node_id}") + "ip_adapter_image_embeddings": self._last_embeddings, + "unet_out": components.get_model_info(unet["model_id"]), + "guider_out": guider, } +# (2) pipeline nodes: nodes that run (part of) the pipeline (model inference is usually involved) +# EncodePrompt +# Denoise +# DecodeLatents +# IPAdapterInput (it loads also runs the image_encoder) +class EncodePrompt(NodeBase): + def __init__(self, node_id=None): + super().__init__(node_id) + self._text_encoder_node = text_block + self._text_encoder_node.setup_loader(component_manager=components) + + def execute(self, guider, text_encoders, **kwargs): + logger.debug(f" EncodePrompt ({self.node_id}) received parameters:") + logger.debug(f" - guider: {guider}") + logger.debug(f" - text_encoders: {text_encoders}") + logger.debug(f" - kwargs: {kwargs.keys()}") + + text_encoder_components = { + "text_encoder": components.get_one(text_encoders["text_encoder"]["model_id"]), + "text_encoder_2": components.get_one( + text_encoders["text_encoder_2"]["model_id"] + ), + "tokenizer": components.get_one(text_encoders["tokenizer"]["model_id"]), + "tokenizer_2": components.get_one(text_encoders["tokenizer_2"]["model_id"]), + } + + self._text_encoder_node.loader.update(**text_encoder_components) + if guider is not None: + self._text_encoder_node.loader.update(guider=guider) + + logger.debug(f" EncodePrompt: text_encoder_node loader: {self._text_encoder_node.loader}") + logger.debug(f" EncodePrompt: running text_encoder_node with kwargs: {kwargs}") + text_state = self._text_encoder_node.run(**kwargs) + # Return the intermediates dict instead of the PipelineState + return {"embeddings": text_state.intermediates, "guider_out": guider} + class Denoise(NodeBase): def __init__(self, node_id=None): super().__init__(node_id) - sdxl_auto_blocks = SDXLAutoBlocks() - self._denoise_node = ModularPipeline.from_block(sdxl_auto_blocks) - self._num_inference_steps = 0 - - def step_progress_update(self, _pipe, step, _timestep, data): - if self.node_id: - try: - progress = int((step + 1) / self._num_inference_steps * 100) - asyncio.run_coroutine_threadsafe( - web_server.client_queue.put( - { - "client_id": self._client_id, - "data": { - "type": "progress", - "nodeId": self.node_id, - "progress": progress, - }, - } - ), - web_server.event_loop, - ) - except Exception as e: - logger.warning(f"Error queuing progress update: {str(e)}") - - return data + self._denoise_node = auto_blocks + self._denoise_node.setup_loader(component_manager=components) def execute( self, @@ -623,7 +576,6 @@ def execute( scheduler, embeddings, steps, - cfg, seed, width, height, @@ -631,25 +583,37 @@ def execute( controlnet, ip_adapter_image_embeddings=None, ): - unet_component = components.get(unet["model_id"]) - scheduler_component = components.get(scheduler["model_id"]) - - self._denoise_node.update_states( - unet=unet_component, scheduler=scheduler_component + logger.debug(f" Denoise ({self.node_id}) received parameters:") + logger.debug(f" - unet: {unet}") + logger.debug(f" - scheduler: {scheduler}") + logger.debug(f" - embeddings: {embeddings}") + logger.debug(f" - steps: {steps}") + logger.debug(f" - seed: {seed}") + logger.debug(f" - width: {width}") + logger.debug(f" - height: {height}") + logger.debug(f" - guider: {guider}") + logger.debug(f" - controlnet: {controlnet}") + logger.debug(f" - ip_adapter_image_embeddings: {ip_adapter_image_embeddings}") + + unet_component = components.get_one(unet["model_id"]) + scheduler_component = components.get_one(scheduler["model_id"]) + self._denoise_node.loader.update( + unet=unet_component, + scheduler=scheduler_component ) + + if guider is not None: + self._denoise_node.loader.update(guider=guider) generator = torch.Generator(device="cpu").manual_seed(seed) - self._num_inference_steps = steps denoise_kwargs = { **embeddings, # Now embeddings is already a dict "generator": generator, - "guidance_scale": cfg, "height": height, "width": width, + "num_inference_steps": steps, "output": "latents", - "num_inference_steps": self._num_inference_steps, - "callback_on_step_end": self.step_progress_update, } if ip_adapter_image_embeddings is not None: @@ -664,40 +628,44 @@ def execute( model_ids = controlnet["controlnet_model"]["model_id"] if isinstance(model_ids, list): controlnet_components = [ - components.get(model_id) for model_id in model_ids + components.get_one(model_id) for model_id in model_ids ] controlnet_components = MultiControlNetModel(controlnet_components) else: - controlnet_components = components.get(model_ids) - - self._denoise_node.update_states(controlnet=controlnet_components) + controlnet_components = components.get_one(model_ids) - if guider is not None: - self._denoise_node.update_states(guider=guider["guider"]) - denoise_kwargs["guider_kwargs"] = guider["guider_kwargs"] - else: - # reset the guider in case if it was set before - self._denoise_node.update_states(guider=CFGGuider()) - denoise_kwargs["guider_kwargs"] = None + self._denoise_node.loader.update(controlnet=controlnet_components) - latents = self._denoise_node(**denoise_kwargs) - logger.debug(f" Components after Denoise: {components}") + logger.debug(f" running denoise_node with these kwargs: {denoise_kwargs}") + latents = self._denoise_node.run(**denoise_kwargs) return {"latents": latents} class DecodeLatents(NodeBase): def __init__(self, node_id=None): super().__init__(node_id) - decoder_block = StableDiffusionXLAutoDecodeStep() - self._decoder_node = ModularPipeline.from_block(decoder_block) + self._decoder_node = decoder_block + self._decoder_node.setup_loader(component_manager=components) def execute(self, vae, latents): - vae_component = components.get(vae["model_id"]) - self._decoder_node.update_states(vae=vae_component) - images_output = self._decoder_node(latents=latents, output="images") - return {"images": images_output.images} + logger.debug(f" DecodeLatents ({self.node_id}) received parameters:") + logger.debug(f" - vae: {vae}") + logger.debug(f" - latents: {latents.shape}") + vae_component = components.get_one(vae["model_id"]) + self._decoder_node.loader.update(vae=vae_component) + images_output = self._decoder_node.run(latents=latents, output="images") + return {"images": images_output} + + +# (3) input nodes: nodes that take user inputs and prepare them for other nodes +# Lora +# MultiLora +# PAGOptionalGuider +# Controlnet +# ControlnetUnion +# MultiControlNet class Lora(NodeBase): def execute(self, path, scale, is_local=False): if is_local: @@ -730,19 +698,23 @@ def execute(self, lora_list): class PAGOptionalGuider(NodeBase): - def execute(self, pag_scale, pag_layers): + def execute(self, guidance_scale, skip_layer_guidance_scale): # TODO: Maybe do some validations to ensure correct layers - layers = [item.strip() for item in pag_layers.split(",")] - guider = PAGGuider(pag_applied_layers=layers) - guider_kwargs = {"pag_scale": pag_scale} - return {"guider": {"guider": guider, "guider_kwargs": guider_kwargs}} - - -class APGOptionalGuider(NodeBase): - def execute(self, momentum, rescale_factor): - guider = APGGuider() - guider_kwargs = {"momentum": momentum, "rescale_factor": rescale_factor} - return {"guider": {"guider": guider, "guider_kwargs": guider_kwargs}} + peg_config = { + "guidance_scale": guidance_scale, + "skip_layer_guidance_scale": skip_layer_guidance_scale, + "skip_layer_config": LayerSkipConfig( + indices=[2, 3, 7, 8], + fqn="mid_block.attentions.0.transformer_blocks", + skip_attention=False, + skip_ff=False, + skip_attention_scores=True, + ), + "start": 0.0, + "stop": 1.0, + } + pag_spec = ComponentSpec(name="guider", type_hint=SkipLayerGuidance, config=peg_config, default_creation_method="from_config") + return {"guider": pag_spec} class Controlnet(NodeBase): @@ -847,113 +819,3 @@ def execute( return {"ip_adapter_input": ip_adapter_input} - -class IPAdapter(NodeBase): - def __init__(self, node_id=None): - super().__init__(node_id) - ip_adapter_block = StableDiffusionXLIPAdapterStep() - self._ip_adapter_node = ModularPipeline.from_block(ip_adapter_block) - self._last_embeddings = None # Store last valid embeddings - - def __del__(self): - components.remove(f"image_encoder_{self.node_id}") - components.remove(f"feature_extractor_{self.node_id}") - self._ip_adapter_node.unload_ip_adapter() - super().__del__() - - def __call__(self, **kwargs): - # Convert ip_adapter_inputs list to combined map before tracking params - if "ip_adapter_inputs" in kwargs: - if not isinstance(kwargs["ip_adapter_inputs"], list): - kwargs["ip_adapter_inputs"] = [kwargs["ip_adapter_inputs"]] - kwargs["ip_adapter_inputs"] = combine_multi_inputs( - kwargs["ip_adapter_inputs"] - ) - self._old_params = self.params.copy() - return super().__call__(**kwargs) - - def execute(self, unet, ip_adapter_inputs): - new_params = {"unet": unet, "ip_adapter_inputs": ip_adapter_inputs} - unet_changed = check_nested_changes(self._old_params, new_params, "unet") - - ip_adapter_image_changed = check_nested_changes( - self._old_params, new_params, "ip_adapter_inputs.ip_adapter_image" - ) - ip_adapter_config_changed = check_nested_changes( - self._old_params, new_params, "ip_adapter_inputs.ip_adapter_config" - ) - ip_adapter_scale_changed = check_nested_changes( - self._old_params, new_params, "ip_adapter_inputs.scale" - ) - - if not ip_adapter_inputs: - print(f" unload ip_adapter from components: ip_adapter_{self.node_id}") - self._ip_adapter_node.unload_ip_adapter() - self._last_embeddings = None - return {"ip_adapter_image_embeddings": None, "unet_out": unet} - - repo_ids = ip_adapter_inputs["ip_adapter_config"]["repo_id"] - subfolders = ip_adapter_inputs["ip_adapter_config"]["subfolder"] - weight_names = ip_adapter_inputs["ip_adapter_config"]["weight_name"] - image_encoder_paths = ip_adapter_inputs["ip_adapter_config"][ - "image_encoder_path" - ] - scale = ip_adapter_inputs["scale"] - image = ip_adapter_inputs["ip_adapter_image"] - - need_process = False - - if ip_adapter_config_changed: - # Load the first image encoder (they should all be the same) - logger.debug( - f" load image_encoder from repo_id: {repo_ids[0]}, subfolder: {subfolders[0]}" - ) - image_encoder = CLIPVisionModelWithProjection.from_pretrained( - repo_ids[0], - subfolder=image_encoder_paths[0], - torch_dtype=torch.float16, # TODO: Make configurable - ) - feature_extractor = CLIPImageProcessor(size=224, crop_size=224) - logger.debug( - f" add image_encoder to components: image_encoder_{self.node_id}" - ) - components.add(f"image_encoder_{self.node_id}", image_encoder) - logger.debug( - f" add feature_extractor to components: feature_extractor_{self.node_id}" - ) - components.add(f"feature_extractor_{self.node_id}", feature_extractor) - need_process = True - else: - image_encoder = components.get(f"image_encoder_{self.node_id}") - feature_extractor = components.get(f"feature_extractor_{self.node_id}") - - if unet_changed or ip_adapter_config_changed: - logger.debug(" update ip_adapter state") - self._ip_adapter_node.unload_ip_adapter() - self._ip_adapter_node.update_states( - unet=components.get(unet["model_id"]), - image_encoder=image_encoder, - feature_extractor=feature_extractor, - ) - - logger.debug(" load ip_adapter(s)") - self._ip_adapter_node.load_ip_adapter( - repo_ids, - subfolder=subfolders, - weight_name=weight_names, - ) - need_process = True - - if ip_adapter_scale_changed: - logger.debug(f" set ip_adapter scale: {scale}") - self._ip_adapter_node.set_ip_adapter_scale(scale) - - if ip_adapter_image_changed or need_process: - logger.debug(" process ip_adapter image") - ip_adapter_state = self._ip_adapter_node(ip_adapter_image=image) - self._last_embeddings = ip_adapter_state.intermediates - - return { - "ip_adapter_image_embeddings": self._last_embeddings, - "unet_out": components.get_model_info(unet["model_id"]), - } From 1e4252a88f9a228cc991127d06f865ba86a836d0 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 19 May 2025 22:43:46 +0200 Subject: [PATCH 2/3] update --- __init__.py | 12 ++++++------ mellon-modular-diffusers.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/__init__.py b/__init__.py index 0c885c9..1faee81 100644 --- a/__init__.py +++ b/__init__.py @@ -92,7 +92,7 @@ "text_encoders": { "label": "Text Encoders", "display": "output", - "type": "diffusers_text_encoders", + "type": "text_encoders", }, "unet_out": { "label": "UNet", @@ -107,7 +107,7 @@ "scheduler": { "label": "Scheduler", "display": "output", - "type": "diffusers_scheduler", + "type": "scheduler", }, }, }, @@ -118,7 +118,7 @@ "text_encoders": { "label": "Text Encoders", "display": "input", - "type": "diffusers_text_encoders", + "type": "text_encoders", }, "guider": { "label": "Guider", @@ -156,12 +156,12 @@ "input_scheduler": { "label": "Scheduler", "display": "input", - "type": "diffusers_scheduler", + "type": "scheduler", }, "output_scheduler": { "label": "Scheduler", "display": "output", - "type": "diffusers_scheduler", + "type": "scheduler", }, }, }, @@ -177,7 +177,7 @@ "scheduler": { "label": "Scheduler", "display": "input", - "type": "diffusers_scheduler", + "type": "scheduler", }, "embeddings": { "label": "Embeddings", diff --git a/mellon-modular-diffusers.py b/mellon-modular-diffusers.py index eeb580b..cdf4f35 100644 --- a/mellon-modular-diffusers.py +++ b/mellon-modular-diffusers.py @@ -9,7 +9,6 @@ ClassifierFreeGuidance, SkipLayerGuidance, LayerSkipConfig, - ComponentsManager, StableDiffusionXLAutoPipeline, ComponentSpec, StableDiffusionXLModularLoader @@ -32,7 +31,8 @@ image_encoder_block = auto_blocks.blocks.pop("image_encoder") # Initialize components manager -components = ComponentsManager() +# components = ComponentsManager() +from custom import components def _has_changed(old_params, new_params): From e70fb207664f153b02f4b29bbf3ed97fea17e959 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Wed, 21 May 2025 23:34:12 -1000 Subject: [PATCH 3/3] Update README.MD --- README.MD | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.MD b/README.MD index 19fa292..c676678 100644 --- a/README.MD +++ b/README.MD @@ -9,9 +9,30 @@ cd custom git clone https://github.com/asomoza/mellon-modular-diffusers.git ``` -Uninstall diffusers and install the requirements (just the modular diffusers branch). Probably works just updating too. +Uninstall diffusers and install from the latest modular diffusers branch. https://github.com/huggingface/diffusers/pull/11235 ``` -pip uninstall diffusers -pip install -r requirements.txt +pip uninstall diffusers -y +pip install git+https://github.com/huggingface/diffusers.git@modular-refactor-0501 ``` + +To use diffusers components manager across different custom nodes, add this in your `custom/__init__.py` + + +```python +from diffusers import ComponentsManager +components = ComponentsManager() +``` + +and add this in all your custom nodes module file +```python +from custom import components +``` + +``` +custom/ +├── __init__.py # create your components manager here +├── mellon-modular-diffusers/ +│ └── mellon-modular-diffusers.py # use in this +└── diff-diff/ + └── diff-diff.py # use in this