v2

Author: hlky

src/diffusers/loaders/ip_adapter.py

@@ -549,25 +549,41 @@ def load_ip_adapter(
         # load ip-adapter into transformer
         self.transformer._load_ip_adapter_weights(state_dicts, low_cpu_mem_usage=low_cpu_mem_usage)
 
-    def set_ip_adapter_scale(self, scale):
+    def set_ip_adapter_scale(self, scale: Union[float, List[float], List[List[float]]]):
         """
         Set IP-Adapter scales per-transformer block. Input `scale` could be a single config or a list of configs for
-        granular control over each IP-Adapter behavior. A config can be a float or a dictionary.
+        granular control over each IP-Adapter behavior. A config can be a float or a list.
+
+        `float` is converted to list and repeated for the number of blocks and the number of IP adapters.
+        `List[float]` length match the number of blocks, it is repeated for each IP adapter.
+        `List[List[float]]` must match the number of IP adapters and each must match the number of blocks.
 
         Example:
 
         ```py
         # To use original IP-Adapter
         scale = 1.0
         pipeline.set_ip_adapter_scale(scale)
+        def LinearStrengthModel(start, finish, size):
+            return [
+                (start + (finish - start) * (i / (size - 1))) for i in range(size)
+            ]
+
+        ip_strengths = LinearStrengthModel(0.3, 0.92, 19)
+        pipeline.set_ip_adapter_scale(ip_strengths)
         ```
         """
         transformer = self.transformer
         if not isinstance(scale, list):
+            scale = [[scale] * transformer.config.num_layers]
+        elif isinstance(scale, list) and isinstance(scale[0], int) or isinstance(scale[0], float):
+            if len(scale) != transformer.config.num_layers:
+                raise ValueError(f"Expected list of {transformer.config.num_layers} scales, got {len(scale)}.")
             scale = [scale]
 
         scale_configs = scale
 
+        key_id = 0
         for attn_name, attn_processor in transformer.attn_processors.items():
             if isinstance(attn_processor, (FluxIPAdapterJointAttnProcessor2_0)):
                 if len(scale_configs) != len(attn_processor.scale):
@@ -578,7 +594,8 @@ def set_ip_adapter_scale(self, scale):
                 elif len(scale_configs) == 1:
                     scale_configs = scale_configs * len(attn_processor.scale)
                 for i, scale_config in enumerate(scale_configs):
-                    attn_processor.scale[i] = scale_config
+                    attn_processor.scale[i] = scale_config[key_id]
+                key_id += 1
 
     def unload_ip_adapter(self):
         """

src/diffusers/loaders/transformer_flux.py

@@ -62,10 +62,11 @@ def _convert_ip_adapter_image_proj_to_diffusers(self, state_dict, low_cpu_mem_us
 
         if "proj.weight" in state_dict:
             # IP-Adapter
-            # TODO: fix for XLabs-AI/flux-ip-adapter-v2
             num_image_text_embeds = 4
+            if state_dict["proj.weight"].shape[0] == 65536:
+                num_image_text_embeds = 16
             clip_embeddings_dim = state_dict["proj.weight"].shape[-1]
-            cross_attention_dim = state_dict["proj.weight"].shape[0] // 4
+            cross_attention_dim = state_dict["proj.weight"].shape[0] // num_image_text_embeds
 
             with init_context():
                 image_projection = ImageProjection(
@@ -124,9 +125,11 @@ def _convert_ip_adapter_attn_to_diffusers(self, state_dicts, low_cpu_mem_usage=F
                 num_image_text_embeds = []
                 for state_dict in state_dicts:
                     if "proj.weight" in state_dict["image_proj"]:
+                        num_image_text_embed = 4
+                        if state_dict["image_proj"]["proj.weight"].shape[0] == 65536:
+                            num_image_text_embed = 16
                         # IP-Adapter
-                        # TODO: change for XLabs-AI/flux-ip-adapter-v2
-                        num_image_text_embeds += [4]
+                        num_image_text_embeds += [num_image_text_embed]
 
                 with init_context():
                     attn_procs[name] = attn_processor_class(