[Modular] Qwen

[yiyixuxu]

Qwen-Image

• Text2Image
• controlnet
• inpaint
• controlnet + inpaint
• img2img
• controlnet + img2img
• diffdiff (next PR!)

Test Script for Qwen-Image Auto Pipeline

# test modular auto (qwen image)
# use standard repo
import os
import torch

from diffusers import ModularPipeline, ComponentsManager
from diffusers.modular_pipelines.qwenimage import ALL_BLOCKS

from diffusers.utils import load_image
from image_gen_aux import DepthPreprocessor
import numpy as np
from PIL import Image

import logging
logging.getLogger().setLevel(logging.INFO)
logging.getLogger("diffusers").setLevel(logging.INFO)

device = "cuda:2"
output_name_prefix = "test_modular_qwen_out"

components = ComponentsManager()
components.enable_auto_cpu_offload(device=device)

pipeline = ModularPipeline.from_pretrained("Qwen/Qwen-Image", components_manager=components)
print(pipeline)

pipeline.load_components(torch_dtype=torch.bfloat16)

print("pipeline loaded")
print(pipeline)
print(f" ")
print(f"pipeline.blocks")
print(pipeline.blocks)
print(f" ")

print(f"components:")
print(components)
print(f" ")


# test1: text2image with custom height/width

prompt = "现实主义风格的人像摄影作品，画面主体是一位容貌惊艳的女性面部特写。她拥有一头自然微卷的短发，发丝根根分明，蓬松的刘海修饰着额头，增添俏皮感。头上佩戴一顶绿色格子蕾丝边头巾，增添复古与柔美气息。身着一件简约绿色背心裙，在纯白色背景下格外突出。两只手分别握着半个红色桃子，双手轻轻贴在脸颊两侧，营造出可爱又富有创意的视觉效果。  人物表情生动，一只眼睛睁开，另一只微微闭合，展现出调皮与自信的神态。整体构图采用个性视角、非对称构图，聚焦人物主体，增强现场感和既视感。背景虚化处理，层次丰富，景深效果强烈，营造出低光氛围下浓厚的情绪张力。  画面细节精致，色彩生动饱满却不失柔和，呈现出富士胶片独有的温润质感。光影运用充满美学张力，带有轻微超现实的光效处理，提升整体画面高级感。整体风格为现实主义人像摄影，强调细腻的纹理与艺术化的光线表现，堪称一幅细节丰富、氛围拉满的杰作。超清，4K，电影级构图"
inputs = {
    "prompt": prompt,
    "generator": torch.manual_seed(0),
    "negative_prompt": " ",
    "height": 1328,
    "width": 1328,
    "num_inference_steps": 50,
    "num_images_per_prompt": 1,
}

output_images = pipeline(**inputs, output="images")
for i, image in enumerate(output_images):
    assert image.size == (1328, 1328)
    image.save(f"{output_name_prefix}_1_text2image_1328_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_1_text2image_1328_{i}.png')}")


# test2: text2image with default height and width
prompt = "现实主义风格的人像摄影作品，画面主体是一位容貌惊艳的女性面部特写。她拥有一头自然微卷的短发，发丝根根分明，蓬松的刘海修饰着额头，增添俏皮感。头上佩戴一顶绿色格子蕾丝边头巾，增添复古与柔美气息。身着一件简约绿色背心裙，在纯白色背景下格外突出。两只手分别握着半个红色桃子，双手轻轻贴在脸颊两侧，营造出可爱又富有创意的视觉效果。  人物表情生动，一只眼睛睁开，另一只微微闭合，展现出调皮与自信的神态。整体构图采用个性视角、非对称构图，聚焦人物主体，增强现场感和既视感。背景虚化处理，层次丰富，景深效果强烈，营造出低光氛围下浓厚的情绪张力。  画面细节精致，色彩生动饱满却不失柔和，呈现出富士胶片独有的温润质感。光影运用充满美学张力，带有轻微超现实的光效处理，提升整体画面高级感。整体风格为现实主义人像摄影，强调细腻的纹理与艺术化的光线表现，堪称一幅细节丰富、氛围拉满的杰作。超清，4K，电影级构图"
inputs = {
    "prompt": prompt,
    "generator": torch.manual_seed(0),
    "negative_prompt": " ",
    "num_inference_steps": 50,
    "num_images_per_prompt": 1,
}

output_images = pipeline(**inputs, output="images")
for i, image in enumerate(output_images):
    assert image.size == (1024, 1024)
    image.save(f"{output_name_prefix}_1_text2image_1024_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_1_text2image_1024_{i}.png')}")

# test3: inpaint

prompt = "cat wizard with red hat, gandalf, lord of the rings, detailed, fantasy, cute, adorable, Pixar, Disney"
negative_prompt = " "
source = load_image("https://github.com/Trgtuan10/Image_storage/blob/main/cute_cat.png?raw=true")
mask = load_image("https://github.com/Trgtuan10/Image_storage/blob/main/mask_cat.png?raw=true")

strengths = [0.9]

print(f"source.size: {source.size}")

for strength in strengths:
    image = pipeline(
        prompt=prompt,
        negative_prompt=negative_prompt,
        height=source.size[1],
        width=source.size[0],
        image=source,
        mask_image=mask,
        strength=strength,
        num_inference_steps=35,
        generator=torch.Generator(device="cuda").manual_seed(42),
        output="images"
    )[0]
    image.save(f"{output_name_prefix}_2_inpaint_{strength}.png")
    assert image.size == source.size
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_2_inpaint_{strength}.png')}")


# test4: controlnet

print("test controlnet")

# canny
from diffusers import QwenImageControlNetModel, QwenImageMultiControlNetModel

controlnet_spec = pipeline.get_component_spec("controlnet")
controlnet_spec.repo = "InstantX/Qwen-Image-ControlNet-Union"
controlnet = controlnet_spec.load(torch_dtype=torch.bfloat16)

pipeline.update_components(controlnet=controlnet)

print("pipeline (with controlnet)")
print(pipeline)
print(f" ")
print("components (with controlnet)")
print(components)
print(f" ")


control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/qwencond_input.png")
prompt = "Aesthetics art, traditional asian pagoda, elaborate golden accents, sky blue and white color palette, swirling cloud pattern, digital illustration, east asian architecture, ornamental rooftop, intricate detailing on building, cultural representation."
controlnet_conditioning_scale = 1.0

print(f"control_image.size: {control_image.size}")

images = pipeline(
    prompt=prompt,
    negative_prompt=" ",
    control_image=control_image,
    controlnet_conditioning_scale=controlnet_conditioning_scale,
    width=control_image.size[0],
    height=control_image.size[1],
    generator=torch.Generator(device="cuda").manual_seed(42),
    output="images"
)
for i, image in enumerate(images):
    assert image.size == control_image.size
    image.save(f"{output_name_prefix}_3_controlnet_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_3_controlnet_{i}.png')}")

print(f" components:")
print(components)
print(f" ")

# test5: multi-controlnet 
multi_controlnet = QwenImageMultiControlNetModel([controlnet])
pipeline.update_components(controlnet=multi_controlnet)

images = pipeline(
    prompt=prompt,
    negative_prompt=" ",
    control_image=[control_image, control_image],
    controlnet_conditioning_scale=[controlnet_conditioning_scale/2, controlnet_conditioning_scale/2],
    width=control_image.size[0],
    height=control_image.size[1],
    generator=torch.Generator(device="cuda").manual_seed(42),
    output="images"
)
for i, image in enumerate(images):
    assert image.size == control_image.size
    image.save(f"{output_name_prefix}_3_controlnet_multi_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_3_controlnet_multi_{i}.png')}")



# test6: multi-controlnet, default height/width and num_images_per_prompt = 2
multi_controlnet = QwenImageMultiControlNetModel([controlnet])
pipeline.update_components(controlnet=multi_controlnet)

images = pipeline(
    prompt=prompt,
    negative_prompt=" ",
    control_image=[control_image, control_image],
    controlnet_conditioning_scale=[controlnet_conditioning_scale/2, controlnet_conditioning_scale/2],
    num_images_per_prompt=2,
    generator=torch.Generator(device="cuda").manual_seed(42),
    output="images"
)
for i, image in enumerate(images):
    assert image.size == (1024, 1024)
    image.save(f"{output_name_prefix}_3_controlnet_multi_2_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_3_controlnet_multi_2_{i}.png')}")

# test7: controlnet + inpaint

pipeline.update_components(controlnet=controlnet)

prompt = "a blue robot singing opera with human-like expressions"
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robot.png")

head_mask = np.zeros_like(image)
head_mask[65:580,300:642] = 255
mask_image = Image.fromarray(head_mask)

processor = DepthPreprocessor.from_pretrained("LiheYoung/depth-anything-large-hf")
control_image = processor(image)[0].convert("RGB")

print(f"image.size: {image.size}")
print(f"control_image.size: {control_image.size}")
print(f"mask_image.size: {mask_image.size}")

image_output = pipeline(
    prompt=prompt,
    image=image,
    mask_image=mask_image,
    control_image=control_image,
    strength=0.9,
    num_inference_steps=30,
    output="images",
)
for i, image in enumerate(image_output):
    assert image.size == (1024, 1024)
    image.save(f"{output_name_prefix}_4_controlnet_inpaint_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_4_controlnet_inpaint_{i}.png')}")



# test8: update guider (PAG)

from diffusers import LayerSkipConfig, PerturbedAttentionGuidance

# make a copy of the cfg guider to swith back later
cfg_guider_spec = pipeline.get_component_spec("guider")

pag_config = LayerSkipConfig(indices=[2, 9], skip_attention=False, skip_attention_scores=True, skip_ff=False)
pag_guider = PerturbedAttentionGuidance(
    guidance_scale=5.0, perturbed_guidance_scale=2.5, perturbed_guidance_config=pag_config
)
pipeline.update_components(guider=pag_guider)

print("pipeline.guider")
print(pipeline.guider)


# prompt = "A painting of a squirrel eating a burger"
prompt = "现实主义风格的人像摄影作品，画面主体是一位容貌惊艳的女性面部特写。她拥有一头自然微卷的短发，发丝根根分明，蓬松的刘海修饰着额头，增添俏皮感。头上佩戴一顶绿色格子蕾丝边头巾，增添复古与柔美气息。身着一件简约绿色背心裙，在纯白色背景下格外突出。两只手分别握着半个红色桃子，双手轻轻贴在脸颊两侧，营造出可爱又富有创意的视觉效果。  人物表情生动，一只眼睛睁开，另一只微微闭合，展现出调皮与自信的神态。整体构图采用个性视角、非对称构图，聚焦人物主体，增强现场感和既视感。背景虚化处理，层次丰富，景深效果强烈，营造出低光氛围下浓厚的情绪张力。  画面细节精致，色彩生动饱满却不失柔和，呈现出富士胶片独有的温润质感。光影运用充满美学张力，带有轻微超现实的光效处理，提升整体画面高级感。整体风格为现实主义人像摄影，强调细腻的纹理与艺术化的光线表现，堪称一幅细节丰富、氛围拉满的杰作。超清，4K，电影级构图"
inputs = {
    "prompt": prompt,
    "generator": torch.manual_seed(0),
    "negative_prompt": " ",
    "height": 1328,
    "width": 1328,
    "num_inference_steps": 50,
    "num_images_per_prompt": 1,
}

output_images = pipeline(**inputs, output="images")
for i, image in enumerate(output_images):
    assert image.size == (1328, 1328)
    image.save(f"{output_name_prefix}_5_guider_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_5_guider_{i}.png')}")



# test9: img2img

print(f"pipeline.guider")
print(pipeline.guider)
pipeline.update_components(guider=cfg_guider_spec)
print(f"pipeline.guider")
print(pipeline.guider)


init_image = load_image("https://github.com/Trgtuan10/Image_storage/blob/main/cute_cat.png?raw=true")
prompt = "wizard dog, Gandalf-inspired, Lord of the Rings aesthetic, majestic yet cute, Studio Ghibli style"

strengths = [0.6, 0.9, 1.0]

for s in strengths:
    out = pipeline(
        prompt=prompt,
        image=init_image,
        height=init_image.size[1],
        width=init_image.size[0],
        strength=s,
        num_inference_steps=35,
        generator=torch.Generator(device="cuda").manual_seed(42),
    )
    out.images[0].save(f"yiyi_test_5_output_6_img2img_{s}.png")
    print(f"image saved at {os.path.abspath(f'yiyi_test_5_output_6_img2img_{s}.png')}")


# test10: img2img + controlnet

# extract canny
get_image_step = ModularPipeline.from_pretrained("YiYiXu/image_inputs", trust_remote_code=True)
control_image = get_image_step(image=init_image, processor_id="canny", output="image")
controlnet_conditioning_scale = 1.0

strengths = [0.6, 0.9, 1.0]

for s in strengths:
    out = pipeline(
        prompt=prompt,
        image=init_image,
        control_image=control_image,
        controlnet_conditioning_scale=controlnet_conditioning_scale,
        height=init_image.size[1],
        width=init_image.size[0],
        strength=s,
        num_inference_steps=35,
        generator=torch.Generator(device="cuda").manual_seed(42),
    )
    out.images[0].save(f"yiyi_test_5_output_6_img2img_controlnet_{s}.png")
    print(f"image saved at {os.path.abspath(f'yiyi_test_5_output_6_img2img_controlnet_{s}.png')}")

print(f" components:")
print(components)
print(f" ")

QwenImage Edit

• Edit
• Edit + Inpaint
• diffdiff (next PR!)

Test script for QwenImage-Edit in Modular

# test modular auto (qwen image edit)
# use standard repo
import os
import torch

from diffusers import ModularPipeline, ComponentsManager
from diffusers.modular_pipelines.qwenimage import ALL_BLOCKS

from diffusers.utils import load_image
from image_gen_aux import DepthPreprocessor
import numpy as np
from PIL import Image

import logging
logging.getLogger().setLevel(logging.INFO)
logging.getLogger("diffusers").setLevel(logging.INFO)

device = "cuda:2"
output_name_prefix = "test_modular_qwen_edit_output"

components = ComponentsManager()
components.enable_auto_cpu_offload(device=device)

pipeline = ModularPipeline.from_pretrained("Qwen/Qwen-Image-Edit", components_manager=components)
print(pipeline)

pipeline.load_components(torch_dtype=torch.bfloat16)

print("pipeline loaded")
print(pipeline)
print(f" ")
print(f"pipeline.blocks")
print(pipeline.blocks)
print(f" ")

print(f"components:")
print(components)
print(f" ")


# edit

prompt = "change the hat to red"
negative_prompt = " "
source = load_image("https://github.com/Trgtuan10/Image_storage/blob/main/cute_cat.png?raw=true")
mask = load_image("https://github.com/Trgtuan10/Image_storage/blob/main/mask_cat.png?raw=true")

# edit
print(f"source size: {source.size}")
print(f"mask size: {mask.size}")
output_images = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=source,
    num_inference_steps=35,
    generator=torch.Generator(device="cuda").manual_seed(42),
).images
for i, image in enumerate(output_images):
    image.save(f"{output_name_prefix}_1_edit_{source.size[1]}_{source.size[0]}_{i}.png")
    print(f"image size: {image.size}")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_1_edit_{source.size[1]}_{source.size[0]}_{i}.png')}")

# edit + update guider (guidance_scale=4.5)

cfg_guider_spec = pipeline.get_component_spec("guider")
cfg_guider_spec.config["guidance_scale"] = 4.5
pipeline.update_components(guider=cfg_guider_spec)

print(f" print pipeline.guider")
print(pipeline.guider)

output_images = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=source,
    num_inference_steps=35,
    generator=torch.Generator(device="cuda").manual_seed(42),
).images
for i, image in enumerate(output_images):
    image.save(f"{output_name_prefix}_2_edit_guidance_scale_4.5_{i}.png")
    print(f"image size: {image.size}")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_2_edit_guidance_scale_4.5_{i}.png')}")

# edit + num_images_per_prompt==2
output_images = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=source,
    num_inference_steps=35,
    generator=torch.Generator(device="cuda").manual_seed(42),
    num_images_per_prompt=2,
).images
for i, image in enumerate(output_images):
    image.save(f"{output_name_prefix}_3_edit_num_images_per_prompt_2_{i}.png")
    print(f"image size: {image.size}")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_3_edit_num_images_per_prompt_2_{i}.png')}")

# edit + pag 
from diffusers import LayerSkipConfig, PerturbedAttentionGuidance

pag_config = LayerSkipConfig(indices=[2, 9], skip_attention=False, skip_attention_scores=True, skip_ff=False)
pag_guider = PerturbedAttentionGuidance(
    guidance_scale=5.0, perturbed_guidance_scale=2.5, perturbed_guidance_config=pag_config
)
pipeline.update_components(guider=pag_guider)

print(f" print pipeline.guider")
print(pipeline.guider)

output_images = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=source,
    num_inference_steps=35,
    generator=torch.Generator(device="cuda").manual_seed(42),
).images
for i, image in enumerate(output_images):
    image.save(f"{output_name_prefix}_4_edit_pag_{i}.png")
    print(f"image size: {image.size}")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_4_edit_pag_{i}.png')}")


# inpaint

strengths = [0.9, 1.0]
print(f" pipeline.guider")
print(pipeline.guider)

for strength in strengths:
    image_output = pipeline(
        prompt=prompt,
        negative_prompt=negative_prompt,
        image=source,
        mask_image=mask,
        strength=strength,
        num_inference_steps=35,
        generator=torch.Generator(device="cuda").manual_seed(42),
    ).images[0]
    image_output.save(f"{output_name_prefix}_5_inpaint_pag_{strength}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_5_inpaint_pag_{strength}.png')}")


# edit + cfg guider

pipeline.update_components(guider=cfg_guider_spec)

print(f" print pipeline.guider")
print(pipeline.guider)

input_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/qwenedit_input.png")
seed = 43
prompt = "The woman is displaying a plush toy product in her hand, while preserving her exact facial features, expression, clothing, and pose. Maintain the same background, natural lighting, and overall photographic composition and style."
inputs = {
    "image": input_image,
    "prompt": prompt,
    "generator": torch.Generator(device=device).manual_seed(seed),
    "num_inference_steps": 50,
    # "height": 1024,
    # "width": 1024,
}

output_images = pipeline(**inputs, output="images")
for i, image in enumerate(output_images):
    image.save(f"{output_name_prefix}_6_inpaint_cfg_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_6_inpaint_cfg_{i}.png')}")




# edit + cfg guider + custom size

pipeline.update_components(guider=cfg_guider_spec)

print(f" print pipeline.guider")
print(pipeline.guider)

input_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/qwenedit_input.png")
seed = 43
prompt = "The woman is displaying a plush toy product in her hand, while preserving her exact facial features, expression, clothing, and pose. Maintain the same background, natural lighting, and overall photographic composition and style."
inputs = {
    "image": input_image,
    "prompt": prompt,
    "generator": torch.Generator(device=device).manual_seed(seed),
    "num_inference_steps": 50,
    "height": 1024,
    "width": 1024,
}

output_images = pipeline(**inputs, output="images")
for i, image in enumerate(output_images):
    assert image.size == (1024, 1024)
    image.save(f"{output_name_prefix}_7_inpaint_cfg_custom_size_{i}.png")
    print(f"image saved at {os.path.abspath(f'{output_name_prefix}_7_inpaint_cfg_custom_size_{i}.png')}")

How to Use

the shorter version, check the test scripts above for complete, runnable examples

to load from standard repo

import torch
from diffusers import ModularPipeline, ComponentsManager

repo_id = "Qwen/Qwen-Image"
# repo_id = "Qwen/Qwen-Image-Edit"

components = ComponentsManager()
components.enable_auto_cpu_offload(device="cuda")
pipeline = ModularPipeline.from_pretrained(repo_id, components_manager=components)
pipeline.load_components(torch_dtype=torch.float16)
print(pipeline)

add controlnet (we currently only have controlnet for Qwen-Image)

from diffusers import QwenImageControlNetModel, QwenImageMultiControlNetModel

controlnet_spec = pipeline.get_component_spec("controlnet")
controlnet_spec.repo = "InstantX/Qwen-Image-ControlNet-Union"
controlnet = controlnet_spec.load(torch_dtype=torch.bfloat16)
pipeline.update_components(controlnet=controlnet)

update guider

change guidance_scale

cfg_guider_spec = pipeline.get_component_spec("guider")
cfg_guider_spec.config["guidance_scale"] = 4.5
pipeline.update_components(guider=cfg_guider_spec)

use a different guidance method

from diffusers import LayerSkipConfig, PerturbedAttentionGuidance

pag_config = LayerSkipConfig(indices=[2, 9], skip_attention=False, skip_attention_scores=True, skip_ff=False)
pag_guider = PerturbedAttentionGuidance(
    guidance_scale=5.0, perturbed_guidance_scale=2.5, perturbed_guidance_config=pag_config
)
pipeline.update_components(guider=pag_guider)

to run inference

You can use same pipeline to run all tasks we support, the code is pretty much same as in regular pipelines

# text2image
pipeline(prompt=prompt, ...).images[0]

# image2image
pipeline(prompt=prompt, image=...,  strength=..., ...).images[0]

# inpaint 
pipeline(prompt=prompt, image=...,  mask_image=..., strength=...,).images[0]

add controlnet to text2image, img2img, inpaint, just pass control_image along with any other controlnet related arguments

# text2image + controlnet
pipeline(prompt=prompt, control_image=, ...).images[0]

# image2image + controlnet
pipeline(prompt=prompt, image=...,  strength=..., control_image=..., ...).images[0]

# inpaint + controlnet
pipeline(prompt=prompt, image=...,  mask_image=..., strength=..., control_image=..., ...).images[0]

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

[sayakpaul]

Thank you!

[sayakpaul]

For my understanding. This one is for?

[yiyixuxu]

for guiders/hooks

[sayakpaul]

Really nice!

(not for this PR, we could attempt to have an example of the processor for an inpaint pipeline)

[sayakpaul]

For the QwenImage pipeline, guidance_scale is akin to the one we have in Flux. However, I think we want to enable CFG with this which is done through true_cfg_scale. Should this be taken into account?

[yiyixuxu]

good questions,

the true_cfg_scale in flux/qwen is actually just guidance_scale in every other pipeline - it is part of guider and should be set in guider

we had to use a different name (true_cfg_scale) for flux because guidance_scale was already taken to use as an input for distilled model. I think it would have been a lot better if we had gave the distilled guidance a different name so that we can keep the definition of guidance_scale consistent across all pipelines

I'd like to fix it here in modular. IMO It won't confuse user too much because they won't be able to use guidance_scale or true_cfg_scale during runtime in modular as it is, so they will have to take some time to figure out how to use guidance properly and we will have chance to explain.

cc @DN6 @asomoza too, let me know if you have any thoughts around this

[sayakpaul]

I think it would have been a lot better if we had gave the distilled guidance a different name so that we can keep the definition of guidance_scale consistent across all pipelines

I like this point a lot! However, we have guidance_scale in Flux (without the use of the Guider component):

diffusers/src/diffusers/modular_pipelines/flux/before_denoise.py

Line 337 in 67ffa70

InputParam("guidance_scale", default=3.5),

Maybe we could change that to something better suited (something like distilled_guidance_scale). This way, we can keep the meaning of guidance_scale consistent across the pipelines.

[asomoza]

I completely agree, let's keep the guidance_scale consistent and use a different one for the distilled models.

[DN6]

So the proposal is that guidance scale would always imply CFG guidance scale?

I would argue that keeping guidance_scale for all guidance methods makes sense since it implies how large of a step you want take in the guidance direction.

Alternatively we could introduce the concept of a DistilledGuidance guider which is effectively a no-op and it makes it more explicit about exactly what's happening with latents rather than having to introduce new scale parameters, internal checks for negative embeds or checks like self._is_cfg_enabled?

[sayakpaul]

Cool. Just for clarity, guidance_scale here would mean what true_cfg_scale means in the QwenImage pipelines, right?

[yiyixuxu]

yes

[yiyixuxu]

and we don't require passing a negative prompt to use it in modular

[sayakpaul]

Just a few notes for understanding.

I see that the default negative prompt we're using in modular is "":

diffusers/src/diffusers/modular_pipelines/qwenimage/encoders.py

Line 375 in f50b18e

negative_prompt = block_state.negative_prompt or ""

However, Qwen usually does " ". So, in case we don't require the user to not pass e negative prompt to enable CFG, maybe we could use " " instead of "".

[yiyixuxu]

sounds good!

[jferments]

Will there be any features added soon for training Qwen models with HF libraries? What are the major barriers right now to making this happen?

[sayakpaul]

This is not the right PR to discuss Qwen training.

You can check out https://github.com/huggingface/diffusers/blob/main/examples/dreambooth/README_qwen.md as well as https://github.com/ostris/ai-toolkit for training with HF libs.

[yiyixuxu]

Suggested change

	else:
	raise ValueError(f"Unsupported image type: {type(image)}")

remove this for now, will update in a seperate PR since it requires changes into the regular qwen/flux pipelines