Skip to content

Flux1 Kontext (Dev) support #707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 29, 2025
Merged

Flux1 Kontext (Dev) support #707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3d50c7
Kontext support
stduhpf Jun 27, 2025
8967889
add edit mode
leejet Jun 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion diffusion_model.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ struct DiffusionModel {
struct ggml_tensor* c_concat,
struct ggml_tensor* y,
struct ggml_tensor* guidance,
std::vector<ggml_tensor*> ref_latents = {},
int num_video_frames = -1,
std::vector<struct ggml_tensor*> controls = {},
float control_strength = 0.f,
Expand Down Expand Up @@ -68,6 +69,7 @@ struct UNetModel : public DiffusionModel {
struct ggml_tensor* c_concat,
struct ggml_tensor* y,
struct ggml_tensor* guidance,
std::vector<ggml_tensor*> ref_latents = {},
int num_video_frames = -1,
std::vector<struct ggml_tensor*> controls = {},
float control_strength = 0.f,
Expand Down Expand Up @@ -118,6 +120,7 @@ struct MMDiTModel : public DiffusionModel {
struct ggml_tensor* c_concat,
struct ggml_tensor* y,
struct ggml_tensor* guidance,
std::vector<ggml_tensor*> ref_latents = {},
int num_video_frames = -1,
std::vector<struct ggml_tensor*> controls = {},
float control_strength = 0.f,
Expand Down Expand Up @@ -169,13 +172,14 @@ struct FluxModel : public DiffusionModel {
struct ggml_tensor* c_concat,
struct ggml_tensor* y,
struct ggml_tensor* guidance,
std::vector<ggml_tensor*> ref_latents = {},
int num_video_frames = -1,
std::vector<struct ggml_tensor*> controls = {},
float control_strength = 0.f,
struct ggml_tensor** output = NULL,
struct ggml_context* output_ctx = NULL,
std::vector<int> skip_layers = std::vector<int>()) {
return flux.compute(n_threads, x, timesteps, context, c_concat, y, guidance, output, output_ctx, skip_layers);
return flux.compute(n_threads, x, timesteps, context, c_concat, y, guidance, ref_latents, output, output_ctx, skip_layers);
}
};

Expand Down
86 changes: 82 additions & 4 deletions examples/cli/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,15 @@ const char* modes_str[] = {
"txt2img",
"img2img",
"img2vid",
"edit",
"convert",
};

enum SDMode {
TXT2IMG,
IMG2IMG,
IMG2VID,
EDIT,
CONVERT,
MODE_COUNT
};
Expand All @@ -89,6 +91,7 @@ struct SDParams {
std::string input_path;
std::string mask_path;
std::string control_image_path;
std::vector<std::string> ref_image_paths;

std::string prompt;
std::string negative_prompt;
Expand Down Expand Up @@ -154,6 +157,10 @@ void print_params(SDParams params) {
printf(" init_img: %s\n", params.input_path.c_str());
printf(" mask_img: %s\n", params.mask_path.c_str());
printf(" control_image: %s\n", params.control_image_path.c_str());
printf(" ref_images_paths:\n");
for (auto& path : params.ref_image_paths) {
printf(" %s\n", path.c_str());
};
printf(" clip on cpu: %s\n", params.clip_on_cpu ? "true" : "false");
printf(" controlnet cpu: %s\n", params.control_net_cpu ? "true" : "false");
printf(" vae decoder on cpu:%s\n", params.vae_on_cpu ? "true" : "false");
Expand Down Expand Up @@ -208,6 +215,7 @@ void print_usage(int argc, const char* argv[]) {
printf(" -i, --init-img [IMAGE] path to the input image, required by img2img\n");
printf(" --mask [MASK] path to the mask image, required by img2img with mask\n");
printf(" --control-image [IMAGE] path to image condition, control net\n");
printf(" -r, --ref_image [PATH] reference image for Flux Kontext models (can be used multiple times) \n");
printf(" -o, --output OUTPUT path to write result image to (default: ./output.png)\n");
printf(" -p, --prompt [PROMPT] the prompt to render\n");
printf(" -n, --negative-prompt PROMPT the negative prompt (default: \"\")\n");
Expand Down Expand Up @@ -243,7 +251,7 @@ void print_usage(int argc, const char* argv[]) {
printf(" This might crash if it is not supported by the backend.\n");
printf(" --control-net-cpu keep controlnet in cpu (for low vram)\n");
printf(" --canny apply canny preprocessor (edge detection)\n");
printf(" --color Colors the logging tags according to level\n");
printf(" --color colors the logging tags according to level\n");
printf(" -v, --verbose print extra info\n");
}

Expand Down Expand Up @@ -629,6 +637,12 @@ void parse_args(int argc, const char** argv, SDParams& params) {
break;
}
params.skip_layer_end = std::stof(argv[i]);
} else if (arg == "-r" || arg == "--ref-image") {
if (++i >= argc) {
invalid_arg = true;
break;
}
params.ref_image_paths.push_back(argv[i]);
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
print_usage(argc, argv);
Expand Down Expand Up @@ -657,7 +671,13 @@ void parse_args(int argc, const char** argv, SDParams& params) {
}

if ((params.mode == IMG2IMG || params.mode == IMG2VID) && params.input_path.length() == 0) {
fprintf(stderr, "error: when using the img2img mode, the following arguments are required: init-img\n");
fprintf(stderr, "error: when using the img2img/img2vid mode, the following arguments are required: init-img\n");
print_usage(argc, argv);
exit(1);
}

if (params.mode == EDIT && params.ref_image_paths.size() == 0) {
fprintf(stderr, "error: when using the edit mode, the following arguments are required: ref-image\n");
print_usage(argc, argv);
exit(1);
}
Expand Down Expand Up @@ -826,6 +846,7 @@ int main(int argc, const char* argv[]) {
uint8_t* input_image_buffer = NULL;
uint8_t* control_image_buffer = NULL;
uint8_t* mask_image_buffer = NULL;
std::vector<sd_image_t> ref_images;

if (params.mode == IMG2IMG || params.mode == IMG2VID) {
vae_decode_only = false;
Expand Down Expand Up @@ -877,6 +898,37 @@ int main(int argc, const char* argv[]) {
free(input_image_buffer);
input_image_buffer = resized_image_buffer;
}
} else if (params.mode == EDIT) {
vae_decode_only = false;
for (auto& path : params.ref_image_paths) {
int c = 0;
int width = 0;
int height = 0;
uint8_t* image_buffer = stbi_load(path.c_str(), &width, &height, &c, 3);
if (image_buffer == NULL) {
fprintf(stderr, "load image from '%s' failed\n", path.c_str());
return 1;
}
if (c < 3) {
fprintf(stderr, "the number of channels for the input image must be >= 3, but got %d channels\n", c);
free(image_buffer);
return 1;
}
if (width <= 0) {
fprintf(stderr, "error: the width of image must be greater than 0\n");
free(image_buffer);
return 1;
}
if (height <= 0) {
fprintf(stderr, "error: the height of image must be greater than 0\n");
free(image_buffer);
return 1;
}
ref_images.push_back({(uint32_t)width,
(uint32_t)height,
3,
image_buffer});
}
}

sd_ctx_t* sd_ctx = new_sd_ctx(params.model_path.c_str(),
Expand Down Expand Up @@ -968,7 +1020,7 @@ int main(int argc, const char* argv[]) {
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end);
} else {
} else if (params.mode == IMG2IMG || params.mode == IMG2VID) {
sd_image_t input_image = {(uint32_t)params.width,
(uint32_t)params.height,
3,
Expand Down Expand Up @@ -1038,6 +1090,32 @@ int main(int argc, const char* argv[]) {
params.skip_layer_start,
params.skip_layer_end);
}
} else { // EDIT
results = edit(sd_ctx,
ref_images.data(),
ref_images.size(),
params.prompt.c_str(),
params.negative_prompt.c_str(),
params.clip_skip,
params.cfg_scale,
params.guidance,
params.eta,
params.width,
params.height,
params.sample_method,
params.sample_steps,
params.strength,
params.seed,
params.batch_count,
control_image,
params.control_strength,
params.style_ratio,
params.normalize_input,
params.skip_layers.data(),
params.skip_layers.size(),
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end);
}

if (results == NULL) {
Expand Down Expand Up @@ -1117,4 +1195,4 @@ int main(int argc, const char* argv[]) {
free(input_image_buffer);

return 0;
}
}
Loading
Loading