Skip to content

Commit f3c2c64

Browse files
stduhpfstduhpf
committed
add uncond slg variant
1 parent fb44a88 commit f3c2c64

File tree

3 files changed

+50
-21
lines changed

3 files changed

+50
-21
lines changed

‎examples/cli/main.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ struct SDParams {
129129
float slg_scale = 0.0f;
130130
float skip_layer_start = 0.01f;
131131
float skip_layer_end = 0.2f;
132+
bool slg_uncond = false;
132133

133134
float apg_eta = 1.0f;
134135
float apg_momentum = 0.0f;
@@ -225,11 +226,14 @@ void print_usage(int argc, const char* argv[]) {
225226
printf(" (replaces saturation with a smooth approximation)\n");
226227
printf(" --slg-scale SCALE skip layer guidance (SLG) scale, only for DiT models: (default: 0)\n");
227228
printf(" 0 means disabled, a value of 2.5 is nice for sd3.5 medium\n");
228-
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
229+
printf(" --slg-uncond Use CFG's forward pass for SLG instead of a separate pass, only for DiT models\n");
230+
printf(" To use this, it's recommended to keep slg-scale to 0, both for performance and quality reasons\n");
231+
printf(" This should be slightly faster than normal cfg when cfg_scale != 1.\n");
229232
printf(" --skip-layers LAYERS Layers to skip for SLG steps: (default: [7,8,9])\n");
230233
printf(" --skip-layer-start START SLG enabling point: (default: 0.01)\n");
231234
printf(" --skip-layer-end END SLG disabling point: (default: 0.2)\n");
232235
printf(" SLG will be enabled at step int([STEPS]*[START]) and disabled at int([STEPS]*[END])\n");
236+
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
233237
printf(" --strength STRENGTH strength for noising/unnoising (default: 0.75)\n");
234238
printf(" --style-ratio STYLE-RATIO strength for keeping input identity (default: 20%%)\n");
235239
printf(" --control-strength STRENGTH strength to apply Control Net (default: 0.9)\n");
@@ -590,6 +594,8 @@ void parse_args(int argc, const char** argv, SDParams& params) {
590594
break;
591595
}
592596
params.slg_scale = std::stof(argv[i]);
597+
} else if (arg == "--slg-uncond") {
598+
params.slg_uncond = true;
593599
} else if (arg == "--skip-layers") {
594600
if (++i >= argc) {
595601
invalid_arg = true;
@@ -766,6 +772,7 @@ std::string get_image_params(SDParams params, int64_t seed) {
766772
}
767773
}
768774
if (params.slg_scale != 0 && params.skip_layers.size() != 0) {
775+
parameter_string += "Unconditional SLG: " + std::string(params.slg_uncond ? "True" : "False") + ", ";
769776
parameter_string += "SLG scale: " + std::to_string(params.cfg_scale) + ", ";
770777
parameter_string += "Skip layers: [";
771778
for (const auto& layer : params.skip_layers) {
@@ -1013,7 +1020,8 @@ int main(int argc, const char* argv[]) {
10131020
params.skip_layers.size(),
10141021
params.slg_scale,
10151022
params.skip_layer_start,
1016-
params.skip_layer_end},
1023+
params.skip_layer_end,
1024+
params.slg_uncond},
10171025
sd_apg_params_t{params.apg_eta,
10181026
params.apg_momentum,
10191027
params.apg_norm_threshold,
@@ -1086,7 +1094,8 @@ int main(int argc, const char* argv[]) {
10861094
params.skip_layers.size(),
10871095
params.slg_scale,
10881096
params.skip_layer_start,
1089-
params.skip_layer_end},
1097+
params.skip_layer_end,
1098+
params.slg_uncond},
10901099
sd_apg_params_t{params.apg_eta,
10911100
params.apg_momentum,
10921101
params.apg_norm_threshold,

‎stable-diffusion.cpp‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -800,7 +800,7 @@ class StableDiffusionGGML {
800800
const std::vector<float>& sigmas,
801801
int start_merge_step,
802802
SDCondition id_cond,
803-
sd_slg_params_t slg_params = {NULL, 0, 0, 0, 0},
803+
sd_slg_params_t slg_params = {NULL, 0, 0, 0, false},
804804
sd_apg_params_t apg_params = {1, 0, 0, 0},
805805
ggml_tensor* noise_mask = nullptr) {
806806
std::vector<int> skip_layers(slg_params.skip_layers, slg_params.skip_layers + slg_params.skip_layers_count);
@@ -827,7 +827,7 @@ class StableDiffusionGGML {
827827
struct ggml_tensor* noised_input = ggml_dup_tensor(work_ctx, noise);
828828

829829
bool has_unconditioned = cfg_scale != 1.0 && uncond.c_crossattn != NULL;
830-
bool has_skiplayer = slg_params.scale != 0.0 && skip_layers.size() > 0;
830+
bool has_skiplayer = (slg_params.scale != 0.0 || slg_params.slg_uncond) && skip_layers.size() > 0;
831831

832832
// denoise wrapper
833833
struct ggml_tensor* out_cond = ggml_dup_tensor(work_ctx, x);
@@ -839,7 +839,9 @@ class StableDiffusionGGML {
839839
}
840840
if (has_skiplayer) {
841841
if (sd_version_is_dit(version)) {
842-
out_skip = ggml_dup_tensor(work_ctx, x);
842+
if (slg_params.scale != 0.0) {
843+
out_skip = ggml_dup_tensor(work_ctx, x);
844+
}
843845
} else {
844846
has_skiplayer = false;
845847
LOG_WARN("SLG is incompatible with %s models", model_version_to_str[version]);
@@ -908,6 +910,8 @@ class StableDiffusionGGML {
908910
control_strength,
909911
&out_cond);
910912
}
913+
int step_count = sigmas.size();
914+
bool is_skiplayer_step = has_skiplayer && step > (int)(slg_params.skip_layer_start * step_count) && step < (int)(slg_params.skip_layer_end * step_count);
911915

912916
float* negative_data = NULL;
913917
if (has_unconditioned) {
@@ -916,24 +920,39 @@ class StableDiffusionGGML {
916920
control_net->compute(n_threads, noised_input, control_hint, timesteps, uncond.c_crossattn, uncond.c_vector);
917921
controls = control_net->controls;
918922
}
919-
diffusion_model->compute(n_threads,
920-
noised_input,
921-
timesteps,
922-
uncond.c_crossattn,
923-
uncond.c_concat,
924-
uncond.c_vector,
925-
guidance_tensor,
926-
-1,
927-
controls,
928-
control_strength,
929-
&out_uncond);
923+
if (is_skiplayer_step && slg_params.slg_uncond) {
924+
LOG_DEBUG("Skipping layers at uncond step %d\n", step);
925+
diffusion_model->compute(n_threads,
926+
noised_input,
927+
timesteps,
928+
uncond.c_crossattn,
929+
uncond.c_concat,
930+
uncond.c_vector,
931+
guidance_tensor,
932+
-1,
933+
controls,
934+
control_strength,
935+
&out_uncond,
936+
NULL,
937+
skip_layers);
938+
} else {
939+
diffusion_model->compute(n_threads,
940+
noised_input,
941+
timesteps,
942+
uncond.c_crossattn,
943+
uncond.c_concat,
944+
uncond.c_vector,
945+
guidance_tensor,
946+
-1,
947+
controls,
948+
control_strength,
949+
&out_uncond);
950+
}
930951
negative_data = (float*)out_uncond->data;
931952
}
932953

933-
int step_count = sigmas.size();
934-
bool is_skiplayer_step = has_skiplayer && step > (int)(slg_params.skip_layer_start * step_count) && step < (int)(slg_params.skip_layer_end * step_count);
935954
float* skip_layer_data = NULL;
936-
if (is_skiplayer_step) {
955+
if (is_skiplayer_step && slg_params.scale != 0.0) {
937956
LOG_DEBUG("Skipping layers at step %d\n", step);
938957
// skip layer (same as conditionned)
939958
diffusion_model->compute(n_threads,
@@ -1021,7 +1040,7 @@ class StableDiffusionGGML {
10211040
latent_result = positive_data[i] + (cfg_scale - 1) * delta;
10221041
}
10231042
}
1024-
if (is_skiplayer_step) {
1043+
if (is_skiplayer_step && slg_params.scale != 0.0) {
10251044
latent_result = latent_result + (positive_data[i] - skip_layer_data[i]) * slg_params.scale;
10261045
}
10271046
// v = latent_result, eps = latent_result

‎stable-diffusion.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ typedef struct {
140140
float scale;
141141
float skip_layer_start;
142142
float skip_layer_end;
143+
bool slg_uncond;
143144
} sd_slg_params_t;
144145

145146
typedef struct sd_ctx_t sd_ctx_t;

0 commit comments

Comments
 (0)