Guidance on denoised predictions

Author: hlky

src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py

@@ -1088,8 +1088,10 @@ def __call__(
 
                 # perform guidance
                 if self.do_classifier_free_guidance:
-                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                     if adaptive_projected_guidance:
+                        sigma = self.scheduler.sigmas[self.scheduler.step_index]
+                        noise_pred = latents - sigma * noise_pred
+                        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                         noise_pred = normalized_guidance(
                             noise_pred_text,
                             noise_pred_uncond,
@@ -1098,7 +1100,9 @@ def __call__(
                             eta,
                             adaptive_projected_guidance_rescale_factor,
                         )
+                        noise_pred = (latents - noise_pred) / sigma
                     else:
+                        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                         noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
 
                 if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:

src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py

@@ -1281,8 +1281,10 @@ def __call__(
 
                 # perform guidance
                 if self.do_classifier_free_guidance:
-                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                     if adaptive_projected_guidance:
+                        sigma = self.scheduler.sigmas[self.scheduler.step_index]
+                        noise_pred = latents - sigma * noise_pred
+                        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                         noise_pred = normalized_guidance(
                             noise_pred_text,
                             noise_pred_uncond,
@@ -1291,7 +1293,9 @@ def __call__(
                             eta,
                             adaptive_projected_guidance_rescale_factor,
                         )
+                        noise_pred = (latents - noise_pred) / sigma
                     else:
+                        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
                         noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
 
                 if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: