Skip to content

Commit 56d75e1

Browse files
feat(backend): use VAE mean encoding for Kontext reference images
Use distribution mean without sampling noise for more stable and consistent reference image encoding, matching ComfyUI implementation
1 parent df77a12 commit 56d75e1

File tree

1 file changed

+9
-2
lines changed

1 file changed

+9
-2
lines changed

invokeai/backend/flux/extensions/kontext_extension.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
from PIL import Image
55

66
from invokeai.app.invocations.fields import FluxKontextConditioningField
7-
from invokeai.app.invocations.flux_vae_encode import FluxVaeEncodeInvocation
87
from invokeai.app.invocations.model import VAEField
98
from invokeai.app.services.shared.invocation_context import InvocationContext
9+
from invokeai.backend.flux.modules.autoencoder import AutoEncoder
1010
from invokeai.backend.flux.sampling_utils import pack
1111
from invokeai.backend.flux.util import PREFERED_KONTEXT_RESOLUTIONS
12+
from invokeai.backend.util.devices import TorchDevice
1213

1314

1415
def generate_img_ids_with_offset(
@@ -149,7 +150,13 @@ def _prepare_kontext(self) -> tuple[torch.Tensor, torch.Tensor]:
149150
image_tensor = image_tensor.to(self._device)
150151

151152
# Continue with VAE encoding
152-
kontext_latents_unpacked = FluxVaeEncodeInvocation.vae_encode(vae_info=vae_info, image_tensor=image_tensor)
153+
# Don't sample from the distribution for reference images - use the mean (matching ComfyUI)
154+
with vae_info as vae:
155+
assert isinstance(vae, AutoEncoder)
156+
vae_dtype = next(iter(vae.parameters())).dtype
157+
image_tensor = image_tensor.to(device=TorchDevice.choose_torch_device(), dtype=vae_dtype)
158+
# Use sample=False to get the distribution mean without noise
159+
kontext_latents_unpacked = vae.encode(image_tensor, sample=False)
153160

154161
# Extract tensor dimensions
155162
batch_size, _, latent_height, latent_width = kontext_latents_unpacked.shape

0 commit comments

Comments
 (0)