refactor: unify image/text functions into generate_image, update submodules

Author: william-murray1204

README.md

@@ -265,7 +265,7 @@ stable_diffusion = StableDiffusion(
       model_path="../models/v1-5-pruned-emaonly.safetensors",
       # wtype="default", # Weight type (e.g. "q8_0", "f16", etc) (The "default" setting is automatically applied and determines the weight type of a model file)
 )
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       prompt="a lovely cat",
       width=512, # Must be a multiple of 64
       height=512, # Must be a multiple of 64
@@ -291,7 +291,7 @@ stable_diffusion = StableDiffusion(
       model_path="../models/v1-5-pruned-emaonly.safetensors",
       lora_model_dir="../models/", # This should point to folder where LoRA weights are stored (not an individual file)
 )
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       prompt="a lovely cat<lora:marblesh:1>",
 )
 ```
@@ -319,9 +319,9 @@ stable_diffusion = StableDiffusion(
     clip_l_path="../models/clip_l.safetensors",
     t5xxl_path="../models/t5xxl_fp16.safetensors",
     vae_path="../models/ae.safetensors",
-    vae_decode_only=True, # Can be True if we dont use img_to_img
+    vae_decode_only=True, # Can be True if not generating image to image
 )
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       prompt="a lovely cat holding a sign says 'flux.cpp'",
       sample_steps=4,
       cfg_scale=1.0, # a cfg_scale of 1 is recommended for FLUX
@@ -357,7 +357,7 @@ stable_diffusion = StableDiffusion(
     vae_path="../models/ae.safetensors",
     vae_decode_only=False, # Must be False for FLUX Kontext
 )
-output = stable_diffusion.edit(
+output = stable_diffusion.generate_image(
       prompt="make the cat blue",
       images=["input.png"],
       cfg_scale=1.0, # a cfg_scale of 1 is recommended for FLUX
@@ -380,9 +380,9 @@ stable_diffusion = StableDiffusion(
     diffusion_model_path="../models/chroma-unlocked-v40-Q4_0.gguf", # In place of model_path
     t5xxl_path="../models/t5xxl_fp16.safetensors",
     vae_path="../models/ae.safetensors",
-    vae_decode_only=True, # Can be True if we dont use img_to_img
+    vae_decode_only=True, # Can be True if we are not generating image to image
 )
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       prompt="a lovely cat holding a sign says 'chroma.cpp'",
       sample_steps=4,
       cfg_scale=4.0, # a cfg_scale of 4 is recommended for Chroma
@@ -410,7 +410,7 @@ stable_diffusion = StableDiffusion(
     clip_g_path="../models/clip_g.safetensors",
     t5xxl_path="../models/t5xxl_fp16.safetensors",
 )
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       prompt="a lovely cat holding a sign says 'Stable diffusion 3.5 Large'",
       height=1024,
       width=1024,
@@ -432,9 +432,9 @@ INPUT_IMAGE = "../input.png"
 
 stable_diffusion = StableDiffusion(model_path="../models/v1-5-pruned-emaonly.safetensors")
 
-output = stable_diffusion.img_to_img(
+output = stable_diffusion.generate_image(
       prompt="blue eyes",
-      image=INPUT_IMAGE, # Note: The input image will be automatically resized to the match the width and height arguments (default: 512x512)
+      init_image=INPUT_IMAGE, # Note: The input image will be automatically resized to the match the width and height arguments (default: 512x512)
       strength=0.4,
 )
 ```
@@ -447,9 +447,9 @@ from stable_diffusion_cpp import StableDiffusion
 # Note: Inpainting with a base model gives poor results. A model fine-tuned for inpainting is recommended.
 stable_diffusion = StableDiffusion(model_path="../models/v1-5-pruned-emaonly.safetensors")
 
-output = stable_diffusion.img_to_img(
+output = stable_diffusion.generate_image(
       prompt="blue eyes",
-      image="../input.png",
+      init_image="../input.png",
       mask_image="../mask.png", # A grayscale image where 0 is masked and 255 is unmasked
       strength=0.4,
 )
@@ -478,7 +478,7 @@ stable_diffusion = StableDiffusion(
       # keep_vae_on_cpu=True,  # If on low memory GPUs (<= 8GB), setting this to True is recommended to get artifact free images
 )
 
-output = stable_diffusion.txt_to_img(
+output = stable_diffusion.generate_image(
       cfg_scale=5.0, # a cfg_scale of 5.0 is recommended for PhotoMaker
       height=1024,
       width=1024,
@@ -553,6 +553,15 @@ c_image = sd_cpp.sd_image_t(
             ctypes.POINTER(ctypes.c_uint8),
       ),
 ) # Create a new C sd_image_t
+
+# Convert a model from safetensors to gguf format
+sd_cpp.convert(
+      "../models/v1-5-pruned-emaonly.safetensors".encode("utf-8"), # input_path
+      "".encode("utf-8"), # vae_path
+      "../models/v1-5-pruned-emaonly.gguf".encode("utf-8"), # output_path
+      sd_cpp.GGMLType.SD_TYPE_Q8_0, # output_type
+      "".encode("utf-8"), # tensor_type_rules
+)
 ```
 
 ## Development

stable_diffusion_cpp/__init__.py

@@ -4,4 +4,4 @@
 
 # isort: on
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

stable_diffusion_cpp/_internals.py

@@ -1,8 +1,8 @@
 import os
+import ctypes
 from contextlib import ExitStack
 
 import stable_diffusion_cpp.stable_diffusion_cpp as sd_cpp
-
 from ._utils import suppress_stdout_stderr
 
 # ============================================
@@ -27,7 +27,7 @@ def __init__(
         taesd_path: str,
         control_net_path: str,
         lora_model_dir: str,
-        embed_dir: str,
+        embedding_dir: str,
         stacked_id_embed_dir: str,
         vae_decode_only: bool,
         vae_tiling: bool,
@@ -36,43 +36,43 @@ def __init__(
         rng_type: int,
         schedule: int,
         keep_clip_on_cpu: bool,
-        keep_control_net_cpu: bool,
+        keep_control_net_on_cpu: bool,
         keep_vae_on_cpu: bool,
         diffusion_flash_attn: bool,
         chroma_use_dit_mask: bool,
         chroma_use_t5_mask: bool,
         chroma_t5_mask_pad: int,
         verbose: bool,
     ):
-        self.model_path = model_path
-        self.clip_l_path = clip_l_path
-        self.clip_g_path = clip_g_path
-        self.t5xxl_path = t5xxl_path
-        self.diffusion_model_path = diffusion_model_path
-        self.vae_path = vae_path
-        self.taesd_path = taesd_path
-        self.control_net_path = control_net_path
-        self.lora_model_dir = lora_model_dir
-        self.embed_dir = embed_dir
-        self.stacked_id_embed_dir = stacked_id_embed_dir
-        self.vae_decode_only = vae_decode_only
-        self.vae_tiling = vae_tiling
-        self.n_threads = n_threads
-        self.wtype = wtype
-        self.rng_type = rng_type
-        self.schedule = schedule
-        self.keep_clip_on_cpu = keep_clip_on_cpu
-        self.keep_control_net_cpu = keep_control_net_cpu
-        self.keep_vae_on_cpu = keep_vae_on_cpu
-        self.diffusion_flash_attn = diffusion_flash_attn
-        self.chroma_use_dit_mask = chroma_use_dit_mask
-        self.chroma_use_t5_mask = chroma_use_t5_mask
-        self.chroma_t5_mask_pad = chroma_t5_mask_pad
-        self.verbose = verbose
-
         self._exit_stack = ExitStack()
-
         self.model = None
+        self.params = sd_cpp.sd_ctx_params_t(
+            model_path=model_path.encode("utf-8"),
+            clip_l_path=clip_l_path.encode("utf-8"),
+            clip_g_path=clip_g_path.encode("utf-8"),
+            t5xxl_path=t5xxl_path.encode("utf-8"),
+            diffusion_model_path=diffusion_model_path.encode("utf-8"),
+            vae_path=vae_path.encode("utf-8"),
+            taesd_path=taesd_path.encode("utf-8"),
+            control_net_path=control_net_path.encode("utf-8"),
+            lora_model_dir=lora_model_dir.encode("utf-8"),
+            embedding_dir=embedding_dir.encode("utf-8"),
+            stacked_id_embed_dir=stacked_id_embed_dir.encode("utf-8"),
+            vae_decode_only=vae_decode_only,
+            vae_tiling=vae_tiling,
+            free_params_immediately=False,  # Don't unload model
+            n_threads=n_threads,
+            wtype=wtype,
+            rng_type=rng_type,
+            schedule=schedule,
+            keep_clip_on_cpu=keep_clip_on_cpu,
+            keep_control_net_on_cpu=keep_control_net_on_cpu,
+            keep_vae_on_cpu=keep_vae_on_cpu,
+            diffusion_flash_attn=diffusion_flash_attn,
+            chroma_use_dit_mask=chroma_use_dit_mask,
+            chroma_use_t5_mask=chroma_use_t5_mask,
+            chroma_t5_mask_pad=chroma_t5_mask_pad,
+        )
 
         # Load the free_sd_ctx function
         self._free_sd_ctx = sd_cpp._lib.free_sd_ctx
@@ -88,34 +88,8 @@ def __init__(
 
         if model_path or diffusion_model_path:
             with suppress_stdout_stderr(disable=verbose):
-                # Load the Stable Diffusion model ctx
-                self.model = sd_cpp.new_sd_ctx(
-                    self.model_path.encode("utf-8"),
-                    self.clip_l_path.encode("utf-8"),
-                    self.clip_g_path.encode("utf-8"),
-                    self.t5xxl_path.encode("utf-8"),
-                    self.diffusion_model_path.encode("utf-8"),
-                    self.vae_path.encode("utf-8"),
-                    self.taesd_path.encode("utf-8"),
-                    self.control_net_path.encode("utf-8"),
-                    self.lora_model_dir.encode("utf-8"),
-                    self.embed_dir.encode("utf-8"),
-                    self.stacked_id_embed_dir.encode("utf-8"),
-                    self.vae_decode_only,
-                    self.vae_tiling,
-                    False,  # Free params immediately (unload model)
-                    self.n_threads,
-                    self.wtype,
-                    self.rng_type,
-                    self.schedule,
-                    self.keep_clip_on_cpu,
-                    self.keep_control_net_cpu,
-                    self.keep_vae_on_cpu,
-                    self.diffusion_flash_attn,
-                    self.chroma_use_dit_mask,
-                    self.chroma_use_t5_mask,
-                    self.chroma_t5_mask_pad,
-                )
+                # Call function with a pointer to params
+                self.model = sd_cpp.new_sd_ctx(ctypes.byref(self.params))
 
             # Check if the model was loaded successfully
             if self.model is None: