add backend support for changing the VAE tiling threshold

Author: wbruna

expose.h

@@ -162,7 +162,7 @@ struct sd_load_model_inputs
     const int threads = 0;
     const int quant = 0;
     const bool taesd = false;
-    const bool notile = false;
+    const int tiled_vae_threshold = 0;
     const char * t5xxl_filename = nullptr;
     const char * clipl_filename = nullptr;
     const char * clipg_filename = nullptr;

koboldcpp.py

@@ -267,7 +267,7 @@ class sd_load_model_inputs(ctypes.Structure):
                 ("threads", ctypes.c_int),
                 ("quant", ctypes.c_int),
                 ("taesd", ctypes.c_bool),
-                ("notile", ctypes.c_bool),
+                ("tiled_vae_threshold", ctypes.c_int),
                 ("t5xxl_filename", ctypes.c_char_p),
                 ("clipl_filename", ctypes.c_char_p),
                 ("clipg_filename", ctypes.c_char_p),
@@ -1534,7 +1534,7 @@ def sd_load_model(model_filename,vae_filename,lora_filename,t5xxl_filename,clipl
     inputs.threads = thds
     inputs.quant = quant
     inputs.taesd = True if args.sdvaeauto else False
-    inputs.notile = True if args.sdnotile else False
+    inputs.tiled_vae_threshold = -1 if args.sdnotile else 0
     inputs.vae_filename = vae_filename.encode("UTF-8")
     inputs.lora_filename = lora_filename.encode("UTF-8")
     inputs.lora_multiplier = args.sdloramult

otherarch/sdcpp/sdtype_adapter.cpp

@@ -118,7 +118,7 @@ static uint8_t * input_image_buffer = NULL;
 static uint8_t * input_mask_buffer = NULL;
 
 static std::string sdplatformenv, sddeviceenv, sdvulkandeviceenv;
-static bool notiling = false;
+static int cfg_tiled_vae_threshold = 0;
 static int cfg_square_limit = 0;
 static int cfg_side_limit = 0;
 static bool sd_is_quiet = false;
@@ -134,7 +134,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
     std::string t5xxl_filename = inputs.t5xxl_filename;
     std::string clipl_filename = inputs.clipl_filename;
     std::string clipg_filename = inputs.clipg_filename;
-    notiling = inputs.notile;
+    cfg_tiled_vae_threshold = inputs.tiled_vae_threshold;
     cfg_side_limit = inputs.side_limit;
     cfg_square_limit = inputs.square_limit;
     printf("\nImageGen Init - Load Model: %s\n",inputs.model_filename);
@@ -474,7 +474,18 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
         printf("\nKCPP SD: Requested dimensions %dx%d changed to %dx%d\n", inputs.width, inputs.height, sd_params->width, sd_params->height);
     }
 
-    bool dotile = (sd_params->width>768 || sd_params->height>768) && !notiling;
+    int tiled_vae_threshold;
+    if (cfg_tiled_vae_threshold == 0) {
+        tiled_vae_threshold = 768;
+    } else {
+        if (cfg_tiled_vae_threshold > 0) {
+            tiled_vae_threshold = cfg_tiled_vae_threshold;
+        } else {
+            tiled_vae_threshold = 8192; // effectively avoids tiling
+        }
+    }
+
+    bool dotile = (sd_params->width>tiled_vae_threshold || sd_params->height>tiled_vae_threshold);
     set_sd_vae_tiling(sd_ctx,dotile); //changes vae tiling, prevents memory related crash/oom
 
     if (sd_params->clip_skip <= 0) {