Added controlnet-canny edge support

Author: arisha07

README.md

@@ -9,7 +9,7 @@ This branch is currently under development. <br>Dedicated for GIMP 3, Python 3 a
 [2] Style-Transfer <br>
 [3] Inpainting <br>
 [4] Semantic-Segmentation <br>
-[5] Stable-Diffusion (Suppports - SD 1.4, SD 1.5 (landscape & portrait), SD 1.5 Inpainting, SD 1.5 Controlnet-Openpose) <br>
+[5] Stable-Diffusion (Suppports - SD 1.4, SD 1.5 (landscape & portrait), SD 1.5 Inpainting, SD 1.5 Controlnet-OpenPose, SD 1.5 Controlnet-CannyEdge) <br>
 
 # Objectives
 [1] Provides a set of OpenVino based plugins that add AI features to GIMP. <br>
@@ -102,6 +102,15 @@ Skip steps 1 and 2 if you already have Python3 and Git on Windows
 ![](gifs/controlnet-openpose.png)
 
 
+#### E. Stable-Diffusion-1.5 Controlnet-CannyEdge - Make sure to download and convert the model during install process. 
+1. Open an image that you want to use for generating the new image. <br>
+2. Select Stable Diffusion from the drop down list in layers -> OpenVINO-AI-Plugins <br>
+3. Choose the controlnet_canny model and device from the drop down list.<br>
+4. Make sure to select -- "Use Initial Image" option from the GUI. If not selected then it will fail. 
+5. Follow steps 4,5 from section A. <br>
+5. Click on “Run Inference”. Wait for the total inference steps to get completed. <br>
+
+
 *If create gif option is selected, please note that performance will reduce. The generated gif is located in below path. You can play it in GIMP by going to Filters -> Animations -> Playback* <br>
 ```C:\Users\<user_name>\openvino-ai-plugins-gimp\gif\stable_diffusion.gif``` <br>
 

choose_sd_model.py

@@ -25,26 +25,28 @@
     print("5 - Landscape_768x512 (768x512 output image),  will take time since model is large ")
     print("6 - SD-1.5 Inapinting model (512x512 output image) ")
     print("7 - SD-1.5 Controlnet-Openpose model (512x512 output image) ")
-    print("8 - ALL the above SD-1.5 models ")
-    print("9 - Only Square, Landscape, Portrait")
-    print("10 - Skip All SD-1.5 Model setup ") 
+    print("8 - SD-1.5 Controlnet-CannyEdge model (512x512 output image) ")
+    print("9 - ALL the above SD-1.5 models ")
+    print("10 - Only Square, Landscape, Portrait")
+    print("11 - Skip All SD-1.5 Model setup ") 
     choice = input("Enter the Number for the model you want to download & convert: ")
 
     # setup all the SD1.5 models
-    if choice=="8":
+    if choice=="9":
         for i in range(1,7):
 
             subprocess.call([sd_python_path, chose_model, str(i)])
 
         subprocess.call([sd_python_path, chose_model_controlnet, "7"])    
+        subprocess.call([sd_python_path, chose_model_controlnet, "8"]) 
         break
     # setup only square, landscape and Portrait
-    elif choice=="9":
+    elif choice=="10":
         for i in range(1,4):
 
             subprocess.call([sd_python_path, chose_model, str(i)])
-        break
-    elif choice=="10":
+        
+    elif choice=="11":
         print("Exiting SD-1.5 Model setup.........")
         break
     elif choice in ["1","2","3","4","5","6"]:
@@ -53,8 +55,11 @@
     elif choice == "7":
 
         subprocess.call([sd_python_path, chose_model_controlnet, choice]) 
+    elif choice == "8":
+  
+        subprocess.call([sd_python_path, chose_model_controlnet, choice])        
 
-        #break
+       
 
     else:
         print("Wrong option selected.")

controlnet_model_conversion.py

@@ -55,10 +55,7 @@
     ht = 512
     weight_path = os.path.join(SD_path, "controlnet-openpose")
 
-    
-    
-    
-    
+       
     controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_openpose", torch_dtype=torch.float32)
 
     pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet)
@@ -103,33 +100,56 @@
 
         print('OpenPose successfully converted to IR')
     else:
-        print(f"OpenPose will be loaded from {OPENPOSE_OV_PATH}")    
+        print(f"OpenPose will be loaded from {OPENPOSE_OV_PATH}") 
+
+    if not os.path.isdir(weight_path):
+            os.makedirs(weight_path)
+
+    print("weight path is :", weight_path)
+
+
+    CONTROLNET_ONNX_PATH = Path(weight_path) / 'controlnet-pose.onnx'
+    CONTROLNET_OV_PATH = Path(weight_path) / 'controlnet-pose.xml'        
 
 
-#elif choice == "2":
-#    wt = 640
-#    ht = 360
-#    weight_path = os.path.join(SD_path, "landscape")
-#    print("============SD-1.5 landscape Model setup============")
-#elif choice == "3":
-#    wt = 360
-#    ht = 640
-#    weight_path = os.path.join(SD_path, "portrait")
-#    print("============SD-1.5 portrait Model setup============")
+if choice == "8":
+    print("============SD-1.5 Controlnet-Canny Model setup============----")
+    wt = 512
+    ht = 512
+    weight_path = os.path.join(SD_path, "controlnet-canny")
+    
+       
+    controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny", torch_dtype=torch.float32)
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        "runwayml/stable-diffusion-v1-5", controlnet=controlnet
+    )
+    
+    
+    if not os.path.isdir(os.path.join(SD_path,"UniPCMultistepScheduler_config")):
+        print("Saving scheduler config") # TODO: change
+        pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet)
+        scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+        scheduler.save_config(os.path.join(SD_path,"UniPCMultistepScheduler_config"))               
+    
+    if not os.path.isdir(weight_path):
+            os.makedirs(weight_path)
+
+    print("weight path is :", weight_path)
+
+
+    CONTROLNET_ONNX_PATH = Path(weight_path) / 'controlnet-canny.onnx'
+    CONTROLNET_OV_PATH = Path(weight_path) / 'controlnet-canny.xml'     
 
 else:
     print("============Select option 7============----")
+    
+  
 
 
 
-if not os.path.isdir(weight_path):
-        os.makedirs(weight_path)
 
-print("weight path is :", weight_path)
 
 
-CONTROLNET_ONNX_PATH = Path(weight_path) / 'controlnet-pose.onnx'
-CONTROLNET_OV_PATH = Path(weight_path) / 'controlnet-pose.xml' 
 UNET_ONNX_PATH = Path(weight_path) / 'unet_controlnet' / 'unet_controlnet.onnx'
 UNET_OV_PATH = Path(weight_path) / 'unet_controlnet.xml'
 

gimpopenvino/plugins/stable-diffusion-ov/stable-diffusion-ov.py

@@ -97,7 +97,7 @@ class SDDialogResponse(IntEnum):
 def list_models(weight_path, SD):
     model_list = []
     flag = False
-    flag_openpose = False
+    flag_controlnet = False
     if SD == "SD_1.4":
         dir_path = os.path.join(weight_path, "stable-diffusion-ov/stable-diffusion-1.4")
         flag = True
@@ -107,17 +107,23 @@ def list_models(weight_path, SD):
         flag = True
     if SD == "controlnet_openpose":
         dir_path = os.path.join(weight_path, "stable-diffusion-ov/controlnet-openpose")
-        flag_openpose = True
-        print("flag_openpose", flag_openpose)
+        flag_controlnet = True
+        print("flag_controlnet", flag_controlnet)
 
-    if flag_openpose:
+    if SD == "controlnet_canny":
+        dir_path = os.path.join(weight_path, "stable-diffusion-ov/controlnet-canny")
+        flag_controlnet = True
+        print("flag_controlnet", flag_controlnet)
+        
+    if flag_controlnet:
         text = Path(dir_path) / 'text_encoder.xml'
         unet = Path(dir_path) / 'unet_controlnet.xml'
         vae = Path(dir_path) / 'vae_decoder.xml'
 
         if os.path.isfile(text) and os.path.isfile(unet) and os.path.isfile(vae):
                 print("ALL OKAY !?")
                 model_list.append(SD)
+        flag_controlnet = False
 
         return model_list   
 
@@ -418,7 +424,7 @@ def run(procedure, run_mode, image, n_drawables, layer, args, data):
         if n_layers == 2:
             model_list = list_models(config_path_output["weight_path"],"SD_1.5_Inpainting")
         else:
-            model_list = list_models(config_path_output["weight_path"],"SD_1.4") + list_models(config_path_output["weight_path"],"SD_1.5") + list_models(config_path_output["weight_path"],"controlnet_openpose")
+            model_list = list_models(config_path_output["weight_path"],"SD_1.4") + list_models(config_path_output["weight_path"],"SD_1.5") + list_models(config_path_output["weight_path"],"controlnet_openpose") + list_models(config_path_output["weight_path"],"controlnet_canny")
 
         model_name_enum = DeviceEnum(model_list)