implement new designer persona and tools / spec

Author: SamSaffron

config/locales/server.en.yml

@@ -292,6 +292,9 @@ en:
         artist:
           name: Artist
           description: "AI Bot specialized in generating images"
+        designer:
+          name: Designer
+          description: "AI Bot specialized in generating and editing images"
         sql_helper:
           name: SQL Helper
           description: "AI Bot specialized in helping craft SQL queries on this Discourse instance"
@@ -379,6 +382,8 @@ en:
         dall_e: "Generate image"
         search_meta_discourse: "Search Meta Discourse"
         javascript_evaluator: "Evaluate JavaScript"
+        create_image: "Creating image"
+        edit_image: "Editing image"
       tool_help:
         read_artifact: "Read a web artifact using the AI Bot"
         update_artifact: "Update a web artifact using the AI Bot"
@@ -395,6 +400,8 @@ en:
         time: "Find time in various time zones"
         summary: "Summarize a topic"
         image: "Generate image using Stable Diffusion"
+        create_image: "Generate image using Open AI GPT image model"
+        edit_image: "Edit image using Open AI GPT image model"
         google: "Search Google for a query"
         read: "Read public topic on the forum"
         setting_context: "Look up site setting context"
@@ -417,6 +424,8 @@ en:
         time: "Time in %{timezone} is %{time}"
         summarize: "Summarized <a href='%{url}'>%{title}</a>"
         dall_e: "%{prompt}"
+        create_image: "%{prompt}"
+        edit_image: "%{prompt}"
         image: "%{prompt}"
         categories:
           one: "Found %{count} category"

lib/inference/open_ai_image_generator.rb

@@ -69,7 +69,6 @@ def self.create_edited_upload!(
             quality: quality,
           )
 
-        # Create uploads in the main thread
         create_uploads_from_responses([api_response], user_id, for_private_message).first
       end
 
@@ -211,7 +210,7 @@ def self.edit_images(
           attempts += 1
           sleep 2
           retry if attempts < 3
-          if Rails.env.development?
+          if Rails.env.development? || Rails.env.test?
             puts "Error editing image(s) with prompt: #{prompt} #{e}"
             p e
           end
@@ -326,6 +325,7 @@ def self.perform_edit_api_call!(
         # Add model
         body << "--#{boundary}\r\n"
         body << "Content-Disposition: form-data; name=\"model\"\r\n\r\n"
+
         body << "#{model}\r\n"
 
         # Add images

lib/personas/designer.rb

@@ -0,0 +1,28 @@
+#frozen_string_literal: true
+
+module DiscourseAi
+  module Personas
+    class Designer < Persona
+      def tools
+        [Tools::CreateImage, Tools::EditImage]
+      end
+
+      def required_tools
+        [Tools::CreateImage, Tools::EditImage]
+      end
+
+      def system_prompt
+        <<~PROMPT
+            You are designerbot and you are here to help people generate and edit images.
+
+            - A good prompt needs to be detailed and specific.
+            - You can specify subject, medium (e.g. oil on canvas), artist (person who drew it or photographed it)
+            - You can specify details about lighting or time of day.
+            - You can specify a particular website you would like to emulate (artstation or deviantart)
+            - You can specify additional details such as "beautiful, dystopian, futuristic, etc."
+            - Be extremely detailed with image prompts
+          PROMPT
+      end
+    end
+  end
+end

lib/personas/persona.rb

@@ -46,6 +46,7 @@ def system_personas
             WebArtifactCreator => -10,
             Summarizer => -11,
             ShortSummarizer => -12,
+            Designer => -13,
           }
         end
 
@@ -111,7 +112,12 @@ def all_available_tools
           tools << Tools::ListTags if SiteSetting.tagging_enabled
           tools << Tools::Image if SiteSetting.ai_stability_api_key.present?
 
-          tools << Tools::DallE if SiteSetting.ai_openai_api_key.present?
+          if SiteSetting.ai_openai_api_key.present?
+            tools << Tools::DallE
+            tools << Tools::CreateImage
+            tools << Tools::EditImage
+          end
+
           if SiteSetting.ai_google_custom_search_api_key.present? &&
                SiteSetting.ai_google_custom_search_cx.present?
             tools << Tools::Google

lib/personas/tools/create_image.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Personas
+    module Tools
+      class CreateImage < Tool
+        def self.signature
+          {
+            name: name,
+            description: "Renders images from supplied descriptions",
+            parameters: [
+              {
+                name: "prompts",
+                description:
+                  "The prompts used to generate or create or draw the image (5000 chars or less, be creative) up to 4 prompts, usually only supply a single prompt",
+                type: "array",
+                item_type: "string",
+                required: true,
+              },
+            ],
+          }
+        end
+
+        def self.name
+          "create_image"
+        end
+
+        def prompts
+          parameters[:prompts]
+        end
+
+        def chain_next_response?
+          false
+        end
+
+        def invoke
+          # max 4 prompts
+          max_prompts = prompts.take(4)
+          progress = prompts.first
+
+          yield(progress)
+
+          results = nil
+
+          results =
+            DiscourseAi::Inference::OpenAiImageGenerator.create_uploads!(
+              max_prompts,
+              model: "gpt-image-1",
+              user_id: bot_user.id,
+            )
+
+          if results.blank?
+            return { prompts: max_prompts, error: "Something went wrong, could not generate image" }
+          end
+
+          self.custom_raw = <<~RAW
+
+            [grid]
+            #{
+            results
+              .map { |item| "![#{item[:prompt].gsub(/\|\'\"/, "")}](#{item[:upload].short_url})" }
+              .join(" ")
+          }
+            [/grid]
+          RAW
+
+          {
+            prompts: results.map { |item| { prompt: item[:prompt], url: item[:upload].short_url } },
+          }
+        end
+
+        protected
+
+        def description_args
+          { prompt: prompts.first }
+        end
+      end
+    end
+  end
+end

lib/personas/tools/edit_image.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Personas
+    module Tools
+      class EditImage < Tool
+        def self.signature
+          {
+            name: name,
+            description: "Renders images from supplied descriptions",
+            parameters: [
+              {
+                name: "prompt",
+                description:
+                  "instructions for the image to be edited (5000 chars or less, be creative)",
+                type: "string",
+                required: true,
+              },
+              {
+                name: "image_urls",
+                description:
+                  "The images to provides as context for the edit (minimum 1, maximum 10), use the short url eg: upload://qUm0DGR49PAZshIi7HxMd3cAlzn.png",
+                type: "array",
+                item_type: "string",
+                required: true,
+              },
+            ],
+          }
+        end
+
+        def self.name
+          "edit_image"
+        end
+
+        def prompt
+          parameters[:prompt]
+        end
+
+        def chain_next_response?
+          false
+        end
+
+        def image_urls
+          parameters[:image_urls]
+        end
+
+        def invoke
+          yield(prompt)
+
+          return { prompt: prompt, error: "No valid images provided" } if image_urls.blank?
+
+          uploads =
+            image_urls
+              .map do |url|
+                sha1 = Upload.sha1_from_short_url(url)
+                Upload.find_by(sha1: sha1)
+              end
+              .compact
+              .take(10)
+
+          return { prompt: prompt, error: "No valid images provided" } if uploads.blank?
+
+          result =
+            DiscourseAi::Inference::OpenAiImageGenerator.create_edited_upload!(
+              uploads,
+              prompt,
+              user_id: bot_user.id,
+            )
+
+          if result.blank?
+            return { prompt: prompt, error: "Something went wrong, could not generate image" }
+          end
+
+          self.custom_raw = "![#{result[:prompt].gsub(/\|\'\"/, "")}](#{result[:upload].short_url})"
+
+          { prompt: result[:prompt], url: result[:upload].short_url }
+        end
+
+        protected
+
+        def description_args
+          { prompt: prompt }
+        end
+      end
+    end
+  end
+end