FEATURE: add OpenAI image generation and editing capabilities (#1293)

This commit enhances the AI image generation functionality by adding support for:

1. OpenAI's GPT-based image generation model (gpt-image-1)
2. Image editing capabilities through the OpenAI API
3. A new "Designer" persona specialized in image generation and editing
4. Two new AI tools: CreateImage and EditImage

Technical changes include:
- Renaming `ai_openai_dall_e_3_url` to `ai_openai_image_generation_url` with a migration
- Adding `ai_openai_image_edit_url` setting for the image edit API endpoint
- Refactoring image generation code to handle both DALL-E and the newer GPT models
- Supporting multipart/form-data for image editing requests

* wild guess but maybe quantization is breaking the test sometimes

this increases distance

* Update lib/personas/designer.rb

Co-authored-by: Alan Guo Xiang Tan <[email protected]>

* simplify and de-flake code

* fix, in chat we need enough context so we know exactly what uploads a user uploaded.

* Update lib/personas/tools/edit_image.rb

Co-authored-by: Alan Guo Xiang Tan <[email protected]>

* cleanup downloaded files right away

* fix implementation

---------

Co-authored-by: Alan Guo Xiang Tan <[email protected]>

Author: SamSaffron

config/locales/server.en.yml

@@ -55,7 +55,9 @@ en:
     ai_nsfw_flag_threshold_sexy: "Threshold for an image classified as sexy to be considered NSFW."
     ai_nsfw_models: "Models to use for NSFW inference."
 
-    ai_openai_api_key: "API key for OpenAI API. ONLY used for Dall-E. For GPT use the LLM config tab"
+    ai_openai_api_key: "API key for OpenAI API. ONLY used for Image creation and edits. For GPT use the LLM config tab"
+    ai_openai_image_generation_url: "URL for OpenAI image generation API"
+    ai_openai_image_edit_url: "URL for OpenAI image edit API"
 
     ai_helper_enabled: "Enable the AI helper."
     composer_ai_helper_allowed_groups: "Users on these groups will see the AI helper button in the composer."
@@ -290,6 +292,9 @@ en:
         artist:
           name: Artist
           description: "AI Bot specialized in generating images"
+        designer:
+          name: Designer
+          description: "AI Bot specialized in generating and editing images"
         sql_helper:
           name: SQL Helper
           description: "AI Bot specialized in helping craft SQL queries on this Discourse instance"
@@ -377,6 +382,8 @@ en:
         dall_e: "Generate image"
         search_meta_discourse: "Search Meta Discourse"
         javascript_evaluator: "Evaluate JavaScript"
+        create_image: "Creating image"
+        edit_image: "Editing image"
       tool_help:
         read_artifact: "Read a web artifact using the AI Bot"
         update_artifact: "Update a web artifact using the AI Bot"
@@ -393,6 +400,8 @@ en:
         time: "Find time in various time zones"
         summary: "Summarize a topic"
         image: "Generate image using Stable Diffusion"
+        create_image: "Generate image using Open AI GPT image model"
+        edit_image: "Edit image using Open AI GPT image model"
         google: "Search Google for a query"
         read: "Read public topic on the forum"
         setting_context: "Look up site setting context"
@@ -415,6 +424,8 @@ en:
         time: "Time in %{timezone} is %{time}"
         summarize: "Summarized <a href='%{url}'>%{title}</a>"
         dall_e: "%{prompt}"
+        create_image: "%{prompt}"
+        edit_image: "%{prompt}"
         image: "%{prompt}"
         categories:
           one: "Found %{count} category"

config/settings.yml

@@ -26,7 +26,8 @@ discourse_ai:
     default: 60
     hidden: true
 
-  ai_openai_dall_e_3_url: "https://api.openai.com/v1/images/generations"
+  ai_openai_image_generation_url: "https://api.openai.com/v1/images/generations"
+  ai_openai_image_edit_url: "https://api.openai.com/v1/images/edits"
   ai_openai_embeddings_url:
     hidden: true
     default: "https://api.openai.com/v1/embeddings"

db/migrate/20250429060311_move_dall_e_url.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+class MoveDallEUrl < ActiveRecord::Migration[7.2]
+  def up
+    execute <<~SQL
+      UPDATE site_settings
+      SET name = 'ai_openai_image_generation_url'
+      WHERE name = 'ai_openai_dall_e_3_url'
+      AND NOT EXISTS (
+        SELECT 1
+        FROM site_settings
+        WHERE name = 'ai_openai_image_generation_url')
+    SQL
+
+    execute <<~SQL
+      DELETE FROM site_settings
+      WHERE name = 'ai_openai_dall_e_3_url'
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

lib/ai_helper/painter.rb

@@ -21,17 +21,18 @@ def commission_thumbnails(input, user)
 
           base64_to_image(artifacts, user.id)
         elsif model == "dall_e_3"
-          api_key = SiteSetting.ai_openai_api_key
-          api_url = SiteSetting.ai_openai_dall_e_3_url
-
-          artifacts =
-            DiscourseAi::Inference::OpenAiImageGenerator
-              .perform!(input, api_key: api_key, api_url: api_url)
-              .dig(:data)
-              .to_a
-              .map { |art| art[:b64_json] }
-
-          base64_to_image(artifacts, user.id)
+          attribution =
+            I18n.t(
+              "discourse_ai.ai_helper.painter.attribution.#{SiteSetting.ai_helper_illustrate_post_model}",
+            )
+          results =
+            DiscourseAi::Inference::OpenAiImageGenerator.create_uploads!(
+              input,
+              model: "dall-e-3",
+              user_id: user.id,
+              title: attribution,
+            )
+          results.map { |result| UploadSerializer.new(result[:upload], root: false) }
         end
       end
 

lib/completions/prompt_messages_builder.rb

@@ -71,6 +71,11 @@ def self.messages_from_chat(
             thread_title = m.thread&.title if include_thread_titles && m.thread_id
             mapped_message = "(#{thread_title})\n#{m.message}" if thread_title
 
+            if m.uploads.present?
+              mapped_message =
+                "#{mapped_message} -- uploaded(#{m.uploads.map(&:short_url).join(", ")})"
+            end
+
             builder.push(
               type: :user,
               content: mapped_message,