FEATURE: first class support for OpenRouter

This new implementation supports picking quantization and provider pref

Also:

- Improve logging for summary generation
- Improve error message when contacting LLMs fails

Author: SamSaffron

app/models/ai_api_audit_log.rb

@@ -15,6 +15,7 @@ module Provider
     Ollama = 7
     SambaNova = 8
     Mistral = 9
+    OpenRouter = 10
   end
 
   def next_log_id

app/models/llm_model.rb

@@ -47,6 +47,11 @@ def self.provider_params
         disable_system_prompt: :checkbox,
         enable_native_tool: :checkbox,
       },
+      open_router: {
+        disable_native_tools: :checkbox,
+        provider_order: :text,
+        provider_quantizations: :text,
+      },
     }
   end
 

config/locales/client.en.yml

@@ -351,6 +351,7 @@ en:
           CDCK: "CDCK"
           samba_nova: "SambaNova"
           mistral: "Mistral"
+          open_router: "OpenRouter"
           fake: "Custom"
 
         provider_fields:
@@ -360,6 +361,8 @@ en:
           disable_system_prompt: "Disable system message in prompts"
           enable_native_tool: "Enable native tool support"
           disable_native_tools: "Disable native tool support (use XML based tools)"
+          provider_order: "Provider order (comma delimited list)"
+          provider_quantizations: "Order of provider quantizations (comma delimited list eg: fp16,fp8)"
 
       related_topics:
         title: "Related topics"

config/locales/server.en.yml

@@ -252,6 +252,7 @@ en:
         failed_to_share: "Failed to share the conversation"
         conversation_deleted: "Conversation share deleted successfully"
     ai_bot:
+      reply_error: "Sorry, it looks like our system encountered an unexpected issue while trying to reply.\n\n[details='Error details']\n%{details}\n[/details]"
       default_pm_prefix: "[Untitled AI bot PM]"
       personas:
         default_llm_required: "Default LLM model is required prior to enabling Chat"

lib/ai_bot/playground.rb

@@ -533,14 +533,26 @@ def reply_to(post, custom_instructions: nil, &blk)
           reply_post.post_custom_prompt.update!(custom_prompt: prompt)
         end
 
+        reply_post
+      rescue => e
+        if reply_post
+          details = e.message.to_s
+          reply = "#{reply}\n\n#{I18n.t("discourse_ai.ai_bot.reply_error", details: details)}"
+          reply_post.revise(
+            bot.bot_user,
+            { raw: reply },
+            skip_validations: true,
+            skip_revision: true,
+          )
+        end
+        raise e
+      ensure
         # since we are skipping validations and jobs we
         # may need to fix participant count
-        if reply_post.topic.private_message? && reply_post.topic.participant_count < 2
+        if reply_post && reply_post.topic && reply_post.topic.private_message? &&
+             reply_post.topic.participant_count < 2
           reply_post.topic.update!(participant_count: 2)
         end
-
-        reply_post
-      ensure
         post_streamer&.finish(skip_callback: true)
         publish_final_update(reply_post) if stream_reply
         if reply_post && post.post_number == 1 && post.topic.private_message?

lib/completions/dialects/chat_gpt.rb

@@ -6,7 +6,8 @@ module Dialects
       class ChatGpt < Dialect
         class << self
           def can_translate?(llm_model)
-            llm_model.provider == "open_ai" || llm_model.provider == "azure"
+            llm_model.provider == "open_router" || llm_model.provider == "open_ai" ||
+              llm_model.provider == "azure"
           end
         end
 

lib/completions/endpoints/base.rb

@@ -21,6 +21,7 @@ def endpoint_for(provider_name)
               DiscourseAi::Completions::Endpoints::Cohere,
               DiscourseAi::Completions::Endpoints::SambaNova,
               DiscourseAi::Completions::Endpoints::Mistral,
+              DiscourseAi::Completions::Endpoints::OpenRouter,
             ]
 
             endpoints << DiscourseAi::Completions::Endpoints::Ollama if Rails.env.development?

lib/completions/endpoints/open_router.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Completions
+    module Endpoints
+      class OpenRouter < OpenAi
+        def self.can_contact?(model_provider)
+          %w[open_router].include?(model_provider)
+        end
+
+        def prepare_request(payload)
+          headers = { "Content-Type" => "application/json" }
+          api_key = llm_model.api_key
+
+          headers["Authorization"] = "Bearer #{api_key}"
+          headers["X-Title"] = "Discourse AI"
+          headers["HTTP-Referer"] = "https://www.discourse.org/ai"
+
+          Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }
+        end
+
+        def prepare_payload(prompt, model_params, dialect)
+          payload = super
+
+          if quantizations = llm_model.provider_params["provider_quantizations"].presence
+            options = quantizations.split(",").map(&:strip)
+
+            payload[:provider] = { quantizations: options }
+          end
+
+          if order = llm_model.provider_params["provider_order"].presence
+            options = order.split(",").map(&:strip)
+            payload[:provider] ||= {}
+            payload[:provider][:order] = options
+          end
+
+          payload
+        end
+      end
+    end
+  end
+end

lib/completions/llm.rb

@@ -108,6 +108,24 @@ def presets
                   endpoint: "https://api.mistral.ai/v1/chat/completions",
                   provider: "mistral",
                 },
+                {
+                  id: "open_router",
+                  models: [
+                    {
+                      name: "meta-llama/llama-3.3-70b-instruct",
+                      tokens: 128_000,
+                      display_name: "Llama 3.3 70B",
+                    },
+                    {
+                      name: "google/gemini-flash-1.5-exp",
+                      tokens: 1_000_000,
+                      display_name: "Gemini Flash 1.5 Exp",
+                    },
+                  ],
+                  tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer,
+                  endpoint: "https://openrouter.ai/api/v1/chat/completions",
+                  provider: "open_router",
+                },
               ]
             end
         end
@@ -124,6 +142,7 @@ def provider_names
             azure
             samba_nova
             mistral
+            open_router
           ]
           if !Rails.env.production?
             providers << "fake"

lib/summarization/strategies/hot_topic_gists.rb

@@ -64,7 +64,7 @@ def summary_extension_prompt(summary, contents)
               .map { |item| "(#{item[:id]} #{item[:poster]} said: #{item[:text]} " }
               .join("\n")
 
-          prompt = DiscourseAi::Completions::Prompt.new(<<~TEXT.strip)
+          prompt = DiscourseAi::Completions::Prompt.new(<<~TEXT.strip, topic_id: target.id)
             You are an advanced summarization bot. Your task is to update an existing single-sentence summary by integrating new developments from a conversation.
             Analyze the most recent messages to identify key updates or shifts in the main topic and reflect these in the updated summary.
             Emphasize new significant information or developments within the context of the initial conversation theme.
@@ -103,7 +103,7 @@ def first_summary_prompt(contents)
           statements =
             contents.to_a.map { |item| "(#{item[:id]} #{item[:poster]} said: #{item[:text]} " }
 
-          prompt = DiscourseAi::Completions::Prompt.new(<<~TEXT.strip)
+          prompt = DiscourseAi::Completions::Prompt.new(<<~TEXT.strip, topic_id: target.id)
             You are an advanced summarization bot. Analyze a given conversation and produce a concise,
             single-sentence summary that conveys the main topic and current developments to someone with no prior context.
 
@@ -124,9 +124,9 @@ def first_summary_prompt(contents)
             ### Context:
 
             #{content_title.present? ? "The discussion title is: " + content_title + ". (DO NOT REPEAT THIS IN THE SUMMARY)\n" : ""}
-            
+
             The conversation began with the following statement:
-        
+
             #{statements.shift}\n
           TEXT