FEATURE: Add periodic problem checks for each LLM in use

app/models/llm_model.rb

@@ -13,6 +13,11 @@ class LlmModel < ActiveRecord::Base
   validates_presence_of :name, :api_key
   validates :max_prompt_tokens, numericality: { greater_than: 0 }
   validate :required_provider_params
+  scope :in_use,
+        -> do
+          model_ids = DiscourseAi::Configuration::LlmEnumerator.global_usage.keys
+          where(id: model_ids)
+        end
 
   def self.provider_params
     {

app/services/problem_check/ai_llm_status.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class ProblemCheck::AiLlmStatus < ProblemCheck
+  self.priority = "high"
+  self.perform_every = 6.hours
+
+  def call
+    llm_errors
+  end
+
+  def base_path
+    Discourse.base_path
+  end
+
+  private
+
+  def llm_errors
+    return [] if !SiteSetting.discourse_ai_enabled
+    LlmModel.in_use.find_each.filter_map do |model|
+      try_validate(model) { validator.run_test(model) }
+    end
+  end
+
+  def try_validate(model, &blk)
+    begin
+      blk.call
+      nil
+    rescue => e
+      error_message = parse_error_message(e.message)
+      message =
+        "#{I18n.t("dashboard.problem.ai_llm_status", { base_path: base_path, model_name: model.display_name, model_id: model.id })}"
+
+      Problem.new(
+        message,
+        priority: "high",
+        identifier: "ai_llm_status",
+        target: model.id,
+        details: {
+          model_id: model.id,
+          model_name: model.display_name,
+          error: error_message,
+        },
+      )
+    end
+  end
+
+  def validator
+    @validator ||= DiscourseAi::Configuration::LlmValidator.new
+  end
+
+  def parse_error_message(message)
+    begin
+      JSON.parse(message)["message"]
+    rescue JSON::ParserError
+      message.to_s
+    end
+  end
+end

config/locales/server.en.yml

@@ -453,3 +453,6 @@ en:
       no_default_llm: The persona must have a default_llm defined.
       user_not_allowed: The user is not allowed to participate in the topic.
       prompt_message_length: The message %{idx} is over the 1000 character limit.
+  dashboard:
+    problem:
+      ai_llm_status: "The LLM model: %{model_name} is encountering issues. Please check the <a href='%{base_path}/admin/plugins/discourse-ai/ai-llms/%{model_id}'>model's configuration page</a>."

plugin.rb

@@ -75,6 +75,8 @@ def self.public_asset_path(name)
     DiscourseAi::AiModeration::EntryPoint.new,
   ].each { |a_module| a_module.inject_into(self) }
 
+  register_problem_check ProblemCheck::AiLlmStatus
+
   register_reviewable_type ReviewableAiChatMessage
   register_reviewable_type ReviewableAiPost
 

spec/services/problem_check/ai_llm_status_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe ProblemCheck::AiLlmStatus do
+  subject(:check) { described_class.new }
+
+  fab!(:llm_model)
+
+  let(:post_url) { "https://api.openai.com/v1/chat/completions" }
+  let(:success_response) do
+    {
+      model: "gpt-4-turbo",
+      usage: {
+        max_prompt_tokens: 131_072,
+      },
+      choices: [
+        { message: { role: "assistant", content: "test" }, finish_reason: "stop", index: 0 },
+      ],
+    }.to_json
+  end
+
+  let(:error_response) do
+    { message: "API key error! Please check you have supplied the correct API key." }.to_json
+  end
+
+  before do
+    stub_request(:post, post_url).to_return(status: 200, body: success_response, headers: {})
+    SiteSetting.ai_summarization_model = "custom:#{llm_model.id}"
+    SiteSetting.ai_summarization_enabled = true
+  end
+
+  describe "#call" do
+    it "does nothing if discourse-ai plugin disabled" do
+      SiteSetting.discourse_ai_enabled = false
+      expect(check).to be_chill_about_it
+    end
+
+    context "with discourse-ai plugin enabled for the site" do
+      before { SiteSetting.discourse_ai_enabled = true }
+
+      it "returns a problem with an LLM model" do
+        stub_request(:post, post_url).to_return(status: 403, body: error_response, headers: {})
+        message =
+          "#{I18n.t("dashboard.problem.ai_llm_status", { base_path: Discourse.base_path, model_name: llm_model.display_name, model_id: llm_model.id })}"
+
+        expect(described_class.new.call).to contain_exactly(
+          have_attributes(
+            identifier: "ai_llm_status",
+            target: llm_model.id,
+            priority: "high",
+            message: message,
+            details: {
+              model_id: llm_model.id,
+              model_name: llm_model.display_name,
+              error: JSON.parse(error_response)["message"],
+            },
+          ),
+        )
+      end
+
+      it "does not return a problem if the LLM models are working" do
+        stub_request(:post, post_url).to_return(status: 200, body: success_response, headers: {})
+        expect(check).to be_chill_about_it
+      end
+    end
+  end
+end