FEATURE: Add periodic problem checks for each LLM in use

Author: keegangeorge

app/services/problem_check/ai_llm_status.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ProblemCheck::AiLlmStatus < ProblemCheck
+  self.priority = "high"
+  # self.perform_every = 1.hour
+
+  def call
+    return no_problem if !SiteSetting.discourse_ai_enabled?
+    return no_problem if llm_operational?
+
+    problem
+  end
+
+  private
+
+  def llm_operational?
+    model_ids = DiscourseAi::Configuration::LlmEnumerator.global_usage.keys
+    models_to_check = LlmModel.where(id: model_ids)
+
+    models_to_check.each do |model|
+      begin
+        result = validator.run_test(model)
+        return false unless result
+      rescue StandardError => e
+        Rails.logger.warn("LlmValidator encountered an error for model #{model.id}: #{e.message}")
+        return false
+      end
+    end
+
+    return true
+  end
+
+  def validator
+    @validator ||= DiscourseAi::Configuration::LlmValidator.new
+  end
+end

config/locales/server.en.yml

@@ -452,3 +452,6 @@ en:
       no_default_llm: The persona must have a default_llm defined.
       user_not_allowed: The user is not allowed to participate in the topic.
       prompt_message_length: The message %{idx} is over the 1000 character limit.
+  dashboard:
+    problem:
+      ai_llm_status: "An LLM Model needs attention!"

plugin.rb

@@ -73,6 +73,8 @@ def self.public_asset_path(name)
     DiscourseAi::AiBot::EntryPoint.new,
   ].each { |a_module| a_module.inject_into(self) }
 
+  register_problem_check ProblemCheck::AiLlmStatus
+
   register_reviewable_type ReviewableAiChatMessage
   register_reviewable_type ReviewableAiPost