DEV: better

Author: keegangeorge

app/models/llm_model.rb

@@ -13,6 +13,11 @@ class LlmModel < ActiveRecord::Base
   validates_presence_of :name, :api_key
   validates :max_prompt_tokens, numericality: { greater_than: 0 }
   validate :required_provider_params
+  scope :in_use,
+        -> do
+          model_ids = DiscourseAi::Configuration::LlmEnumerator.global_usage.keys
+          where(id: model_ids)
+        end
 
   def self.provider_params
     {

app/services/problem_check/ai_llm_status.rb

@@ -5,29 +5,44 @@ class ProblemCheck::AiLlmStatus < ProblemCheck
   # self.perform_every = 1.hour
 
   def call
-    return no_problem if !SiteSetting.discourse_ai_enabled?
-    return no_problem if llm_operational?
-
-    problem
+    [*llm_errors]
   end
 
   private
 
-  def llm_operational?
-    model_ids = DiscourseAi::Configuration::LlmEnumerator.global_usage.keys
-    models_to_check = LlmModel.where(id: model_ids)
-
-    models_to_check.each do |model|
-      begin
-        result = validator.run_test(model)
-        return false unless result
-      rescue StandardError => e
-        Rails.logger.warn("LlmValidator encountered an error for model #{model.id}: #{e.message}")
-        return false
-      end
+  def targets
+    [*LlmModel.in_use.pluck(:id)]
+  end
+
+  def llm_errors
+    return [] if !SiteSetting.discourse_ai_enabled
+
+    LlmModel.in_use.find_each.filter_map do |model|
+      try_validate(model) { validator.run_test(model) }
     end
+  end
 
-    return true
+  def try_validate(model, &blk)
+    begin
+      blk.call
+      nil
+    rescue => e
+      error_message = JSON.parse(e.message)["message"]
+      message =
+        "#{I18n.t("dashboard.problem.ai_llm_status", { model_name: model.display_name, model_id: model.id })}"
+
+      Problem.new(
+        message,
+        priority: "high",
+        identifier: "ai_llm_checker",
+        target: model.id,
+        details: {
+          model_id: model.id,
+          model_name: model.display_name,
+          error: error_message,
+        },
+      )
+    end
   end
 
   def validator

config/locales/server.en.yml

@@ -455,4 +455,4 @@ en:
       prompt_message_length: The message %{idx} is over the 1000 character limit.
   dashboard:
     problem:
-      ai_llm_status: "An LLM Model needs attention!"
+      ai_llm_status: "The LLM model: %{model_name} is encountering issues. Please check the <a href='/admin/plugins/discourse-ai/ai-llms/%{model_id}'>model's configuration</a>."