work in progress, eval

Author: SamSaffron

evals/cases/ai_helper/simple_proofread.yml

@@ -0,0 +1,9 @@
+id: simple_proofread
+name: Simple Proofread
+description: A simple proofread evaluation
+type: helper
+args:
+  name: proofread
+  input: "Haw are you doing todayy?"
+expected_output: "How are you doing today?"
+

evals/lib/llm.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module DiscourseAi::Evals
+end
+
+class DiscourseAi::Evals::Llm
+  CONFIGS = {
+    "gpt-4o" => {
+      display_name: "GPT-4o",
+      name: "gpt-4o",
+      tokenizer: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+      api_key_env: "OPENAI_API_KEY",
+      provider: "open_ai",
+      url: "https://api.openai.com/v1/chat/completions",
+      max_prompt_tokens: 131_072,
+      vision_enabled: true,
+    },
+    "gpt-4o-mini" => {
+      display_name: "GPT-4o-mini",
+      name: "gpt-4o-mini",
+      tokenizer: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+      api_key_env: "OPENAI_API_KEY",
+      provider: "open_ai",
+      url: "https://api.openai.com/v1/chat/completions",
+      max_prompt_tokens: 131_072,
+      vision_enabled: true,
+    },
+    "claude-3.5-haiku" => {
+      display_name: "Claude 3.5 Haiku",
+      name: "claude-3-5-haiku-latest",
+      tokenizer: "DiscourseAi::Tokenizer::AnthropicTokenizer",
+      api_key_env: "ANTHROPIC_API_KEY",
+      provider: "anthropic",
+      url: "https://api.anthropic.com/v1/messages",
+      max_prompt_tokens: 200_000,
+      vision_enabled: false,
+    },
+    "claude-3.5-sonnet" => {
+      display_name: "Claude 3.5 Sonnet",
+      name: "claude-3-5-sonnet-latest",
+      tokenizer: "DiscourseAi::Tokenizer::AnthropicTokenizer",
+      api_key_env: "ANTHROPIC_API_KEY",
+      provider: "anthropic",
+      url: "https://api.anthropic.com/v1/messages",
+      max_prompt_tokens: 200_000,
+      vision_enabled: true,
+    },
+    "gemini-2.0-flash" => {
+      display_name: "Gemini 2.0 Flash",
+      name: "gemini-2-0-flash",
+      tokenizer: "DiscourseAi::Tokenizer::GeminiTokenizer",
+      api_key_env: "GEMINI_API_KEY",
+      provider: "google",
+      url: "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash",
+      max_prompt_tokens: 1_000_000,
+      vision_enabled: true,
+    },
+  }
+
+  def self.choose(config_name)
+    if CONFIGS[config_name].nil?
+      CONFIGS.keys.map { |config_name| new(config_name) }
+    elsif !CONFIGS.include?(config_name)
+      raise "Invalid llm"
+    else
+      [new(config_name)]
+    end
+  end
+
+  attr_reader :llm_model
+  attr_reader :llm
+
+  def initialize(config_name)
+    config = CONFIGS[config_name].dup
+    api_key_env = config.delete(:api_key_env)
+    if !ENV[api_key_env]
+      raise "Missing API key for #{config_name}, should be set via #{api_key_env}"
+    end
+
+    config[:api_key] = ENV[api_key_env]
+    @llm_model = LlmModel.new(config)
+    @llm = DiscourseAi::Completions::Llm.proxy(@llm_model)
+  end
+
+  def eval(type:, args:, expected_output: nil)
+    result =
+      case type
+      when "helper"
+        helper(**args)
+      end
+
+    if expected_output && result == expected_output
+      { result: :pass }
+    else
+      { result: :fail, expected_output: expected_output, actual_output: result }
+    end
+  end
+
+  def name
+    @llm_model.display_name
+  end
+
+  private
+
+  def helper(input:, name:)
+    completion_prompt = CompletionPrompt.find_by(name: name)
+    helper = DiscourseAi::AiHelper::Assistant.new(helper_llm: @llm)
+    result =
+      helper.generate_and_send_prompt(
+        completion_prompt,
+        input,
+        current_user = Discourse.system_user,
+        _force_default_locale = false,
+      )
+
+    result[:suggestions].first
+  end
+end

evals/run

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+discourse_path = File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
+# rubocop:disable Discourse/NoChdir
+Dir.chdir(discourse_path)
+# rubocop:enable Discourse/NoChdir
+
+require "/home/sam/Source/discourse/config/environment"
+require_relative "lib/llm"
+
+# Set up command line argument parsing
+require "optparse"
+ENV["DISCOURSE_AI_NO_DEBUG"] = "1"
+
+options = { eval_name: nil, model: nil, output_dir: File.join(discourse_path, "tmp", "evals") }
+
+OptionParser
+  .new do |opts|
+    opts.banner = "Usage: evals/run [options]"
+
+    opts.on("-e", "--eval NAME", "Name of the evaluation to run") do |eval_name|
+      options[:eval_name] = eval_name
+    end
+
+    opts.on("-m", "--model NAME", "Model to evaluate") { |model| options[:model] = model }
+
+    opts.on("-o", "--output-dir DIR", "Directory for evaluation results") do |dir|
+      options[:output_dir] = dir
+    end
+  end
+  .parse!
+
+# Ensure output directory exists
+FileUtils.mkdir_p(options[:output_dir])
+
+# Load and run the specified evaluation
+if options[:eval_name].nil?
+  puts "Error: Must specify an evaluation name with -e or --eval"
+  exit 1
+end
+
+cases_path = File.join(__dir__, "cases")
+
+cases = Dir.glob(File.join(cases_path, "*/*.yml")).map { |f| [File.basename(f, ".yml"), f] }.to_h
+
+if !cases.keys.include?(options[:eval_name])
+  puts "Error: Unknown evaluation '#{options[:eval_name]}'"
+  exit 1
+end
+
+llms = DiscourseAi::Evals::Llm.choose(options[:model])
+
+if llms.empty?
+  puts "Error: Unknown model '#{options[:model]}'"
+  exit 1
+end
+
+eval_info = YAML.load_file(cases[options[:eval_name]]).symbolize_keys
+
+puts "Running evaluation '#{options[:eval_name]}'"
+
+llms.each do |llm|
+  eval =
+    llm.eval(
+      type: eval_info[:type],
+      args: eval_info[:args].symbolize_keys,
+      expected_output: eval_info[:expected_output],
+    )
+
+  print "#{llm.name}: "
+  if eval[:result] == :fail
+    puts "Error: #{eval.inspect}"
+  elsif eval[:result] == :pass
+    puts "Passed 🟢"
+  else
+    STDERR.puts "Error: Unknown result #{eval.inspect}"
+  end
+end

lib/ai_helper/assistant.rb

@@ -13,6 +13,20 @@ def self.clear_prompt_cache!
         prompt_cache.flush!
       end
 
+      def initialize(helper_llm: nil, image_caption_llm: nil)
+        @helper_llm = helper_llm
+        @image_caption_llm = image_caption_llm
+      end
+
+      def helper_llm
+        @helper_llm || DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model)
+      end
+
+      def image_caption_llm
+        @image_caption_llm ||
+          DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_image_caption_model)
+      end
+
       def available_prompts(user)
         key = "prompt_cache_#{I18n.locale}"
         self
@@ -115,7 +129,7 @@ def localize_prompt!(prompt, user = nil, force_default_locale = false)
       end
 
       def generate_prompt(completion_prompt, input, user, force_default_locale = false, &block)
-        llm = DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model)
+        llm = helper_llm
         prompt = completion_prompt.messages_with_input(input)
         localize_prompt!(prompt, user, force_default_locale)
 
@@ -182,7 +196,7 @@ def generate_image_caption(upload, user)
           )
 
         raw_caption =
-          DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_image_caption_model).generate(
+          image_caption_llm.generate(
             prompt,
             user: user,
             max_tokens: 1024,

lib/completions/endpoints/base.rb

@@ -223,7 +223,7 @@ def perform_completion!(
                 log.duration_msecs = (Time.now - start_time) * 1000
                 log.save!
                 LlmQuota.log_usage(@llm_model, user, log.request_tokens, log.response_tokens)
-                if Rails.env.development?
+                if Rails.env.development? && !ENV["DISCOURSE_AI_NO_DEBUG"]
                   puts "#{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
                 end
               end