infra for pdf evals

Author: SamSaffron

app/models/llm_model.rb

@@ -70,7 +70,7 @@ def self.provider_params
   end
 
   def to_llm
-    DiscourseAi::Completions::Llm.proxy(identifier)
+    DiscourseAi::Completions::Llm.proxy(self)
   end
 
   def identifier

evals/lib/llm.rb

@@ -55,6 +55,16 @@ class DiscourseAi::Evals::Llm
       max_prompt_tokens: 1_000_000,
       vision_enabled: true,
     },
+    "gemini-2.0-pro-exp" => {
+      display_name: "Gemini 2.0 pro",
+      name: "gemini-2-0-pro-exp",
+      tokenizer: "DiscourseAi::Tokenizer::GeminiTokenizer",
+      api_key_env: "GEMINI_API_KEY",
+      provider: "google",
+      url: "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-pro-exp",
+      max_prompt_tokens: 1_000_000,
+      vision_enabled: true,
+    },
   }
 
   def self.choose(config_name)
@@ -97,6 +107,8 @@ def eval(type:, args:, expected_output: nil, expected_output_regex: nil)
       case type
       when "helper"
         helper(**args)
+      when "pdf_to_text"
+        pdf_to_text(**args)
       end
 
     if expected_output
@@ -121,8 +133,38 @@ def name
     @llm_model.display_name
   end
 
+  def vision?
+    @llm_model.vision_enabled
+  end
+
   private
 
+  def pdf_to_text(path:)
+    upload =
+      UploadCreator.new(File.open(path), File.basename(path)).create_for(Discourse.system_user.id)
+
+    uploads =
+      DiscourseAi::Utils::PdfToImages.new(
+        upload: upload,
+        user: Discourse.system_user,
+      ).uploaded_pages
+
+    text = +""
+    uploads.each do |page_upload|
+      DiscourseAi::Utils::ImageToText
+        .new(upload: page_upload, llm_model: @llm_model, user: Discourse.system_user)
+        .extract_text do |chunk, error|
+          text << chunk if chunk
+          text << "\n\n" if chunk
+        end
+      upload.destroy
+    end
+
+    text
+  ensure
+    upload.destroy if upload
+  end
+
   def helper(input:, name:)
     completion_prompt = CompletionPrompt.find_by(name: name)
     helper = DiscourseAi::AiHelper::Assistant.new(helper_llm: @llm)

evals/run

@@ -1,6 +1,30 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
+# got to ensure evals are here
+# rubocop:disable Discourse/Plugins/NamespaceConstants
+EVAL_PATH = File.join(__dir__, "cases")
+# rubocop:enable Discourse/Plugins/NamespaceConstants
+#
+if !Dir.exist?(EVAL_PATH)
+  puts "Evals are missing, cloning from discourse/discourse-ai-evals"
+
+  success =
+    system("git clone [email protected]:discourse/discourse-ai-evals.git '#{EVAL_PATH}' 2>/dev/null")
+
+  # Fall back to HTTPS if SSH fails
+  if !success
+    puts "SSH clone failed, falling back to HTTPS..."
+    success = system("git clone https://github.com/discourse/discourse-ai-evals.git '#{EVAL_PATH}'")
+  end
+
+  if success
+    puts "Successfully cloned evals repository"
+  else
+    abort "Failed to clone evals repository"
+  end
+end
+
 discourse_path = File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
 # rubocop:disable Discourse/NoChdir
 Dir.chdir(discourse_path)
@@ -27,10 +51,8 @@ OptionParser
   end
   .parse!
 
-# Ensure output directory exists
 FileUtils.mkdir_p(options[:output_dir])
 
-# Load and run the specified evaluation
 if options[:eval_name].nil?
   puts "Error: Must specify an evaluation name with -e or --eval"
   exit 1
@@ -54,11 +76,21 @@ end
 
 eval_info = YAML.load_file(cases[options[:eval_name]]).symbolize_keys
 
+# correct relative paths in args
+begin
+  eval_info[:args]&.each do |k, v|
+    if k.to_sym == :path
+      root = File.dirname(cases[options[:eval_name]])
+      eval_info[:args][k] = File.join(root, v)
+    end
+  end
+end
+
 puts "Running evaluation '#{options[:eval_name]}'"
 
 log_filename = "#{options[:eval_name]}-#{Time.now.strftime("%Y%m%d-%H%M%S")}.log"
 logs_dir = File.join(__dir__, "log")
-FileUtils.mkdir_p(logs_dir) # Create directory if it doesn't exist
+FileUtils.mkdir_p(logs_dir)
 log_file = File.join(logs_dir, log_filename)
 
 logger = Logger.new(File.open(log_file, "a"))
@@ -68,6 +100,11 @@ logger.info("Starting evaluation '#{options[:eval_name]}'")
 Thread.current[:llm_audit_log] = logger
 
 llms.each do |llm|
+  if eval_info[:vision] && !llm.vision?
+    logger.info("Skipping LLM: #{llm.name} as it does not support vision")
+    next
+  end
+
   logger.info("Evaluating with LLM: #{llm.name}")
   eval =
     llm.eval(