introduce a log

Author: SamSaffron

.gitignore

@@ -2,3 +2,4 @@ node_modules
 /gems
 /auto_generated
 .env
+evals/log/*

evals/run

@@ -24,10 +24,6 @@ OptionParser
     end
 
     opts.on("-m", "--model NAME", "Model to evaluate") { |model| options[:model] = model }
-
-    opts.on("-o", "--output-dir DIR", "Directory for evaluation results") do |dir|
-      options[:output_dir] = dir
-    end
   end
   .parse!
 
@@ -60,7 +56,19 @@ eval_info = YAML.load_file(cases[options[:eval_name]]).symbolize_keys
 
 puts "Running evaluation '#{options[:eval_name]}'"
 
+log_filename = "#{options[:eval_name]}-#{Time.now.strftime("%Y%m%d-%H%M%S")}.log"
+logs_dir = File.join(__dir__, "log")
+FileUtils.mkdir_p(logs_dir) # Create directory if it doesn't exist
+log_file = File.join(logs_dir, log_filename)
+
+logger = Logger.new(File.open(log_file, "a"))
+
+logger.info("Starting evaluation '#{options[:eval_name]}'")
+
+Thread.current[:llm_audit_log] = logger
+
 llms.each do |llm|
+  logger.info("Evaluating with LLM: #{llm.name}")
   eval =
     llm.eval(
       type: eval_info[:type],
@@ -73,9 +81,15 @@ llms.each do |llm|
     puts "Failed 🔴"
     puts "---- Expected ----\n#{eval[:expected_output]}"
     puts "---- Actual ----\n#{eval[:actual_output]}"
+    logger.error("Evaluation failed with LLM: #{llm.name}")
   elsif eval[:result] == :pass
     puts "Passed 🟢"
+    logger.info("Evaluation passed with LLM: #{llm.name}")
   else
     STDERR.puts "Error: Unknown result #{eval.inspect}"
+    logger.error("Unknown result: #{eval.inspect}")
   end
 end
+
+puts
+puts "Log file: #{log_file}"

lib/completions/endpoints/base.rb

@@ -156,15 +156,15 @@ def perform_completion!(
                 )
 
               if !@streaming_mode
-                return(
+                response_data =
                   non_streaming_response(
                     response: response,
                     xml_tool_processor: xml_tool_processor,
                     xml_stripper: xml_stripper,
                     partials_raw: partials_raw,
                     response_raw: response_raw,
                   )
-                )
+                return response_data
               end
 
               begin
@@ -214,6 +214,16 @@ def perform_completion!(
               decode_chunk_finish.each { |partial| blk.call(partial, cancel) }
               return response_data
             ensure
+              if log && (logger = Thread.current[:llm_audit_log])
+                call_data = <<~LOG
+                  #{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}
+                  request:
+                  #{format_possible_json_payload(log.raw_request_payload)}
+                  response:
+                  #{response_data}
+                LOG
+                logger.info(call_data)
+              end
               if log
                 log.raw_response_payload = response_raw
                 final_log_update(log)
@@ -298,6 +308,14 @@ def disable_streaming?
 
         private
 
+        def format_possible_json_payload(payload)
+          begin
+            JSON.pretty_generate(JSON.parse(payload))
+          rescue JSON::ParserError
+            payload
+          end
+        end
+
         def start_log(
           provider_id:,
           request_body:,