improve eval output

Author: SamSaffron

evals/lib/eval.rb

@@ -85,10 +85,10 @@ def run(llm:)
         { result: :pass }
       end
     else
-      { result: :unknown, actual_output: result }
+      { result: :pass }
     end
-  rescue EvalError
-    { result: :fail }
+  rescue EvalError => e
+    { result: :fail, message: e.message, context: e.context }
   end
 
   def print
@@ -227,10 +227,39 @@ def edit_artifact(llm, css_path:, js_path:, html_path:, instructions_path:)
       raise EvalError.new("Failed to apply all changes", diff.failed_searches)
     end
 
+    raise EvalError.new("Invalid JS", artifact.js) if !valid_javascript?(artifact.js)
+
     version = artifact.versions.last
     output = { css: version.css, js: version.js, html: version.html }
 
     artifact.destroy
     output
   end
+
+  def valid_javascript?(str)
+    require "open3"
+
+    # Create a temporary file with the JavaScript code
+    Tempfile.create(%w[test .js]) do |f|
+      f.write(str)
+      f.flush
+
+      File.write("/tmp/test.js", str)
+
+      begin
+        Discourse::Utils.execute_command(
+          "node",
+          "--check",
+          f.path,
+          failure_message: "Invalid JavaScript syntax",
+          timeout: 30, # reasonable timeout in seconds
+        )
+        true
+      rescue Discourse::Utils::CommandError
+        false
+      end
+    end
+  rescue StandardError
+    false
+  end
 end

evals/lib/runner.rb

@@ -155,9 +155,16 @@ def run!
 
           if result[:result] == :fail
             puts "Failed 🔴"
-            puts "---- Expected ----\n#{result[:expected_output]}"
-            puts "---- Actual ----\n#{result[:actual_output]}"
+            puts "Error: #{result[:message]}" if result[:message]
+            if result[:expected_output] && result[:actual_output]
+              puts "---- Expected ----\n#{result[:expected_output]}"
+              puts "---- Actual ----\n#{result[:actual_output]}"
+            end
             logger.error("Evaluation failed with LLM: #{llm.name}")
+            logger.error("Error: #{result[:message]}") if result[:message]
+            logger.error("Expected: #{result[:expected_output]}") if result[:expected_output]
+            logger.error("Actual: #{result[:actual_output]}") if result[:actual_output]
+            logger.error("Context: #{result[:context]}") if result[:context]
           elsif result[:result] == :pass
             puts "Passed 🟢"
             logger.info("Evaluation passed with LLM: #{llm.name}")