some attempts at improving reliability

Author: SamSaffron

evals/lib/eval.rb

@@ -12,6 +12,15 @@ class DiscourseAi::Evals::Eval
               :expected_output_regex,
               :expected_tool_call
 
+  class EvalError < StandardError
+    attr_reader :context
+
+    def initialize(message, context)
+      super(message)
+      @context = context
+    end
+  end
+
   def initialize(path:)
     @yaml = YAML.load_file(path).symbolize_keys
     @path = path
@@ -78,6 +87,8 @@ def run(llm:)
     else
       { result: :unknown, actual_output: result }
     end
+  rescue EvalError
+    { result: :fail }
   end
 
   def print
@@ -190,14 +201,22 @@ def edit_artifact(llm, css_path:, js_path:, html_path:, instructions_path:)
       )
 
     post = Post.new(topic_id: 1, id: 1)
-    DiscourseAi::AiBot::ArtifactUpdateStrategies::Diff.new(
-      llm: llm.llm_model.to_llm,
-      post: post,
-      user: Discourse.system_user,
-      artifact: artifact,
-      artifact_version: nil,
-      instructions: instructions,
-    ).apply
+    diff =
+      DiscourseAi::AiBot::ArtifactUpdateStrategies::Diff.new(
+        llm: llm.llm_model.to_llm,
+        post: post,
+        user: Discourse.system_user,
+        artifact: artifact,
+        artifact_version: nil,
+        instructions: instructions,
+      )
+    diff.apply
+
+    if diff.failed_searches.present?
+      puts "Eval Errors encountered"
+      p diff.failed_searches
+      raise EvalError.new("Failed to apply all changes", diff.failed_searches)
+    end
 
     version = artifact.versions.last
     output = { css: version.css, js: version.js, html: version.html }

lib/ai_bot/artifact_update_strategies/diff.rb

@@ -3,8 +3,15 @@ module DiscourseAi
   module AiBot
     module ArtifactUpdateStrategies
       class Diff < Base
+        attr_reader :failed_searches
+
         private
 
+        def initialize(**kwargs)
+          super
+          @failed_searches = []
+        end
+
         def build_prompt
           DiscourseAi::Completions::Prompt.new(
             system_prompt,
@@ -58,8 +65,10 @@ def apply_changes(changes)
                     block[:replace],
                   )
               rescue DiscourseAi::Utils::DiffUtils::SimpleDiff::NoMatchError
+                @failed_searches << { section: section, search: block[:search] }
                 # TODO, we may need to inform caller here, LLM made a mistake which it
                 # should correct
+                puts "Failed to find search: #{block[:search]}"
               end
             end
             updated_content[section == :javascript ? :js : section] = content
@@ -112,6 +121,7 @@ def system_prompt
             7. When specifying a SEARCH block, ALWAYS keep it 8 lines or less, you will be interrupted and a retry will be required if you exceed this limit
             8. NEVER EVER ask followup questions, ALL changes must be performed in a single response, you are consumed via an API, there is no opportunity for humans in the loop
             9. When performing a non-contiguous search, ALWAYS use ... to denote the skipped lines
+            10. Be mindful that ... non-contiguous search is not greedy, the following line will only match the first occurrence of the search block
 
             JavaScript libraries must be sourced from the following CDNs, otherwise CSP will reject it:
             #{AiArtifact::ALLOWED_CDN_SOURCES.join("\n")}
@@ -157,9 +167,10 @@ def system_prompt
             >>>>>>> REPLACE
             [/CSS]
 
-            Example - Non contiguous search in CSS (replace all CSS with new CSS)
+            Example - Non contiguous search in CSS (replace most CSS with new CSS)
 
             Original CSS:
+
             [CSS]
             body {
               color: red;
@@ -170,6 +181,9 @@ def system_prompt
             .alert {
               background-color: green;
             }
+            .alert2 {
+              background-color: green;
+            }
             [/CSS]
 
             [CSS]
@@ -184,7 +198,16 @@ def system_prompt
             }
             >>>>>>> REPLACE
 
-            This will replace the entire CSS block with the new CSS block, given that the search block is non-contiguous and unambiguous.
+            RESULT:
+
+            [CSS]
+            body {
+              color: red;
+            }
+            .alert2 {
+              background-color: green;
+            }
+            [/CSS]
 
           PROMPT
         end

lib/utils/diff_utils/simple_diff.rb

@@ -86,7 +86,10 @@ def non_contiguous_match_range(lines, search_lines)
           search_index = 0
 
           lines.each_with_index do |line, idx|
-            search_index += 1 if search_lines[search_index].strip == "..."
+            if search_lines[search_index].strip == "..."
+              search_index += 1
+              break if search_lines[search_index].nil?
+            end
             if line.strip == search_lines[search_index].strip
               first_idx ||= idx
               last_idx = idx