try to add ... for elided lines, it appears to work well

SamSaffron · SamSaffron · commit 9fafaa417175 · 2025-02-19T14:32:57.000+11:00
diff --git a/evals/lib/eval.rb b/evals/lib/eval.rb
@@ -28,9 +28,11 @@ def initialize(path:)
     @expected_tool_call = @yaml[:expected_tool_call]
     @expected_tool_call.symbolize_keys! if @expected_tool_call
 
-    @args[:path] = File.expand_path(File.join(File.dirname(path), @args[:path])) if @args&.key?(
-      :path,
-    )
+    @args.each do |key, value|
+      if (key.to_s.include?("_path") || key.to_s == "path") && value.is_a?(String)
+        @args[key] = File.expand_path(File.join(File.dirname(path), value))
+      end
+    end
   end
 
   def run(llm:)
@@ -44,6 +46,8 @@ def run(llm:)
         image_to_text(llm, **args)
       when "prompt"
         prompt_call(llm, **args)
+      when "edit_artifact"
+        edit_artifact(llm, **args)
       end
 
     if expected_output
@@ -53,7 +57,7 @@ def run(llm:)
         { result: :fail, expected_output: expected_output, actual_output: result }
       end
     elsif expected_output_regex
-      if result.match?(expected_output_regex)
+      if result.to_s.match?(expected_output_regex)
         { result: :pass }
       else
         { result: :fail, expected_output: expected_output_regex, actual_output: result }
@@ -169,4 +173,36 @@ def prompt_call(llm, system_prompt:, message:, tools: nil, stream: false)
     end
     result
   end
+
+  def edit_artifact(llm, css_path:, js_path:, html_path:, instructions_path:)
+    css = File.read(css_path)
+    js = File.read(js_path)
+    html = File.read(html_path)
+    instructions = File.read(instructions_path)
+    artifact =
+      AiArtifact.create!(
+        css: css,
+        js: js,
+        html: html,
+        user_id: Discourse.system_user.id,
+        post_id: 1,
+        name: "eval artifact",
+      )
+
+    post = Post.new(topic_id: 1, id: 1)
+    DiscourseAi::AiBot::ArtifactUpdateStrategies::Diff.new(
+      llm: llm.llm_model.to_llm,
+      post: post,
+      user: Discourse.system_user,
+      artifact: artifact,
+      artifact_version: nil,
+      instructions: instructions,
+    ).apply
+
+    version = artifact.versions.last
+    output = { css: version.css, js: version.js, html: version.html }
+
+    artifact.destroy
+    output
+  end
 end
diff --git a/lib/ai_bot/artifact_update_strategies/diff.rb b/lib/ai_bot/artifact_update_strategies/diff.rb
@@ -110,6 +110,8 @@ def system_prompt
             5. Keep changes minimal and focused
             6. HTML should not include <html>, <head>, or <body> tags, it is injected into a template
             7. When specifying a SEARCH block, ALWAYS keep it 8 lines or less, you will be interrupted and a retry will be required if you exceed this limit
+            8. NEVER EVER ask followup questions, ALL changes must be performed in a single response, you are consumed via an API, there is no opportunity for humans in the loop
+            9. When performing a non-contiguous search, ALWAYS use ... to denote the skipped lines
 
             JavaScript libraries must be sourced from the following CDNs, otherwise CSP will reject it:
             #{AiArtifact::ALLOWED_CDN_SOURCES.join("\n")}
@@ -154,6 +156,36 @@ def system_prompt
             .text { font-size: 16px; }
             >>>>>>> REPLACE
             [/CSS]
+
+            Example - Non contiguous search in CSS (replace all CSS with new CSS)
+
+            Original CSS:
+            [CSS]
+            body {
+              color: red;
+            }
+            .button {
+              color: blue;
+            }
+            .alert {
+              background-color: green;
+            }
+            [/CSS]
+
+            [CSS]
+            <<<<<<< SEARCH
+            body {
+            ...
+              background-color: green;
+            }
+            =======
+            body {
+              color: red;
+            }
+            >>>>>>> REPLACE
+
+            This will replace the entire CSS block with the new CSS block, given that the search block is non-contiguous and unambiguous.
+
           PROMPT
         end
 
diff --git a/lib/utils/diff_utils/simple_diff.rb b/lib/utils/diff_utils/simple_diff.rb
@@ -86,6 +86,7 @@ def non_contiguous_match_range(lines, search_lines)
           search_index = 0
 
           lines.each_with_index do |line, idx|
+            search_index += 1 if search_lines[search_index].strip == "..."
             if line.strip == search_lines[search_index].strip
               first_idx ||= idx
               last_idx = idx
diff --git a/spec/lib/utils/diff_utils/simple_diff_spec.rb b/spec/lib/utils/diff_utils/simple_diff_spec.rb
@@ -185,7 +185,9 @@
 
       search = <<~TEXT
         line1
+        ...
          line3
+        ...
         line1
       TEXT
 

-Original file line number
+Diff line change
       search = <<~TEXT
         line1
 +        ...
          line3
 +        ...
         line1
       TEXT