DEV: Add eval for summarization

Author: keegangeorge

evals/lib/eval.rb

@@ -60,6 +60,8 @@ def run(llm:)
         prompt_call(llm, **args)
       when "edit_artifact"
         edit_artifact(llm, **args)
+      when "summarization"
+        summarization(llm, **args)
       end
 
     if expected_output
@@ -312,4 +314,35 @@ def valid_javascript?(str)
   rescue StandardError
     false
   end
+
+  def summarization(llm, input:)
+    topic =
+      Topic.create!(
+        category: Category.last,
+        title: "Eval topic for topic summarization",
+        id: -99,
+        user_id: Discourse.system_user.id,
+      )
+    post = Post.create!(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
+
+    strategy =
+      DiscourseAi::Summarization::FoldContent.new(
+        llm.llm_proxy,
+        DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
+      )
+
+    result = []
+
+    summary =
+      DiscourseAi::TopicSummarization
+        .new(strategy, Discourse.system_user)
+        .summarize { |partial| result << partial }
+
+    summarized_text = summary.summarized_text
+
+    topic.destroy
+    post.destroy
+
+    summarized_text
+  end
 end