FEATURE: allow researcher to also research specific topics (#1339)

* FEATURE: allow researcher to also research specific topics

Also improve UI around research with more accurate info

* this ensures that under no conditions PMs will be included

Author: SamSaffron

config/locales/server.en.yml

@@ -399,6 +399,7 @@ en:
         create_image: "Creating image"
         edit_image: "Editing image"
         researcher: "Researching"
+        researcher_dry_run: "Preparing research"
       tool_help:
         read_artifact: "Read a web artifact using the AI Bot"
         update_artifact: "Update a web artifact using the AI Bot"
@@ -461,11 +462,11 @@ en:
         setting_context: "Reading context for: %{setting_name}"
         schema: "%{tables}"
         researcher_dry_run:
-          one: "Proposed research: %{goals}\n\nFound %{count} result for '%{filter}'"
-          other: "Proposed research: %{goals}\n\nFound %{count} result for '%{filter}'"
+          one: "Proposed goals: %{goals}\n\nFound %{count} post matching '%{filter}'"
+          other: "Proposed goals: %{goals}\n\nFound %{count} posts matching '%{filter}'"
         researcher:
-          one: "Researching: %{goals}\n\nFound %{count} result for '%{filter}'"
-          other: "Researching: %{goals}\n\nFound %{count} result for '%{filter}'"
+          one: "Researching: %{goals}\n\nFound %{count} post matching '%{filter}'"
+          other: "Researching: %{goals}\n\nFound %{count} posts matching '%{filter}'"
         search_settings:
           one: "Found %{count} result for '%{query}'"
           other: "Found %{count} results for '%{query}'"

lib/personas/tools/researcher.rb

@@ -22,7 +22,7 @@ def signature
                 },
                 {
                   name: "dry_run",
-                  description: "When true, only count matching items without processing data",
+                  description: "When true, only count matching posts without processing data",
                   type: "boolean",
                 },
               ],
@@ -41,6 +41,7 @@ def filter_description
               - keywords (keywords:keyword1,keyword2) - specific words to search for in posts
               - max_results (max_results:10) the maximum number of results to return (optional)
               - order (order:latest, order:oldest, order:latest_topic, order:oldest_topic) - the order of the results (optional)
+              - topic (topic:topic_id1,topic_id2) - add specific topics to the filter, topics will unconditionally be included
 
               If multiple tags or categories are specified, they are treated as OR conditions.
 
@@ -89,7 +90,7 @@ def invoke(&blk)
           blk.call details
 
           if dry_run
-            { dry_run: true, goals: goals, filter: @filter, number_of_results: @result_count }
+            { dry_run: true, goals: goals, filter: @filter, number_of_posts: @result_count }
           else
             process_filter(filter, goals, post, &blk)
           end
@@ -103,6 +104,14 @@ def details
           end
         end
 
+        def summary
+          if @dry_run
+            I18n.t("discourse_ai.ai_bot.tool_summary.researcher_dry_run")
+          else
+            I18n.t("discourse_ai.ai_bot.tool_summary.researcher")
+          end
+        end
+
         def description_args
           { count: @result_count || 0, filter: @filter || "", goals: @goals || "" }
         end

lib/utils/research/filter.rb

@@ -188,6 +188,23 @@ def self.word_to_date(str)
           relation
         end
 
+        register_filter(/\Atopics?:(.*)\z/i) do |relation, topic_param, filter|
+          if topic_param.include?(",")
+            topic_ids = topic_param.split(",").map(&:strip).map(&:to_i).reject(&:zero?)
+            return relation.where("1 = 0") if topic_ids.empty?
+            filter.always_return_topic_ids!(topic_ids)
+            relation
+          else
+            topic_id = topic_param.to_i
+            if topic_id > 0
+              filter.always_return_topic_ids!([topic_id])
+              relation
+            else
+              relation.where("1 = 0") # No results if topic_id is invalid
+            end
+          end
+        end
+
         def initialize(term, guardian: nil, limit: nil, offset: nil)
           @term = term.to_s
           @guardian = guardian || Guardian.new
@@ -196,6 +213,7 @@ def initialize(term, guardian: nil, limit: nil, offset: nil)
           @filters = []
           @valid = true
           @order = :latest_post
+          @topic_ids = nil
 
           @term = process_filters(@term)
         end
@@ -204,17 +222,40 @@ def set_order!(order)
           @order = order
         end
 
+        def always_return_topic_ids!(topic_ids)
+          if @topic_ids
+            @topic_ids = @topic_ids + topic_ids
+          else
+            @topic_ids = topic_ids
+          end
+        end
+
         def limit_by_user!(limit)
           @limit = limit if limit.to_i < @limit.to_i || @limit.nil?
         end
 
         def search
-          filtered = Post.secured(@guardian).joins(:topic).merge(Topic.secured(@guardian))
+          filtered =
+            Post
+              .secured(@guardian)
+              .joins(:topic)
+              .merge(Topic.secured(@guardian))
+              .where("topics.archetype = 'regular'")
+          original_filtered = filtered
 
           @filters.each do |filter_block, match_data|
             filtered = filter_block.call(filtered, match_data, self)
           end
 
+          if @topic_ids.present?
+            filtered =
+              original_filtered.where(
+                "posts.topic_id IN (?) OR posts.id IN (?)",
+                @topic_ids,
+                filtered.select("posts.id"),
+              )
+          end
+
           filtered = filtered.limit(@limit) if @limit.to_i > 0
           filtered = filtered.offset(@offset) if @offset.to_i > 0
 

spec/lib/personas/tools/researcher_spec.rb

@@ -35,7 +35,7 @@
       expect(results[:filter]).to eq("tag:research after:2023")
       expect(results[:goals]).to eq("analyze post patterns")
       expect(results[:dry_run]).to eq(true)
-      expect(results[:number_of_results]).to be > 0
+      expect(results[:number_of_posts]).to be > 0
       expect(researcher.filter).to eq("tag:research after:2023")
       expect(researcher.result_count).to be > 0
     end

spec/lib/utils/research/filter_spec.rb

@@ -2,7 +2,10 @@
 
 describe DiscourseAi::Utils::Research::Filter do
   describe "integration tests" do
-    before_all { SiteSetting.min_topic_title_length = 3 }
+    before_all do
+      SiteSetting.min_topic_title_length = 3
+      SiteSetting.min_personal_message_title_length = 3
+    end
 
     fab!(:user)
 
@@ -51,6 +54,46 @@
     fab!(:feature_bug_post) { Fabricate(:post, topic: feature_bug_topic, user: user) }
     fab!(:no_tag_post) { Fabricate(:post, topic: no_tag_topic, user: user) }
 
+    describe "security filtering" do
+      fab!(:secure_group) { Fabricate(:group) }
+      fab!(:secure_category) { Fabricate(:category, name: "Secure") }
+
+      fab!(:secure_topic) do
+        secure_category.set_permissions(secure_group => :readonly)
+        secure_category.save!
+        Fabricate(
+          :topic,
+          category: secure_category,
+          user: user,
+          title: "This is a secret Secret Topic",
+        )
+      end
+
+      fab!(:secure_post) { Fabricate(:post, topic: secure_topic, user: user) }
+
+      fab!(:pm_topic) { Fabricate(:private_message_topic, user: user) }
+      fab!(:pm_post) { Fabricate(:post, topic: pm_topic, user: user) }
+
+      it "omits secure categories when no guardian is supplied" do
+        filter = described_class.new("")
+        expect(filter.search.pluck(:id)).not_to include(secure_post.id)
+
+        user.groups << secure_group
+        guardian = Guardian.new(user)
+        filter_with_guardian = described_class.new("", guardian: guardian)
+        expect(filter_with_guardian.search.pluck(:id)).to include(secure_post.id)
+      end
+
+      it "omits PMs unconditionally" do
+        filter = described_class.new("")
+        expect(filter.search.pluck(:id)).not_to include(pm_post.id)
+
+        guardian = Guardian.new(user)
+        filter_with_guardian = described_class.new("", guardian: guardian)
+        expect(filter_with_guardian.search.pluck(:id)).not_to include(pm_post.id)
+      end
+    end
+
     describe "tag filtering" do
       it "correctly filters posts by tags" do
         filter = described_class.new("tag:feature")
@@ -76,6 +119,18 @@
         filter = described_class.new("category:Announcements")
         expect(filter.search.pluck(:id)).to contain_exactly(feature_post.id, bug_post.id)
 
+        # it can tack on topics
+        filter =
+          described_class.new(
+            "category:Announcements topic:#{feature_bug_post.topic.id},#{no_tag_post.topic.id}",
+          )
+        expect(filter.search.pluck(:id)).to contain_exactly(
+          feature_post.id,
+          bug_post.id,
+          feature_bug_post.id,
+          no_tag_post.id,
+        )
+
         filter = described_class.new("category:Announcements,Feedback")
         expect(filter.search.pluck(:id)).to contain_exactly(
           feature_post.id,