improve filter logic and feature set

Author: SamSaffron

lib/personas/tools/researcher.rb

@@ -33,19 +33,24 @@ def filter_description
             <<~TEXT
               Filter string to target specific content.
               - Supports user (@username)
+              - post_type:first - only includes first posts in topics
+              - post_type:reply - only replies in topics
               - date ranges (after:YYYY-MM-DD, before:YYYY-MM-DD for posts; topic_after:YYYY-MM-DD, topic_before:YYYY-MM-DD for topics)
-              - categories (category:category1,category2)
-              - tags (tag:tag1,tag2)
-              - groups (group:group1,group2).
+              - categories (category:category1,category2 or categories:category1,category2)
+              - tags (tag:tag1,tag2 or tags:tag1,tag2)
+              - groups (group:group1,group2 or groups:group1,group2)
               - status (status:open, status:closed, status:archived, status:noreplies, status:single_user)
-              - keywords (keywords:keyword1,keyword2) - specific words to search for in posts
-              - max_results (max_results:10) the maximum number of results to return (optional)
-              - order (order:latest, order:oldest, order:latest_topic, order:oldest_topic) - the order of the results (optional)
-              - topic (topic:topic_id1,topic_id2) - add specific topics to the filter, topics will unconditionally be included
+              - keywords (keywords:keyword1,keyword2) - searches for specific words within post content using full-text search
+              - topic_keywords (topic_keywords:keyword1,keyword2) - searches for keywords within topics, returns all posts from matching topics
+              - topics (topic:topic_id1,topic_id2 or topics:topic_id1,topic_id2) - target specific topics by ID
+              - max_results (max_results:10) - limits the maximum number of results returned (optional)
+              - order (order:latest, order:oldest, order:latest_topic, order:oldest_topic, order:likes) - controls result ordering (optional, defaults to latest posts)
 
-              If multiple tags or categories are specified, they are treated as OR conditions.
+              Multiple filters can be combined with spaces for AND logic. Example: '@sam after:2023-01-01 tag:feature'
 
-              Multiple filters can be combined with spaces. Example: '@sam after:2023-01-01 tag:feature'
+              Use OR to combine filter segments for inclusive logic.
+              Example: 'category:feature,bug OR tag:feature-tag' - includes posts in feature OR bug categories, OR posts with feature-tag tag
+              Example: '@sam category:bug' - includes posts by @sam AND in bug category
             TEXT
           end
 

lib/utils/research/filter.rb

@@ -4,7 +4,6 @@ module DiscourseAi
   module Utils
     module Research
       class Filter
-        # Stores custom filter handlers
         def self.register_filter(matcher, &block)
           (@registered_filters ||= {})[matcher] = block
         end
@@ -19,7 +18,6 @@ def self.word_to_date(str)
 
         attr_reader :term, :filters, :order, :guardian, :limit, :offset, :invalid_filters
 
-        # Define all filters at class level
         register_filter(/\Astatus:open\z/i) do |relation, _, _|
           relation.where("topics.closed = false AND topics.archived = false")
         end
@@ -109,6 +107,30 @@ def self.word_to_date(str)
           end
         end
 
+        register_filter(/\Atopic_keywords?:(.*)\z/i) do |relation, keywords_param, _|
+          if keywords_param.blank?
+            relation
+          else
+            keywords = keywords_param.split(",").map(&:strip).reject(&:blank?)
+            if keywords.empty?
+              relation
+            else
+              ts_query = keywords.map { |kw| kw.gsub(/['\\]/, " ") }.join(" | ")
+
+              relation.where(
+                "posts.topic_id IN (
+                  SELECT posts2.topic_id
+                  FROM posts posts2
+                  JOIN post_search_data ON post_search_data.post_id = posts2.id
+                  WHERE post_search_data.search_data @@ to_tsquery(?, ?)
+                )",
+                ::Search.ts_config,
+                ts_query,
+              )
+            end
+          end
+        end
+
         register_filter(/\A(?:categories?|category):(.*)\z/i) do |relation, category_param, _|
           if category_param.include?(",")
             category_names = category_param.split(",").map(&:strip)
@@ -140,26 +162,36 @@ def self.word_to_date(str)
           end
         end
 
-        register_filter(/\Ain:posted\z/i) do |relation, _, filter|
-          if filter.guardian.user
-            relation.where("posts.user_id = ?", filter.guardian.user.id)
-          else
-            relation.where("1 = 0") # No results if not logged in
-          end
-        end
+        register_filter(/\Agroups?:([a-zA-Z0-9_\-,]+)\z/i) do |relation, groups_param, filter|
+          if groups_param.include?(",")
+            group_names = groups_param.split(",").map(&:strip)
+            found_group_ids = []
+            group_names.each do |name|
+              group = Group.find_by("name ILIKE ?", name)
+              found_group_ids << group.id if group
+            end
 
-        register_filter(/\Agroup:([a-zA-Z0-9_\-]+)\z/i) do |relation, name, filter|
-          group = Group.find_by("name ILIKE ?", name)
-          if group
+            return relation.where("1 = 0") if found_group_ids.empty?
             relation.where(
               "posts.user_id IN (
-              SELECT gu.user_id FROM group_users gu
-              WHERE gu.group_id = ?
-            )",
-              group.id,
+                SELECT gu.user_id FROM group_users gu
+                WHERE gu.group_id IN (?)
+              )",
+              found_group_ids,
             )
           else
-            relation.where("1 = 0") # No results if group doesn't exist
+            group = Group.find_by("name ILIKE ?", groups_param)
+            if group
+              relation.where(
+                "posts.user_id IN (
+                SELECT gu.user_id FROM group_users gu
+                WHERE gu.group_id = ?
+              )",
+                group.id,
+              )
+            else
+              relation.where("1 = 0") # No results if group doesn't exist
+            end
           end
         end
 
@@ -188,33 +220,46 @@ def self.word_to_date(str)
           relation
         end
 
+        register_filter(/\Aorder:likes\z/i) do |relation, order_str, filter|
+          filter.set_order!(:likes)
+          relation
+        end
+
         register_filter(/\Atopics?:(.*)\z/i) do |relation, topic_param, filter|
           if topic_param.include?(",")
             topic_ids = topic_param.split(",").map(&:strip).map(&:to_i).reject(&:zero?)
             return relation.where("1 = 0") if topic_ids.empty?
-            filter.always_return_topic_ids!(topic_ids)
-            relation
+            relation.where("posts.topic_id IN (?)", topic_ids)
           else
             topic_id = topic_param.to_i
             if topic_id > 0
-              filter.always_return_topic_ids!([topic_id])
-              relation
+              relation.where("posts.topic_id = ?", topic_id)
             else
               relation.where("1 = 0") # No results if topic_id is invalid
             end
           end
         end
 
+        register_filter(/\Apost_type:(first|reply)\z/i) do |relation, post_type, _|
+          if post_type.downcase == "first"
+            relation.where("posts.post_number = 1")
+          elsif post_type.downcase == "reply"
+            relation.where("posts.post_number > 1")
+          else
+            relation
+          end
+        end
+
         def initialize(term, guardian: nil, limit: nil, offset: nil)
           @guardian = guardian || Guardian.new
           @limit = limit
           @offset = offset
           @filters = []
           @valid = true
           @order = :latest_post
-          @topic_ids = nil
           @invalid_filters = []
           @term = term.to_s.strip
+          @or_groups = []
 
           process_filters(@term)
         end
@@ -223,42 +268,38 @@ def set_order!(order)
           @order = order
         end
 
-        def always_return_topic_ids!(topic_ids)
-          if @topic_ids
-            @topic_ids = @topic_ids + topic_ids
-          else
-            @topic_ids = topic_ids
-          end
-        end
-
         def limit_by_user!(limit)
           @limit = limit if limit.to_i < @limit.to_i || @limit.nil?
         end
 
         def search
-          filtered =
+          base_relation =
             Post
               .secured(@guardian)
               .joins(:topic)
               .merge(Topic.secured(@guardian))
               .where("topics.archetype = 'regular'")
-          original_filtered = filtered
 
-          @filters.each do |filter_block, match_data|
-            filtered = filter_block.call(filtered, match_data, self)
+          # Handle OR groups
+          if @or_groups.any?
+            or_relations =
+              @or_groups.map do |or_group|
+                group_relation = base_relation
+                or_group.each do |filter_block, match_data|
+                  group_relation = filter_block.call(group_relation, match_data, self)
+                end
+                group_relation
+              end
+
+            # Combine OR groups
+            filtered = or_relations.reduce { |combined, current| combined.or(current) }
+          else
+            filtered = base_relation
           end
 
-          if @topic_ids.present?
-            if original_filtered == filtered
-              filtered = original_filtered.where("posts.topic_id IN (?)", @topic_ids)
-            else
-              filtered =
-                original_filtered.where(
-                  "posts.topic_id IN (?) OR posts.id IN (?)",
-                  @topic_ids,
-                  filtered.select("posts.id"),
-                )
-            end
+          # Apply regular AND filters
+          @filters.each do |filter_block, match_data|
+            filtered = filter_block.call(filtered, match_data, self)
           end
 
           filtered = filtered.limit(@limit) if @limit.to_i > 0
@@ -272,17 +313,36 @@ def search
             filtered = filtered.order("topics.created_at DESC, posts.post_number DESC")
           elsif @order == :oldest_topic
             filtered = filtered.order("topics.created_at ASC, posts.post_number ASC")
+          elsif @order == :likes
+            filtered = filtered.order("posts.like_count DESC, posts.created_at DESC")
           end
 
           filtered
         end
 
-        private
-
         def process_filters(term)
           return if term.blank?
 
-          term
+          # Split by OR first, then process each group
+          or_parts = term.split(/\s+OR\s+/i)
+
+          if or_parts.size > 1
+            # Multiple OR groups
+            or_parts.each do |or_part|
+              group_filters = []
+              process_filter_group(or_part.strip, group_filters)
+              @or_groups << group_filters if group_filters.any?
+            end
+          else
+            # Single group (AND logic)
+            process_filter_group(term, @filters)
+          end
+        end
+
+        private
+
+        def process_filter_group(term_part, filter_collection)
+          term_part
             .to_s
             .scan(/(([^" \t\n\x0B\f\r]+)?(("[^"]+")?))/)
             .to_a
@@ -292,7 +352,7 @@ def process_filters(term)
               found = false
               self.class.registered_filters.each do |matcher, block|
                 if word =~ matcher
-                  @filters << [block, $1]
+                  filter_collection << [block, $1]
                   found = true
                   break
                 end

spec/lib/utils/research/filter_spec.rb

@@ -8,13 +8,17 @@
     end
 
     fab!(:user)
+    fab!(:user2) { Fabricate(:user) }
 
     fab!(:feature_tag) { Fabricate(:tag, name: "feature") }
     fab!(:bug_tag) { Fabricate(:tag, name: "bug") }
 
     fab!(:announcement_category) { Fabricate(:category, name: "Announcements") }
     fab!(:feedback_category) { Fabricate(:category, name: "Feedback") }
 
+    fab!(:group1) { Fabricate(:group, name: "group1") }
+    fab!(:group2) { Fabricate(:group, name: "group2") }
+
     fab!(:feature_topic) do
       Fabricate(
         :topic,
@@ -54,6 +58,32 @@
     fab!(:feature_bug_post) { Fabricate(:post, topic: feature_bug_topic, user: user) }
     fab!(:no_tag_post) { Fabricate(:post, topic: no_tag_topic, user: user) }
 
+    describe "group filtering" do
+      before do
+        group1.add(user)
+        group2.add(user2)
+      end
+
+      it "supports filtering by groups" do
+        no_tag_post.update!(user_id: user2.id)
+
+        filter = described_class.new("group:group1")
+        expect(filter.search.pluck(:id)).to contain_exactly(
+          feature_post.id,
+          bug_post.id,
+          feature_bug_post.id,
+        )
+
+        filter = described_class.new("groups:group1,group2")
+        expect(filter.search.pluck(:id)).to contain_exactly(
+          feature_post.id,
+          bug_post.id,
+          feature_bug_post.id,
+          no_tag_post.id,
+        )
+      end
+    end
+
     describe "security filtering" do
       fab!(:secure_group) { Fabricate(:group) }
       fab!(:secure_category) { Fabricate(:category, name: "Secure") }
@@ -122,7 +152,7 @@
         # it can tack on topics
         filter =
           described_class.new(
-            "category:Announcements topic:#{feature_bug_post.topic.id},#{no_tag_post.topic.id}",
+            "category:Announcements OR topic:#{feature_bug_post.topic.id},#{no_tag_post.topic.id}",
           )
         expect(filter.search.pluck(:id)).to contain_exactly(
           feature_post.id,
@@ -175,6 +205,25 @@
         Fabricate(:post, raw: "No fruits here", topic: no_tag_topic, user: user)
       end
 
+      fab!(:reply_on_bananas) do
+        Fabricate(:post, raw: "Just a reply", topic: post_with_bananas.topic, user: user)
+      end
+
+      it "correctly filters posts by topic_keywords" do
+        topic1 = post_with_bananas.topic
+        topic2 = post_with_both.topic
+
+        filter = described_class.new("topic_keywords:banana")
+        expected = topic1.posts.pluck(:id) + topic2.posts.pluck(:id)
+        expect(filter.search.pluck(:id)).to contain_exactly(*expected)
+
+        filter = described_class.new("topic_keywords:banana post_type:first")
+        expect(filter.search.pluck(:id)).to contain_exactly(
+          topic1.posts.order(:post_number).first.id,
+          topic2.posts.order(:post_number).first.id,
+        )
+      end
+
       it "correctly filters posts by full text keywords" do
         filter = described_class.new("keywords:apples")
         expect(filter.search.pluck(:id)).to contain_exactly(post_with_apples.id, post_with_both.id)