[E] Optimize text section node searching

Author: scryptmouse

api/Gemfile

@@ -93,6 +93,7 @@ gem "paper_trail-globalid", "~> 0.2"
 gem "pg", "~> 1.2"
 gem "pg_search", "~> 2.3.6"
 gem "premailer-rails", "~> 1.0"
+gem "progressbar", "~> 1.13.0"
 gem "pry-rails", "~> 0.3.9"
 gem "puma", "~> 6.4"
 gem "rack", ">= 2.2.13"

api/Gemfile.lock

@@ -567,6 +567,7 @@ GEM
       net-smtp
       premailer (~> 1.7, >= 1.7.9)
     prism (1.4.0)
+    progressbar (1.13.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -950,6 +951,7 @@ DEPENDENCIES
   pg (~> 1.2)
   pg_search (~> 2.3.6)
   premailer-rails (~> 1.0)
+  progressbar (~> 1.13.0)
   pry-byebug
   pry-rails (~> 0.3.9)
   puma (~> 6.4)

api/app/jobs/text_sections/index_current_node_content_job.rb

@@ -24,9 +24,9 @@ def build_enumerator(text_section, cursor:)
     end
 
     # @param [TextSectionNode] text_section_node
-    # @param [Project] _project
+    # @param [TextSection] _text_section
     # @return [void]
-    def each_iteration(text_section_node, _project)
+    def each_iteration(text_section_node, _text_section)
       text_section_node.index_contained_content!
     end
   end

api/app/jobs/text_sections/maintain_all_current_nodes_job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module TextSections
+  class MaintainAllCurrentNodesJob < ApplicationJob
+    include JobIteration::Iteration
+
+    queue_as :low_priority
+
+    queue_with_priority 500
+
+    # @param [String, nil] cursor
+    # @return [Enumerator]
+    def build_enumerator(cursor:)
+      enumerator_builder.active_record_on_records(
+        TextSection.all,
+        cursor:,
+      )
+    end
+
+    # @param [TextSection] text_section
+    # @return [void]
+    def each_iteration(text_section)
+      text_section.maintain_current_nodes!
+    end
+  end
+end

api/app/models/text_section.rb

@@ -83,6 +83,8 @@ class TextSection < ApplicationRecord
   scope :in_texts, ->(texts) { where(text: texts) }
   scope :ordered, -> { order(position: :asc) }
 
+  scope :with_unindexed_nodes, -> { where(id: TextSectionNode.sans_search_indexed.select(:text_section_id).distinct) }
+
   multisearches! :body_text
 
   alias_attribute :title, :name
@@ -217,15 +219,20 @@ def extracted_body_content
   end
 
   # @return [void]
-  def index_contained_content!
-    ManifoldApi::Container["text_sections.index_contained_content"].(self).value!
+  def index_contained_content!(**options)
+    ManifoldApi::Container["text_sections.index_contained_content"].(self, **options).value!
   end
 
   # @return [void]
   def index_nodes!
     ManifoldApi::Container["text_sections.index_nodes"].(self).value!
   end
 
+  # @return [void]
+  def maintain_current_nodes!
+    ManifoldApi::Container["text_sections.maintain_current_nodes"].(self).value!
+  end
+
   private
 
   def extract_content_from!(source, data: [])

api/app/models/text_section_node.rb

@@ -34,8 +34,10 @@ class TextSectionNode < ApplicationRecord
   scope :terminal, -> { where(intermediate: false) }
   scope :with_intermediate_tag, -> { where(tag: INTERMEDIATE_TAGS) }
 
-  scope :current, -> { joins(:text_section).where(TextSection.arel_table[:body_hash].eq(arel_table[:body_hash])) }
-  scope :orphaned, -> { joins(:text_section).where(TextSection.arel_table[:body_hash].not_eq(arel_table[:body_hash])) }
+  scope :current, -> { where(current: true) }
+  scope :orphaned, -> { where(current: false) }
+
+  scope :mismatched_current, -> { joins(:text_section).where(arel_mismatched_current) }
 
   scope :sans_search_indexed, -> { where(search_indexed: false) }
 
@@ -119,15 +121,17 @@ class << self
     # @param [<String>] text_section_ids
     # @return [<Hash>]
     def hit_search_for(keyword, text_section_ids: [])
+      # :nocov:
       return {} if text_section_ids.blank?
+      # :nocov:
 
       query = build_hit_query_for(keyword, text_section_ids:)
 
       hit_filters = text_section_ids.index_with { Search::HitFilter.new }
 
       hit_results = text_section_ids.index_with { [] }.merge(nil => Dry::Core::Constants::EMPTY_ARRAY)
 
-      query.each_with_object(hit_results) do |node, hits|
+      TextSectionNode.find_by_sql(query).each_with_object(hit_results) do |node, hits|
         hit_filter = hit_filters.fetch(node.text_section_id)
 
         next unless hit_filter.allow?(node)
@@ -136,89 +140,22 @@ def hit_search_for(keyword, text_section_ids: [])
       end
     end
 
-    private
-
-    def arel_content_highlighted_for(keyword, node_hits:)
-      q = unscoped.keyword_search(keyword)
-
-      tsearch = q.__send__(:tsearch)
-
-      Arel::Nodes::NamedFunction.new(
-        "ts_headline",
-        [
-          tsearch.__send__(:dictionary),
-          node_hits[:contained_content],
-          tsearch.__send__(:arel_wrap, tsearch.__send__(:tsquery)),
-          Arel::Nodes.build_quoted(tsearch.__send__(:ts_headline_options))
-        ]
-      )
-    end
-
-    # @param [String] keyword
-    # @param [<String>] text_section_ids
-    # @return [ActiveRecord::Relation<TextSectionNode>]
+    # @api private
+    # @return [String]
     def build_hit_query_for(keyword, text_section_ids:)
-      inner_query = build_hit_inner_query_for(keyword, text_section_ids:)
-
-      node_hits = Arel::Table.new("node_hits")
-
-      TextSectionNode.from(inner_query.to_sql, "node_hits")
-        .reselect(?*)
-        .select(arel_content_highlighted_for(keyword, node_hits:).as("content_highlighted"))
-        .where(node_hits[:hit_number].lteq(MAX_HIT_COUNT))
-        .order(node_hits[:text_section_id].asc)
-        .order(node_hits[:pg_search_rank].desc)
-        .order(node_hits[:hit_number].asc)
-    end
-
-    # @param [String] keyword
-    # @param [<String>] text_section_ids
-    # @return [Arel::Nodes::Grouping]
-    def build_hit_inner_query_for(keyword, text_section_ids:)
-      base_query = all
-        .keyword_search(keyword)
-        .current
-        .where(text_section_id: text_section_ids)
-        .with_pg_search_rank
-
-      hit_number = hit_number_for(base_query)
-
-      hit_inner_query = base_query.select(hit_number.as("hit_number")).to_sql
-
-      in_text_sections = arel_table[:text_section_id].in(text_section_ids).to_sql
-
-      # We need to move this condition inside the pg_search subquery
-      # or else we fetch way too much
-      hit_inner_query = hit_inner_query.sub(
-        /(WHERE\s+)(.+?)(\) AS #{base_query.pg_search_rank_table_alias} ON)/i,
-        %[\\1 #{in_text_sections} AND (\\2)\\3]
-      )
-
-      arel_grouping(
-        arel_literal(
-          hit_inner_query
-        )
-      ).as("node_hits")
+      ManifoldApi::Container["text_section_nodes.build_hits_query"].(keyword:, text_section_ids:).value!
     end
 
-    # @param [ActiveRecord::Relation<TextSectionNode>] query
-    # @return [Arel::Nodes::NamedFunction]
-    def hit_number_for(query)
-      window = search_window_for query
-
-      arel_named_fn("row_number").over(window)
-    end
+    # @api private
+    # @return [Arel::Nodes::Inequality]
+    def arel_mismatched_current
+      text_sections = TextSection.arel_table
+      text_section_nodes = TextSectionNode.arel_table
 
-    # @param [ActiveRecord::Relation<TextSectionNode>] query
-    # @return [Arel::Nodes::Window]
-    def search_window_for(query)
-      rank_table = Arel::Table.new(query.pg_search_rank_table_alias)
+      body_hash_matches = arel_grouping(text_sections[:body_hash].eq(text_section_nodes[:body_hash]))
+      current = text_section_nodes[:current]
 
-      Arel::Nodes::Window.new
-        .partition(arel_table[:text_section_id])
-        .order(rank_table[:rank].desc)
-        .order(arel_table[:node_indices].desc)
-        .order(arel_table[:id].asc)
+      body_hash_matches.not_eq(current)
     end
   end
 end

api/app/operations/text_section_nodes/build_hits_query.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module TextSectionNodes
+  # @see TextSectionNodes::HitsQueryBuilder
+  class BuildHitsQuery
+    # @return [Dry::Monads::Result]
+    def call(...)
+      TextSectionNodes::HitsQueryBuilder.new(...).call
+    end
+  end
+end

api/app/operations/text_section_nodes/index_contained_content.rb

@@ -19,15 +19,15 @@ class IndexContainedContent
         string_agg(cn.content, ' ' ORDER BY cn.node_indices) FILTER (WHERE cn.content IS NOT NULL AND cn.content ~ '[^[:space:]]+') AS contained_content
         FROM text_section_nodes pn
         INNER JOIN text_section_nodes cn ON cn.text_section_id = %<text_section_id>s AND cn.body_hash = %<body_hash>s AND pn.node_path @> cn.node_path
-        WHERE pn.id = %<text_section_node_id>s
+        WHERE pn.text_section_id = %<text_section_id>s AND pn.id = %<text_section_node_id>s
         GROUP BY pn.id
     )
     UPDATE text_section_nodes tsn SET
       contained_node_uuids = COALESCE(d.contained_node_uuids, '{}'::text[]),
       contained_content = CASE WHEN char_length(d.contained_content) <= 4096 THEN d.contained_content ELSE '' END,
       search_indexed = TRUE
       FROM derived d
-      WHERE tsn.id = %<text_section_node_id>s
+      WHERE tsn.text_section_id = %<text_section_id>s AND tsn.id = %<text_section_node_id>s
     ;
     SQL
 

api/app/operations/text_sections/extrapolate_nodes.rb

@@ -77,7 +77,7 @@ class ExtrapolateNodes
         (#{TAG_IS_INTERMEDIATE}) AS intermediate,
         CURRENT_TIMESTAMP AS extrapolated_at,
         CURRENT_TIMESTAMP AS search_indexed_at,
-        TRUE AS search_indexed
+        TRUE AS current
       FROM nodes
     )
     SQL
@@ -95,7 +95,7 @@ class ExtrapolateNodes
       intermediate,
       extrapolated_at,
       search_indexed_at,
-      search_indexed
+      current
     ) SELECT
       text_section_id, body_hash,
       node_root, node_path, path,
@@ -108,7 +108,7 @@ class ExtrapolateNodes
       intermediate,
       extrapolated_at,
       search_indexed_at,
-      search_indexed
+      current
       FROM finalized
     ON CONFLICT (node_path) DO UPDATE SET
       "text_section_id" = EXCLUDED."text_section_id",
@@ -128,6 +128,7 @@ class ExtrapolateNodes
       "children_count" = EXCLUDED."children_count",
       "intermediate" = EXCLUDED."intermediate",
       "extrapolated_at" = EXCLUDED."extrapolated_at",
+      "current" = TRUE,
       "updated_at" =
       CASE
       WHEN EXCLUDED."text_section_id" IS DISTINCT FROM text_section_nodes."text_section_id"

api/app/operations/text_sections/index_contained_content.rb

@@ -18,25 +18,32 @@ class IndexContainedContent
         pn.text_section_id = %<text_section_id>s
         AND
         pn.body_hash = %<body_hash>s
+        %<search_indexed_filter>s
         GROUP BY pn.id
     )
     UPDATE text_section_nodes tsn SET
       contained_node_uuids = COALESCE(d.contained_node_uuids, '{}'::text[]),
       contained_content = CASE WHEN char_length(d.contained_content) <= 4096 THEN d.contained_content ELSE '' END,
       search_indexed = TRUE
       FROM derived d
-      WHERE tsn.id = d.text_section_node_id
+      WHERE tsn.text_section_id = %<text_section_id>s AND tsn.id = d.text_section_node_id
     ;
     SQL
 
     # @param [TextSection] text_section
     # @return [Dry::Monads::Result]
-    def call(text_section)
+    def call(text_section, unindexed_only: true)
       params = {
         text_section_id: text_section.quoted_id,
         body_hash: text_section.body_hash,
       }
 
+      if unindexed_only
+        params[:search_indexed_filter] = "AND pn.search_indexed = FALSE"
+      else
+        params[:search_indexed_filter] = ""
+      end
+
       query = QUERY % params
 
       response = sql_update!(query)