[E] Background node extrapolation

Related to MNFLD-1100

Author: scryptmouse

api/app/jobs/text_section_nodes/backport_search_index_job.rb

@@ -9,17 +9,16 @@ class BackportSearchIndexJob < ApplicationJob
     unique :until_executed, lock_ttl: 2.days, on_conflict: :log
 
     def build_enumerator(cursor:)
-      enumerator_builder.active_record_on_batch_relations(
+      enumerator_builder.active_record_on_records(
         TextSectionNode.sans_search_indexed,
-        cursor:,
-        batch_size: 1000
+        cursor:
       )
     end
 
-    # @param [ActiveRecord::Relation<TextSectionNode>] batch_relation
+    # @param [TextSectionNode] text_section_node
     # @return [void]
-    def each_iteration(batch_relation)
-      batch_relation.backport_search_index!
+    def each_iteration(text_section_node)
+      text_section_node.index_contained_content!
     end
   end
 end

api/app/jobs/text_sections/index_current_node_content_job.rb

@@ -11,6 +11,8 @@ class IndexCurrentNodeContentJob < ApplicationJob
 
     queue_as :low_priority
 
+    queue_with_priority 500
+
     # @param [TextSection] text_section
     # @param [String, nil] cursor
     # @return [Enumerator]

api/app/jobs/text_sections/index_nodes_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module TextSections
+  # @see TextSections::IndexNodes
+  class IndexNodesJob < ApplicationJob
+    queue_as :default
+
+    queue_with_priority 100
+
+    # @param [TextSection] text_section
+    # @return [void]
+    def perform(text_section)
+      text_section.index_nodes!
+    end
+  end
+end

api/app/models/text_section.rb

@@ -77,8 +77,7 @@ class TextSection < ApplicationRecord
 
   before_validation :update_body_json
   after_destroy :remove_linked_toc_entries
-  after_save :extrapolate_nodes!
-  after_save_commit :asynchronously_index_current_node_content!
+  after_save_commit :asynchronously_index_nodes!
   after_commit :maybe_adopt_or_orphan_annotations!, on: [:update, :destroy]
 
   scope :in_texts, ->(texts) { where(text: texts) }
@@ -196,6 +195,12 @@ def asynchronously_index_current_node_content!
     TextSections::IndexCurrentNodeContentJob.perform_later self
   end
 
+  # @api private
+  # @return [void]
+  def asynchronously_index_nodes!
+    TextSections::IndexNodesJob.perform_later self
+  end
+
   # @api private
   # @return [void]
   def extrapolate_nodes!
@@ -216,6 +221,11 @@ def index_contained_content!
     ManifoldApi::Container["text_sections.index_contained_content"].(self).value!
   end
 
+  # @return [void]
+  def index_nodes!
+    ManifoldApi::Container["text_sections.index_nodes"].(self).value!
+  end
+
   private
 
   def extract_content_from!(source, data: [])

api/app/models/text_section_node.rb

@@ -136,42 +136,6 @@ def hit_search_for(keyword, text_section_ids: [])
       end
     end
 
-    # @api private
-    # @see TextSectionNodes::BackportSearchIndexJob
-    # @return [void]
-    def backport_search_index!
-      ids = all.where_values_hash.fetch("id", [])
-
-      raise "must be called in a batch relation" if ids.blank?
-
-      tsn = arel_table.alias("pn")
-
-      id_in_batch = tsn[:id].in(ids).to_sql
-
-      connection.exec_update(<<~SQL)
-      WITH contained AS (
-        SELECT
-          pn.id AS id,
-          array_agg(cn.node_uuid ORDER BY cn.node_indices) FILTER (WHERE cn.node_uuid IS NOT NULL) AS contained_node_uuids,
-          string_agg(cn.content, ' ' ORDER BY cn.node_indices) FILTER (WHERE cn.content IS NOT NULL AND cn.content ~ '[^[:space:]]+') AS contained_content
-          FROM text_section_nodes pn
-          INNER JOIN text_section_nodes cn ON pn.node_path @> cn.node_path
-          WHERE
-            #{id_in_batch}
-          GROUP BY 1
-      )
-      UPDATE text_section_nodes tsn SET
-        contained_node_uuids = COALESCE(c.contained_node_uuids, '{}'::text[]),
-        contained_content = CASE WHEN char_length(c.contained_content) <= 4096 THEN c.contained_content ELSE '' END,
-        search_indexed_at = CURRENT_TIMESTAMP,
-        search_indexed = TRUE
-      FROM contained c
-      WHERE
-        c.id = tsn.id
-      ;
-      SQL
-    end
-
     private
 
     def arel_content_highlighted_for(keyword, node_hits:)

api/app/operations/text_sections/extrapolate_nodes.rb

@@ -170,13 +170,14 @@ class ExtrapolateNodes
     SQL
 
     def call(**args)
-      corrected = yield correct_intermediate_nodes.(**args)
+      # We only need this if we end up changing the nodes that count as intermediate.
+      # corrected = yield correct_intermediate_nodes.(**args)
 
-      return Success(upserted: 0, corrected:) if args[:text_section].present? && args[:text_section].body_json.blank?
+      return Success(upserted: 0) if args[:text_section].present? && args[:text_section].body_json.blank?
 
       upserted = sql_update! FIRST_PART, interpolate(**args), SECOND_PART, FINAL_PART
 
-      Success(upserted:, corrected:)
+      Success(upserted:)
     end
 
     private

api/app/operations/text_sections/index_nodes.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module TextSections
+  # An operation to extrapolate and prune {TextSectionNode}s for a {TextSection}.
+  # Because this operation can be very large, we run it in the background,
+  # with a separate task to actually index the content for full-text search.
+  # @see TextSections::IndexCurrentNodeContentJob
+  class IndexNodes
+    include Dry::Monads[:result, :do]
+    include ManifoldApi::Deps[
+      extrapolate_nodes: "text_sections.extrapolate_nodes",
+      prune_nodes: "text_sections.prune_nodes",
+    ]
+
+    # @param [TextSection] text_section
+    # @return [Dry::Monads::Result]
+    def call(text_section)
+      results = yield extrapolate_nodes.(text_section:)
+
+      results => { upserted:, }
+
+      pruned = yield prune_nodes.(text_section)
+
+      text_section.asynchronously_index_current_node_content!
+
+      counts = { upserted:, pruned:, }
+
+      Success counts
+    end
+  end
+end

api/app/operations/text_sections/prune_nodes.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module TextSections
+  class PruneNodes
+    include Dry::Monads[:result]
+
+    # @param [TextSection] text_section
+    # @return [Dry::Monads::Success(Integer)]
+    def call(text_section)
+      # :nocov:
+      # We preserve old nodes if there are any annotations
+      # for now. In the future, we need to determine if the
+      # annotations actually reference the nodes being kept.
+      return Success(0) if text_section.annotations.exists?
+      # :nocov:
+
+      pruned = text_section.text_section_nodes.orphaned.delete_all
+
+      Success pruned
+    end
+  end
+end

api/db/migrate/20251016204352_add_contained_content_helper_index.rb

@@ -1,16 +1,8 @@
 # frozen_string_literal: true
 
 class AddContainedContentHelperIndex < ActiveRecord::Migration[7.0]
-  # Needed for CREATE INDEX CONCURRENTLY
-  disable_ddl_transaction!
-
   def change
-    change_table :text_section_nodes do |t|
-      t.index %i[text_section_id body_hash node_path id],
-        name: "index_text_section_nodes_contained_content_indexing",
-        opclass: { node_path: "gist_ltree_ops(siglen=100)" },
-        algorithm: :concurrently,
-        using: :gist
-    end
+    # Intentionally left blank. Old migration file kept
+    # but supplanted with ImproveTextSectionNodeIndex
   end
 end

api/db/migrate/20251017174417_improve_text_section_node_index.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class ImproveTextSectionNodeIndex < ActiveRecord::Migration[7.0]
+  # Needed for CREATE INDEX CONCURRENTLY
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL.squish
+        DROP INDEX CONCURRENTLY IF EXISTS index_text_section_nodes_contained_content_indexing;
+        SQL
+      end
+    end
+
+    change_table :text_section_nodes do |t|
+      t.index %i[text_section_id body_hash node_path id],
+        name: "index_text_section_nodes_contained_content_indexing",
+        opclass: { node_path: "gist_ltree_ops(siglen=24)" },
+        algorithm: :concurrently,
+        using: :gist
+    end
+  end
+end