Batch xml generation (#403)

Author: dcollie2

app/jobs/file_upload_job.rb

@@ -1,7 +1,7 @@
 class FileUploadJob < ApplicationJob
   # Consider removing concurrency limits due to SolidQueue blocking issues
   # or use a more specific key to avoid blocking all jobs for a language
-  limits_concurrency key: ->(language_id, content_id, content_type) { "#{language_id}-#{content_type}-#{content_id}" }
+  limits_concurrency to: 3, key: ->(_language_id, content_id, _content_type) { "hard-limit" }
 
   retry_on AzureFileShares::Errors::ApiError, wait: :exponentially_longer, attempts: 3
   retry_on Timeout::Error, wait: :exponentially_longer, attempts: 2

app/jobs/language_files_job.rb

@@ -1,4 +1,6 @@
 class LanguageFilesJob < ApplicationJob
+  limits_concurrency to: 1, key: ->(language_id) { "hard-limit" }
+
   def perform(language_id)
     language = Language.find(language_id)
     LanguageContentProcessor.new(language).perform

app/services/xml_generator/all_providers.rb

@@ -7,12 +7,22 @@ def initialize(language, **args)
   attr_reader :language, :args
 
   def xml_content(xml)
-    language.providers
-      .select("providers.id, providers.name, topics.id AS topic_id, topics.title AS topic_title, topics.created_at AS topic_created_at")
-      .joins(:topics)
-      .merge(Topic.with_attached_documents)
-      .each do |provider|
-        xml << provider_xml(provider)
+    # Avoid building a massive join result; iterate provider ids in small slices.
+    ActiveRecord::Base.uncached do
+      provider_ids_in_language_in_batches do |ids|
+        Provider.where(id: ids).order(:id).each do |provider|
+          xml << provider_xml(provider) # provider_xml should eager-load topics/attachments per provider
+        end
       end
+    end
+  end
+
+  private
+
+  # Yields slices of provider ids that have topics in this language.
+  def provider_ids_in_language_in_batches(batch_size: 500)
+    Topic.where(language_id: language.id).distinct.pluck(:provider_id).each_slice(batch_size) do |ids|
+      yield ids
+    end
   end
 end

app/services/xml_generator/single_provider.rb

@@ -35,7 +35,10 @@ def provider_xml(provider)
                     end
                   end
                 end
-                title_element << Ox::Element.new("topic_tags").tap { |tags| tags << topic.current_tags_list.join(", ") }
+                title_element << Ox::Element.new("topic_tags").tap do |tags|
+                  names = topic.taggings.map { |tg| tg.tag&.name }.compact.uniq
+                  tags << names.join(", ")
+                end
               end
             end
           end
@@ -45,12 +48,16 @@ def provider_xml(provider)
   end
 
   def grouped_topics(prov)
-    topic_scope(prov).group_by { |topic| [ topic.created_at.year, topic.created_at.strftime("%m_%B") ] }
+    topic_scope(prov).group_by { |topic| [ topic.published_at.year, topic.published_at.strftime("%m_%B") ] }
   end
 
   def topic_scope(prov)
-    return prov.topics.where("created_at > ?", 1.month.ago) if args.fetch(:recent, false)
-
-    prov.topics
+    scope = prov.topics
+    scope = scope.where("published_at > ?", 1.month.ago) if args.fetch(:recent, false)
+    scope
+      .select(:id, :title, :published_at, :language_id, :provider_id)
+      .includes(:language, taggings: :tag)  # eager-load language and taggings->tag
+      .with_attached_documents              # eager-load Active Storage attachments + blobs
+      .order(published_at: :desc)
   end
 end

spec/services/xml_generator/all_providers_spec.rb

@@ -24,8 +24,8 @@
         <?xml version="1.0"?>
         <cmes>
           <content_provider name="#{provider1.name}">
-            <topic_year year="#{topic1.created_at.year}">
-              <topic_month month="#{topic1.created_at.strftime("%m_%B")}">
+            <topic_year year="#{topic1.published_at.year}">
+              <topic_month month="#{topic1.published_at.strftime("%m_%B")}">
                 <title name="#{topic1.title}">
                   <topic_id>#{topic1.id}</topic_id>
                   <topic_files files="Files"/>
@@ -35,8 +35,8 @@
             </topic_year>
           </content_provider>
           <content_provider name="#{provider2.name}">
-            <topic_year year="#{topic2.created_at.year}">
-              <topic_month month="#{topic2.created_at.strftime("%m_%B")}">
+            <topic_year year="#{topic2.published_at.year}">
+              <topic_month month="#{topic2.published_at.strftime("%m_%B")}">
                 <title name="#{topic2.title}">
                   <topic_id>#{topic2.id}</topic_id>
                   <topic_files files="Files">

spec/services/xml_generator/single_provider_spec.rb

@@ -40,8 +40,8 @@
           <?xml version="1.0"?>
           <cmes>
             <content_provider name="#{provider.name}">
-              <topic_year year="#{topic.created_at.year}">
-                <topic_month month="#{topic.created_at.strftime("%m_%B")}">
+              <topic_year year="#{topic.published_at.year}">
+                <topic_month month="#{topic.published_at.strftime("%m_%B")}">
                   <title name="#{topic.title}">
                     <topic_id>#{topic.id}</topic_id>
                     <topic_files files="Files">