FIX: Gate translation to public posts or posts within a certain age

app/jobs/regular/detect_translate_post.rb

@@ -2,6 +2,8 @@
 
 module Jobs
   class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
@@ -10,7 +12,18 @@ def execute(args)
       post = Post.find_by(id: args[:post_id])
       return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
 
-      detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        topic = post.topic
+        return if topic.blank? || topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")
       return if locales.blank?

app/jobs/regular/detect_translate_topic.rb

@@ -2,6 +2,8 @@
 
 module Jobs
   class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
@@ -12,7 +14,17 @@ def execute(args)
         return
       end
 
-      detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")
       return if locales.blank?

app/jobs/regular/translate_categories.rb

@@ -3,6 +3,8 @@
 module Jobs
   class TranslateCategories < ::Jobs::Base
     cluster_concurrency 1
+    sidekiq_options retry: false
+
     BATCH_SIZE = 50
 
     def execute(args)
@@ -21,6 +23,12 @@ def execute(args)
       return if categories.empty?
 
       categories.each do |category|
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content &&
+             category.read_restricted?
+          last_id = category.id
+          next
+        end
+
         CategoryLocalization.transaction do
           locales.each do |locale|
             next if CategoryLocalization.exists?(category_id: category.id, locale: locale)

app/jobs/regular/translate_posts.rb

@@ -28,7 +28,25 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("pl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          public_categories = Category.where(read_restricted: false).pluck(:id)
+          posts =
+            posts
+              .joins(:topic)
+              .where(topics: { category_id: public_categories })
+              .where(topics: { archetype: "regular" })
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?
 

app/jobs/regular/translate_topics.rb

@@ -27,7 +27,21 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("tl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          public_categories = Category.where(read_restricted: false).pluck(:id)
+          topics = topics.where(category_id: public_categories)
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?
 

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -3,22 +3,39 @@
 module Jobs
   class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
     every 5.minutes
+    sidekiq_options retry: false
     cluster_concurrency 1
 
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
       return if SiteSetting.automatic_translation_backfill_rate == 0
 
-      limit = SiteSetting.automatic_translation_backfill_rate
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
           .where.not(raw: [nil, ""])
-          .order(updated_at: :desc)
-          .limit(limit)
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate)
       return if posts.empty?
 
       posts.each do |post|

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -3,21 +3,31 @@
 module Jobs
   class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
     every 5.minutes
+    sidekiq_options retry: false
     cluster_concurrency 1
 
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
-      return if SiteSetting.automatic_translation_backfill_rate == 0
-
       limit = SiteSetting.automatic_translation_backfill_rate
-      topics =
-        Topic
-          .where(locale: nil)
-          .where(deleted_at: nil)
-          .where("topics.user_id > 0")
-          .order(updated_at: :desc)
-          .limit(limit)
+      return if limit == 0
+
+      topics = Topic.where(locale: nil).where(deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        topics = topics.where(category_id: public_categories)
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
       return if topics.empty?
 
       topics.each do |topic|

spec/jobs/detect_translate_post_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ post_id: post.id }) }.not_to raise_error
   end
+
+  it "skips public content when configured" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    post.topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
 end

spec/jobs/detect_translate_topic_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ topic_id: topic.id }) }.not_to raise_error
   end
+
+  it "skips public content when configured" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
 end

spec/jobs/posts_locale_detection_backfill_spec.rb

@@ -76,4 +76,57 @@
 
     job.execute({})
   end
+
+  describe "with public content limitation" do
+    fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }
+    fab!(:private_topic) { Fabricate(:topic, category: private_category) }
+    fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) }
+
+    before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true }
+
+    it "only processes posts from public categories" do
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_limit_to_public_content = false
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once
+
+      job.execute({})
+    end
+  end
+
+  describe "with max age limit" do
+    fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) }
+    fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) }
+
+    before { SiteSetting.automatic_translation_backfill_max_age_days = 5 }
+
+    it "only processes posts within the age limit" do
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_max_age_days = 0
+
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once
+
+      job.execute({})
+    end
+  end
 end

spec/jobs/topics_locale_detection_backfill_spec.rb

@@ -76,4 +76,59 @@
 
     job.execute({})
   end
+
+  describe "with public content limitation" do
+    fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }
+    fab!(:public_topic) { Fabricate(:topic, locale: nil) }
+    fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: nil) }
+
+    before do
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once
+
+      SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    end
+
+    it "only processes topics from public categories" do
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).never
+
+      job.execute({})
+    end
+
+    it "processes all topics when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_limit_to_public_content = false
+
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once
+
+      job.execute({})
+    end
+  end
+
+  describe "with max age limit" do
+    fab!(:old_topic) { Fabricate(:topic, locale: nil, created_at: 10.days.ago) }
+    fab!(:new_topic) { Fabricate(:topic, locale: nil, created_at: 2.days.ago) }
+
+    before do
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once
+
+      SiteSetting.automatic_translation_backfill_max_age_days = 5
+    end
+
+    it "only processes topics within the age limit" do
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never
+
+      job.execute({})
+    end
+
+    it "processes all topics when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_max_age_days = 0
+
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once
+      DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once
+
+      job.execute({})
+    end
+  end
 end

spec/jobs/translate_categories_spec.rb

@@ -131,4 +131,24 @@ def localize_all_categories(*locales)
     Jobs::TranslateCategories.send(:remove_const, :BATCH_SIZE)
     Jobs::TranslateCategories.const_set(:BATCH_SIZE, 50)
   end
+
+  it "skips read-restricted categories when configured" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+
+    category1 = Fabricate(:category, name: "Public Category", read_restricted: false)
+    category2 = Fabricate(:category, name: "Private Category", read_restricted: true)
+
+    DiscourseTranslator::CategoryTranslator.expects(:translate).at_least_once
+
+    DiscourseTranslator::CategoryTranslator
+      .expects(:translate)
+      .with(category1, any_parameters)
+      .twice
+    DiscourseTranslator::CategoryTranslator
+      .expects(:translate)
+      .with(category2, any_parameters)
+      .never
+
+    job.execute({})
+  end
 end