FIX: Gate translation to public posts or posts within a certain age

Author: nattsw

app/jobs/regular/detect_translate_post.rb

@@ -10,6 +10,11 @@ def execute(args)
       post = Post.find_by(id: args[:post_id])
       return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
 
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        topic = post.topic
+        return if topic.blank? || topic.category&.read_restricted?
+      end
+
       detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")

app/jobs/regular/detect_translate_topic.rb

@@ -12,6 +12,10 @@ def execute(args)
         return
       end
 
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
       detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")

app/jobs/regular/translate_categories.rb

@@ -21,6 +21,12 @@ def execute(args)
       return if categories.empty?
 
       categories.each do |category|
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content &&
+             category.read_restricted?
+          last_id = category.id
+          next
+        end
+
         CategoryLocalization.transaction do
           locales.each do |locale|
             next if CategoryLocalization.exists?(category_id: category.id, locale: locale)

app/jobs/regular/translate_posts.rb

@@ -28,7 +28,25 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("pl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          public_categories = Category.where(read_restricted: false).pluck(:id)
+          posts =
+            posts
+              .joins(:topic)
+              .where(topics: { category_id: public_categories })
+              .where(topics: { archetype: "regular" })
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?
 

app/jobs/regular/translate_topics.rb

@@ -27,7 +27,21 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("tl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          public_categories = Category.where(read_restricted: false).pluck(:id)
+          topics = topics.where(category_id: public_categories)
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?
 

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -10,15 +10,31 @@ def execute(args)
       return unless SiteSetting.experimental_content_translation
       return if SiteSetting.automatic_translation_backfill_rate == 0
 
-      limit = SiteSetting.automatic_translation_backfill_rate
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
           .where.not(raw: [nil, ""])
-          .order(updated_at: :desc)
-          .limit(limit)
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate)
       return if posts.empty?
 
       posts.each do |post|

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -8,16 +8,25 @@ class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
-      return if SiteSetting.automatic_translation_backfill_rate == 0
-
       limit = SiteSetting.automatic_translation_backfill_rate
-      topics =
-        Topic
-          .where(locale: nil)
-          .where(deleted_at: nil)
-          .where("topics.user_id > 0")
-          .order(updated_at: :desc)
-          .limit(limit)
+      return if limit == 0
+
+      topics = Topic.where(locale: nil).where(deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        topics = topics.where(category_id: public_categories)
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
       return if topics.empty?
 
       topics.each do |topic|

spec/jobs/detect_translate_post_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ post_id: post.id }) }.not_to raise_error
   end
+
+  it "skips public content when configured" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    post.topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
 end

spec/jobs/detect_translate_topic_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ topic_id: topic.id }) }.not_to raise_error
   end
+
+  it "skips public content when configured" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
 end

spec/jobs/posts_locale_detection_backfill_spec.rb

@@ -76,4 +76,57 @@
 
     job.execute({})
   end
+
+  describe "with public content limitation" do
+    fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }
+    fab!(:private_topic) { Fabricate(:topic, category: private_category) }
+    fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) }
+
+    before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true }
+
+    it "only processes posts from public categories" do
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_limit_to_public_content = false
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once
+
+      job.execute({})
+    end
+  end
+
+  describe "with max age limit" do
+    fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) }
+    fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) }
+
+    before { SiteSetting.automatic_translation_backfill_max_age_days = 5 }
+
+    it "only processes posts within the age limit" do
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_max_age_days = 0
+
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once
+
+      job.execute({})
+    end
+  end
 end