FIX: Gate translation to public posts or posts within a certain age (#307)

We have two existing site settings:
- automatic_translation_backfill_limit_to_public_content
- automatic_translation_backfill_max_age_days

We need to use them for PostLocalization (& topic, cat) as well. They are used in the old feature for DiscourseTranslate::PostTranslation (& topic).

Also added sidekiq_options retry: false for the jobs as they are non-critical.

Author: nattsw

app/jobs/regular/detect_translate_post.rb

@@ -2,6 +2,8 @@
 
 module Jobs
   class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
@@ -10,7 +12,18 @@ def execute(args)
       post = Post.find_by(id: args[:post_id])
       return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
 
-      detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        topic = post.topic
+        return if topic.blank? || topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")
       return if locales.blank?

app/jobs/regular/detect_translate_topic.rb

@@ -2,6 +2,8 @@
 
 module Jobs
   class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
@@ -12,7 +14,17 @@ def execute(args)
         return
       end
 
-      detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
 
       locales = SiteSetting.automatic_translation_target_languages.split("|")
       return if locales.blank?

app/jobs/regular/translate_categories.rb

@@ -3,6 +3,8 @@
 module Jobs
   class TranslateCategories < ::Jobs::Base
     cluster_concurrency 1
+    sidekiq_options retry: false
+
     BATCH_SIZE = 50
 
     def execute(args)
@@ -21,6 +23,12 @@ def execute(args)
       return if categories.empty?
 
       categories.each do |category|
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content &&
+             category.read_restricted?
+          last_id = category.id
+          next
+        end
+
         CategoryLocalization.transaction do
           locales.each do |locale|
             next if CategoryLocalization.exists?(category_id: category.id, locale: locale)

app/jobs/regular/translate_posts.rb

@@ -28,7 +28,26 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("pl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          posts =
+            posts.joins(:topic).where(
+              topics: {
+                category_id: Category.where(read_restricted: false).select(:id),
+                archetype: "regular",
+              },
+            )
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?
 

app/jobs/regular/translate_topics.rb

@@ -27,7 +27,20 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("tl.id IS NULL")
-            .limit(limit)
+
+        if SiteSetting.automatic_translation_backfill_limit_to_public_content
+          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+        end
+
+        if SiteSetting.automatic_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?
 

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -3,22 +3,39 @@
 module Jobs
   class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
     every 5.minutes
+    sidekiq_options retry: false
     cluster_concurrency 1
 
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
       return if SiteSetting.automatic_translation_backfill_rate == 0
 
-      limit = SiteSetting.automatic_translation_backfill_rate
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
           .where.not(raw: [nil, ""])
-          .order(updated_at: :desc)
-          .limit(limit)
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate)
       return if posts.empty?
 
       posts.each do |post|

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -3,21 +3,30 @@
 module Jobs
   class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
     every 5.minutes
+    sidekiq_options retry: false
     cluster_concurrency 1
 
     def execute(args)
       return unless SiteSetting.translator_enabled
       return unless SiteSetting.experimental_content_translation
-      return if SiteSetting.automatic_translation_backfill_rate == 0
-
       limit = SiteSetting.automatic_translation_backfill_rate
-      topics =
-        Topic
-          .where(locale: nil)
-          .where(deleted_at: nil)
-          .where("topics.user_id > 0")
-          .order(updated_at: :desc)
-          .limit(limit)
+      return if limit == 0
+
+      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.automatic_translation_backfill_limit_to_public_content
+        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+      end
+
+      if SiteSetting.automatic_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.automatic_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
       return if topics.empty?
 
       topics.each do |topic|

spec/jobs/detect_translate_post_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ post_id: post.id }) }.not_to raise_error
   end
+
+  it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    post.topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
 end

spec/jobs/detect_translate_topic_spec.rb

@@ -68,4 +68,14 @@
 
     expect { job.execute({ topic_id: topic.id }) }.not_to raise_error
   end
+
+  it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do
+    SiteSetting.automatic_translation_backfill_limit_to_public_content = true
+    topic.category.update!(read_restricted: true)
+
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
 end

spec/jobs/posts_locale_detection_backfill_spec.rb

@@ -76,4 +76,57 @@
 
     job.execute({})
   end
+
+  describe "with public content limitation" do
+    fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) }
+    fab!(:private_topic) { Fabricate(:topic, category: private_category) }
+    fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) }
+
+    before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true }
+
+    it "only processes posts from public categories" do
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_limit_to_public_content = false
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once
+
+      job.execute({})
+    end
+  end
+
+  describe "with max age limit" do
+    fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) }
+    fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) }
+
+    before { SiteSetting.automatic_translation_backfill_max_age_days = 5 }
+
+    it "only processes posts within the age limit" do
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never
+
+      job.execute({})
+    end
+
+    it "processes all posts when setting is disabled" do
+      SiteSetting.automatic_translation_backfill_max_age_days = 0
+
+      # other posts
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once
+
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once
+      DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once
+
+      job.execute({})
+    end
+  end
 end