From cb12b3da8a4a0353a12ccee4ce8473b3f5ddb8bf Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 27 May 2025 14:42:28 +0800 Subject: [PATCH 1/3] FIX: Gate translation to public posts or posts within a certain age --- app/jobs/regular/detect_translate_post.rb | 5 ++ app/jobs/regular/detect_translate_topic.rb | 4 ++ app/jobs/regular/translate_categories.rb | 6 ++ app/jobs/regular/translate_posts.rb | 20 +++++- app/jobs/regular/translate_topics.rb | 16 ++++- .../posts_locale_detection_backfill.rb | 22 +++++- .../topics_locale_detection_backfill.rb | 27 +++++--- spec/jobs/detect_translate_post_spec.rb | 10 +++ spec/jobs/detect_translate_topic_spec.rb | 10 +++ .../posts_locale_detection_backfill_spec.rb | 53 +++++++++++++++ .../topics_locale_detection_backfill_spec.rb | 55 +++++++++++++++ spec/jobs/translate_categories_spec.rb | 20 ++++++ spec/jobs/translate_posts_spec.rb | 67 +++++++++++++++++++ spec/jobs/translate_topics_spec.rb | 66 ++++++++++++++++++ 14 files changed, 367 insertions(+), 14 deletions(-) diff --git a/app/jobs/regular/detect_translate_post.rb b/app/jobs/regular/detect_translate_post.rb index cf5cacf4..d9662819 100644 --- a/app/jobs/regular/detect_translate_post.rb +++ b/app/jobs/regular/detect_translate_post.rb @@ -10,6 +10,11 @@ def execute(args) post = Post.find_by(id: args[:post_id]) return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0 + if SiteSetting.automatic_translation_backfill_limit_to_public_content + topic = post.topic + return if topic.blank? || topic.category&.read_restricted? + end + detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) locales = SiteSetting.automatic_translation_target_languages.split("|") diff --git a/app/jobs/regular/detect_translate_topic.rb b/app/jobs/regular/detect_translate_topic.rb index e4a4305b..6dfb5aca 100644 --- a/app/jobs/regular/detect_translate_topic.rb +++ b/app/jobs/regular/detect_translate_topic.rb @@ -12,6 +12,10 @@ def execute(args) return end + if SiteSetting.automatic_translation_backfill_limit_to_public_content + return if topic.category&.read_restricted? + end + detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) locales = SiteSetting.automatic_translation_target_languages.split("|") diff --git a/app/jobs/regular/translate_categories.rb b/app/jobs/regular/translate_categories.rb index f79e5ff7..2f6df4bd 100644 --- a/app/jobs/regular/translate_categories.rb +++ b/app/jobs/regular/translate_categories.rb @@ -21,6 +21,12 @@ def execute(args) return if categories.empty? categories.each do |category| + if SiteSetting.automatic_translation_backfill_limit_to_public_content && + category.read_restricted? + last_id = category.id + next + end + CategoryLocalization.transaction do locales.each do |locale| next if CategoryLocalization.exists?(category_id: category.id, locale: locale) diff --git a/app/jobs/regular/translate_posts.rb b/app/jobs/regular/translate_posts.rb index 21806c50..6f28364e 100644 --- a/app/jobs/regular/translate_posts.rb +++ b/app/jobs/regular/translate_posts.rb @@ -28,7 +28,25 @@ def execute(args) .where.not(locale: nil) .where.not(locale: locale) .where("pl.id IS NULL") - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + public_categories = Category.where(read_restricted: false).pluck(:id) + posts = + posts + .joins(:topic) + .where(topics: { category_id: public_categories }) + .where(topics: { archetype: "regular" }) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + posts = + posts.where( + "posts.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + posts = posts.order(updated_at: :desc).limit(limit) next if posts.empty? diff --git a/app/jobs/regular/translate_topics.rb b/app/jobs/regular/translate_topics.rb index a3775b96..37472188 100644 --- a/app/jobs/regular/translate_topics.rb +++ b/app/jobs/regular/translate_topics.rb @@ -27,7 +27,21 @@ def execute(args) .where.not(locale: nil) .where.not(locale: locale) .where("tl.id IS NULL") - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + public_categories = Category.where(read_restricted: false).pluck(:id) + topics = topics.where(category_id: public_categories) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + topics = + topics.where( + "topics.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + topics = topics.order(updated_at: :desc).limit(limit) next if topics.empty? diff --git a/app/jobs/scheduled/posts_locale_detection_backfill.rb b/app/jobs/scheduled/posts_locale_detection_backfill.rb index 61b90a13..58c9e31f 100644 --- a/app/jobs/scheduled/posts_locale_detection_backfill.rb +++ b/app/jobs/scheduled/posts_locale_detection_backfill.rb @@ -10,15 +10,31 @@ def execute(args) return unless SiteSetting.experimental_content_translation return if SiteSetting.automatic_translation_backfill_rate == 0 - limit = SiteSetting.automatic_translation_backfill_rate posts = Post .where(locale: nil) .where(deleted_at: nil) .where("posts.user_id > 0") .where.not(raw: [nil, ""]) - .order(updated_at: :desc) - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + public_categories = Category.where(read_restricted: false).pluck(:id) + posts = + posts + .joins(:topic) + .where(topics: { category_id: public_categories }) + .where(topics: { archetype: "regular" }) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + posts = + posts.where( + "posts.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate) return if posts.empty? posts.each do |post| diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb index cbd96bc4..de7f71a1 100644 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ b/app/jobs/scheduled/topics_locale_detection_backfill.rb @@ -8,16 +8,25 @@ class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation - return if SiteSetting.automatic_translation_backfill_rate == 0 - limit = SiteSetting.automatic_translation_backfill_rate - topics = - Topic - .where(locale: nil) - .where(deleted_at: nil) - .where("topics.user_id > 0") - .order(updated_at: :desc) - .limit(limit) + return if limit == 0 + + topics = Topic.where(locale: nil).where(deleted_at: nil).where("topics.user_id > 0") + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + public_categories = Category.where(read_restricted: false).pluck(:id) + topics = topics.where(category_id: public_categories) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + topics = + topics.where( + "topics.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + topics = topics.order(updated_at: :desc).limit(limit) return if topics.empty? topics.each do |topic| diff --git a/spec/jobs/detect_translate_post_spec.rb b/spec/jobs/detect_translate_post_spec.rb index 6807519f..80caeda5 100644 --- a/spec/jobs/detect_translate_post_spec.rb +++ b/spec/jobs/detect_translate_post_spec.rb @@ -68,4 +68,14 @@ expect { job.execute({ post_id: post.id }) }.not_to raise_error end + + it "skips public content when configured" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + post.topic.category.update!(read_restricted: true) + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never + DiscourseTranslator::PostTranslator.expects(:translate).never + + job.execute({ post_id: post.id }) + end end diff --git a/spec/jobs/detect_translate_topic_spec.rb b/spec/jobs/detect_translate_topic_spec.rb index d2830485..13ec06c2 100644 --- a/spec/jobs/detect_translate_topic_spec.rb +++ b/spec/jobs/detect_translate_topic_spec.rb @@ -68,4 +68,14 @@ expect { job.execute({ topic_id: topic.id }) }.not_to raise_error end + + it "skips public content when configured" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + topic.category.update!(read_restricted: true) + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never + DiscourseTranslator::TopicTranslator.expects(:translate).never + + job.execute({ topic_id: topic.id }) + end end diff --git a/spec/jobs/posts_locale_detection_backfill_spec.rb b/spec/jobs/posts_locale_detection_backfill_spec.rb index 227f88a8..50de56b0 100644 --- a/spec/jobs/posts_locale_detection_backfill_spec.rb +++ b/spec/jobs/posts_locale_detection_backfill_spec.rb @@ -76,4 +76,57 @@ job.execute({}) end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category) } + fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes posts from public categories" do + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) } + fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes posts within the age limit" do + # other posts + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + # other posts + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once + + job.execute({}) + end + end end diff --git a/spec/jobs/topics_locale_detection_backfill_spec.rb b/spec/jobs/topics_locale_detection_backfill_spec.rb index b0828d13..2d3d35b7 100644 --- a/spec/jobs/topics_locale_detection_backfill_spec.rb +++ b/spec/jobs/topics_locale_detection_backfill_spec.rb @@ -76,4 +76,59 @@ job.execute({}) end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:public_topic) { Fabricate(:topic, locale: nil) } + fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: nil) } + + before do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once + + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + end + + it "only processes topics from public categories" do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_topic) { Fabricate(:topic, locale: nil, created_at: 10.days.ago) } + fab!(:new_topic) { Fabricate(:topic, locale: nil, created_at: 2.days.ago) } + + before do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once + + SiteSetting.automatic_translation_backfill_max_age_days = 5 + end + + it "only processes topics within the age limit" do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once + + job.execute({}) + end + end end diff --git a/spec/jobs/translate_categories_spec.rb b/spec/jobs/translate_categories_spec.rb index dc50eeea..b83bbec0 100644 --- a/spec/jobs/translate_categories_spec.rb +++ b/spec/jobs/translate_categories_spec.rb @@ -131,4 +131,24 @@ def localize_all_categories(*locales) Jobs::TranslateCategories.send(:remove_const, :BATCH_SIZE) Jobs::TranslateCategories.const_set(:BATCH_SIZE, 50) end + + it "skips read-restricted categories when configured" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + + category1 = Fabricate(:category, name: "Public Category", read_restricted: false) + category2 = Fabricate(:category, name: "Private Category", read_restricted: true) + + DiscourseTranslator::CategoryTranslator.expects(:translate).at_least_once + + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category1, any_parameters) + .twice + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category2, any_parameters) + .never + + job.execute({}) + end end diff --git a/spec/jobs/translate_posts_spec.rb b/spec/jobs/translate_posts_spec.rb index 0c30df02..a44ba350 100644 --- a/spec/jobs/translate_posts_spec.rb +++ b/spec/jobs/translate_posts_spec.rb @@ -119,4 +119,71 @@ job.execute({}) end end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category) } + fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: "es") } + fab!(:public_post) { Fabricate(:post, locale: "es") } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes posts from public categories" do + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once + + DiscourseTranslator::PostTranslator + .expects(:translate) + .with(private_post, any_parameters) + .never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "de").once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_post) { Fabricate(:post, locale: "es", created_at: 10.days.ago) } + fab!(:new_post) { Fabricate(:post, locale: "es", created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes posts within the age limit" do + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, any_parameters).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "de").once + + job.execute({}) + end + end end diff --git a/spec/jobs/translate_topics_spec.rb b/spec/jobs/translate_topics_spec.rb index 318546b4..7135a909 100644 --- a/spec/jobs/translate_topics_spec.rb +++ b/spec/jobs/translate_topics_spec.rb @@ -119,4 +119,70 @@ job.execute({}) end end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: "es") } + fab!(:public_topic) { Fabricate(:topic, locale: "es") } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes topics from public categories" do + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once + + DiscourseTranslator::TopicTranslator + .expects(:translate) + .with(private_topic, any_parameters) + .never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "de").once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_topic) { Fabricate(:topic, locale: "es", created_at: 10.days.ago) } + fab!(:new_topic) { Fabricate(:topic, locale: "es", created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes topics within the age limit" do + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, any_parameters).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "de").once + + job.execute({}) + end + end end From 39b3816c94df33960f923b66dc4684b60d0d1f48 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 27 May 2025 14:57:01 +0800 Subject: [PATCH 2/3] no retry --- app/jobs/regular/detect_translate_post.rb | 10 +++++++++- app/jobs/regular/detect_translate_topic.rb | 10 +++++++++- app/jobs/regular/translate_categories.rb | 2 ++ app/jobs/scheduled/posts_locale_detection_backfill.rb | 1 + app/jobs/scheduled/topics_locale_detection_backfill.rb | 1 + 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/jobs/regular/detect_translate_post.rb b/app/jobs/regular/detect_translate_post.rb index d9662819..e43f7abe 100644 --- a/app/jobs/regular/detect_translate_post.rb +++ b/app/jobs/regular/detect_translate_post.rb @@ -2,6 +2,8 @@ module Jobs class DetectTranslatePost < ::Jobs::Base + sidekiq_options retry: false + def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation @@ -15,7 +17,13 @@ def execute(args) return if topic.blank? || topic.category&.read_restricted? end - detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) + begin + detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) + rescue FinalDestination::SSRFDetector::LookupFailedError + # this job is non-critical + # the backfill job will handle failures + return + end locales = SiteSetting.automatic_translation_target_languages.split("|") return if locales.blank? diff --git a/app/jobs/regular/detect_translate_topic.rb b/app/jobs/regular/detect_translate_topic.rb index 6dfb5aca..a93743e0 100644 --- a/app/jobs/regular/detect_translate_topic.rb +++ b/app/jobs/regular/detect_translate_topic.rb @@ -2,6 +2,8 @@ module Jobs class DetectTranslateTopic < ::Jobs::Base + sidekiq_options retry: false + def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation @@ -16,7 +18,13 @@ def execute(args) return if topic.category&.read_restricted? end - detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) + begin + detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) + rescue FinalDestination::SSRFDetector::LookupFailedError + # this job is non-critical + # the backfill job will handle failures + return + end locales = SiteSetting.automatic_translation_target_languages.split("|") return if locales.blank? diff --git a/app/jobs/regular/translate_categories.rb b/app/jobs/regular/translate_categories.rb index 2f6df4bd..d6745dbc 100644 --- a/app/jobs/regular/translate_categories.rb +++ b/app/jobs/regular/translate_categories.rb @@ -3,6 +3,8 @@ module Jobs class TranslateCategories < ::Jobs::Base cluster_concurrency 1 + sidekiq_options retry: false + BATCH_SIZE = 50 def execute(args) diff --git a/app/jobs/scheduled/posts_locale_detection_backfill.rb b/app/jobs/scheduled/posts_locale_detection_backfill.rb index 58c9e31f..7fbcb88b 100644 --- a/app/jobs/scheduled/posts_locale_detection_backfill.rb +++ b/app/jobs/scheduled/posts_locale_detection_backfill.rb @@ -3,6 +3,7 @@ module Jobs class PostsLocaleDetectionBackfill < ::Jobs::Scheduled every 5.minutes + sidekiq_options retry: false cluster_concurrency 1 def execute(args) diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb index de7f71a1..153c7322 100644 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ b/app/jobs/scheduled/topics_locale_detection_backfill.rb @@ -3,6 +3,7 @@ module Jobs class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled every 5.minutes + sidekiq_options retry: false cluster_concurrency 1 def execute(args) From 57ba8c0e1ebc9eb723b82b6ccb4f87b5311e526b Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 27 May 2025 15:50:12 +0800 Subject: [PATCH 3/3] do not pluck around!!!11! --- app/jobs/regular/translate_posts.rb | 11 ++++++----- app/jobs/regular/translate_topics.rb | 3 +-- .../scheduled/topics_locale_detection_backfill.rb | 5 ++--- spec/jobs/detect_translate_post_spec.rb | 2 +- spec/jobs/detect_translate_topic_spec.rb | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/app/jobs/regular/translate_posts.rb b/app/jobs/regular/translate_posts.rb index 6f28364e..13a3dfc7 100644 --- a/app/jobs/regular/translate_posts.rb +++ b/app/jobs/regular/translate_posts.rb @@ -30,12 +30,13 @@ def execute(args) .where("pl.id IS NULL") if SiteSetting.automatic_translation_backfill_limit_to_public_content - public_categories = Category.where(read_restricted: false).pluck(:id) posts = - posts - .joins(:topic) - .where(topics: { category_id: public_categories }) - .where(topics: { archetype: "regular" }) + posts.joins(:topic).where( + topics: { + category_id: Category.where(read_restricted: false).select(:id), + archetype: "regular", + }, + ) end if SiteSetting.automatic_translation_backfill_max_age_days > 0 diff --git a/app/jobs/regular/translate_topics.rb b/app/jobs/regular/translate_topics.rb index 37472188..adc718e7 100644 --- a/app/jobs/regular/translate_topics.rb +++ b/app/jobs/regular/translate_topics.rb @@ -29,8 +29,7 @@ def execute(args) .where("tl.id IS NULL") if SiteSetting.automatic_translation_backfill_limit_to_public_content - public_categories = Category.where(read_restricted: false).pluck(:id) - topics = topics.where(category_id: public_categories) + topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) end if SiteSetting.automatic_translation_backfill_max_age_days > 0 diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb index 153c7322..4b4875bc 100644 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ b/app/jobs/scheduled/topics_locale_detection_backfill.rb @@ -12,11 +12,10 @@ def execute(args) limit = SiteSetting.automatic_translation_backfill_rate return if limit == 0 - topics = Topic.where(locale: nil).where(deleted_at: nil).where("topics.user_id > 0") + topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0") if SiteSetting.automatic_translation_backfill_limit_to_public_content - public_categories = Category.where(read_restricted: false).pluck(:id) - topics = topics.where(category_id: public_categories) + topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) end if SiteSetting.automatic_translation_backfill_max_age_days > 0 diff --git a/spec/jobs/detect_translate_post_spec.rb b/spec/jobs/detect_translate_post_spec.rb index 80caeda5..20c58b4c 100644 --- a/spec/jobs/detect_translate_post_spec.rb +++ b/spec/jobs/detect_translate_post_spec.rb @@ -69,7 +69,7 @@ expect { job.execute({ post_id: post.id }) }.not_to raise_error end - it "skips public content when configured" do + it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do SiteSetting.automatic_translation_backfill_limit_to_public_content = true post.topic.category.update!(read_restricted: true) diff --git a/spec/jobs/detect_translate_topic_spec.rb b/spec/jobs/detect_translate_topic_spec.rb index 13ec06c2..01255fc1 100644 --- a/spec/jobs/detect_translate_topic_spec.rb +++ b/spec/jobs/detect_translate_topic_spec.rb @@ -69,7 +69,7 @@ expect { job.execute({ topic_id: topic.id }) }.not_to raise_error end - it "skips public content when configured" do + it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do SiteSetting.automatic_translation_backfill_limit_to_public_content = true topic.category.update!(read_restricted: true)