diff --git a/app/jobs/regular/detect_translate_post.rb b/app/jobs/regular/detect_translate_post.rb index cf5cacf4..e43f7abe 100644 --- a/app/jobs/regular/detect_translate_post.rb +++ b/app/jobs/regular/detect_translate_post.rb @@ -2,6 +2,8 @@ module Jobs class DetectTranslatePost < ::Jobs::Base + sidekiq_options retry: false + def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation @@ -10,7 +12,18 @@ def execute(args) post = Post.find_by(id: args[:post_id]) return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0 - detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) + if SiteSetting.automatic_translation_backfill_limit_to_public_content + topic = post.topic + return if topic.blank? || topic.category&.read_restricted? + end + + begin + detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) + rescue FinalDestination::SSRFDetector::LookupFailedError + # this job is non-critical + # the backfill job will handle failures + return + end locales = SiteSetting.automatic_translation_target_languages.split("|") return if locales.blank? diff --git a/app/jobs/regular/detect_translate_topic.rb b/app/jobs/regular/detect_translate_topic.rb index e4a4305b..a93743e0 100644 --- a/app/jobs/regular/detect_translate_topic.rb +++ b/app/jobs/regular/detect_translate_topic.rb @@ -2,6 +2,8 @@ module Jobs class DetectTranslateTopic < ::Jobs::Base + sidekiq_options retry: false + def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation @@ -12,7 +14,17 @@ def execute(args) return end - detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) + if SiteSetting.automatic_translation_backfill_limit_to_public_content + return if topic.category&.read_restricted? + end + + begin + detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) + rescue FinalDestination::SSRFDetector::LookupFailedError + # this job is non-critical + # the backfill job will handle failures + return + end locales = SiteSetting.automatic_translation_target_languages.split("|") return if locales.blank? diff --git a/app/jobs/regular/translate_categories.rb b/app/jobs/regular/translate_categories.rb index f79e5ff7..d6745dbc 100644 --- a/app/jobs/regular/translate_categories.rb +++ b/app/jobs/regular/translate_categories.rb @@ -3,6 +3,8 @@ module Jobs class TranslateCategories < ::Jobs::Base cluster_concurrency 1 + sidekiq_options retry: false + BATCH_SIZE = 50 def execute(args) @@ -21,6 +23,12 @@ def execute(args) return if categories.empty? categories.each do |category| + if SiteSetting.automatic_translation_backfill_limit_to_public_content && + category.read_restricted? + last_id = category.id + next + end + CategoryLocalization.transaction do locales.each do |locale| next if CategoryLocalization.exists?(category_id: category.id, locale: locale) diff --git a/app/jobs/regular/translate_posts.rb b/app/jobs/regular/translate_posts.rb index 21806c50..13a3dfc7 100644 --- a/app/jobs/regular/translate_posts.rb +++ b/app/jobs/regular/translate_posts.rb @@ -28,7 +28,26 @@ def execute(args) .where.not(locale: nil) .where.not(locale: locale) .where("pl.id IS NULL") - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + posts = + posts.joins(:topic).where( + topics: { + category_id: Category.where(read_restricted: false).select(:id), + archetype: "regular", + }, + ) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + posts = + posts.where( + "posts.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + posts = posts.order(updated_at: :desc).limit(limit) next if posts.empty? diff --git a/app/jobs/regular/translate_topics.rb b/app/jobs/regular/translate_topics.rb index a3775b96..adc718e7 100644 --- a/app/jobs/regular/translate_topics.rb +++ b/app/jobs/regular/translate_topics.rb @@ -27,7 +27,20 @@ def execute(args) .where.not(locale: nil) .where.not(locale: locale) .where("tl.id IS NULL") - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + topics = + topics.where( + "topics.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + topics = topics.order(updated_at: :desc).limit(limit) next if topics.empty? diff --git a/app/jobs/scheduled/posts_locale_detection_backfill.rb b/app/jobs/scheduled/posts_locale_detection_backfill.rb index 61b90a13..7fbcb88b 100644 --- a/app/jobs/scheduled/posts_locale_detection_backfill.rb +++ b/app/jobs/scheduled/posts_locale_detection_backfill.rb @@ -3,6 +3,7 @@ module Jobs class PostsLocaleDetectionBackfill < ::Jobs::Scheduled every 5.minutes + sidekiq_options retry: false cluster_concurrency 1 def execute(args) @@ -10,15 +11,31 @@ def execute(args) return unless SiteSetting.experimental_content_translation return if SiteSetting.automatic_translation_backfill_rate == 0 - limit = SiteSetting.automatic_translation_backfill_rate posts = Post .where(locale: nil) .where(deleted_at: nil) .where("posts.user_id > 0") .where.not(raw: [nil, ""]) - .order(updated_at: :desc) - .limit(limit) + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + public_categories = Category.where(read_restricted: false).pluck(:id) + posts = + posts + .joins(:topic) + .where(topics: { category_id: public_categories }) + .where(topics: { archetype: "regular" }) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + posts = + posts.where( + "posts.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate) return if posts.empty? posts.each do |post| diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb index cbd96bc4..4b4875bc 100644 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ b/app/jobs/scheduled/topics_locale_detection_backfill.rb @@ -3,21 +3,30 @@ module Jobs class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled every 5.minutes + sidekiq_options retry: false cluster_concurrency 1 def execute(args) return unless SiteSetting.translator_enabled return unless SiteSetting.experimental_content_translation - return if SiteSetting.automatic_translation_backfill_rate == 0 - limit = SiteSetting.automatic_translation_backfill_rate - topics = - Topic - .where(locale: nil) - .where(deleted_at: nil) - .where("topics.user_id > 0") - .order(updated_at: :desc) - .limit(limit) + return if limit == 0 + + topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0") + + if SiteSetting.automatic_translation_backfill_limit_to_public_content + topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) + end + + if SiteSetting.automatic_translation_backfill_max_age_days > 0 + topics = + topics.where( + "topics.created_at > ?", + SiteSetting.automatic_translation_backfill_max_age_days.days.ago, + ) + end + + topics = topics.order(updated_at: :desc).limit(limit) return if topics.empty? topics.each do |topic| diff --git a/spec/jobs/detect_translate_post_spec.rb b/spec/jobs/detect_translate_post_spec.rb index 6807519f..20c58b4c 100644 --- a/spec/jobs/detect_translate_post_spec.rb +++ b/spec/jobs/detect_translate_post_spec.rb @@ -68,4 +68,14 @@ expect { job.execute({ post_id: post.id }) }.not_to raise_error end + + it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + post.topic.category.update!(read_restricted: true) + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never + DiscourseTranslator::PostTranslator.expects(:translate).never + + job.execute({ post_id: post.id }) + end end diff --git a/spec/jobs/detect_translate_topic_spec.rb b/spec/jobs/detect_translate_topic_spec.rb index d2830485..01255fc1 100644 --- a/spec/jobs/detect_translate_topic_spec.rb +++ b/spec/jobs/detect_translate_topic_spec.rb @@ -68,4 +68,14 @@ expect { job.execute({ topic_id: topic.id }) }.not_to raise_error end + + it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + topic.category.update!(read_restricted: true) + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never + DiscourseTranslator::TopicTranslator.expects(:translate).never + + job.execute({ topic_id: topic.id }) + end end diff --git a/spec/jobs/posts_locale_detection_backfill_spec.rb b/spec/jobs/posts_locale_detection_backfill_spec.rb index 227f88a8..50de56b0 100644 --- a/spec/jobs/posts_locale_detection_backfill_spec.rb +++ b/spec/jobs/posts_locale_detection_backfill_spec.rb @@ -76,4 +76,57 @@ job.execute({}) end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category) } + fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes posts from public categories" do + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) } + fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes posts within the age limit" do + # other posts + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + # other posts + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once + + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once + DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once + + job.execute({}) + end + end end diff --git a/spec/jobs/topics_locale_detection_backfill_spec.rb b/spec/jobs/topics_locale_detection_backfill_spec.rb index b0828d13..2d3d35b7 100644 --- a/spec/jobs/topics_locale_detection_backfill_spec.rb +++ b/spec/jobs/topics_locale_detection_backfill_spec.rb @@ -76,4 +76,59 @@ job.execute({}) end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:public_topic) { Fabricate(:topic, locale: nil) } + fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: nil) } + + before do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once + + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + end + + it "only processes topics from public categories" do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_topic) { Fabricate(:topic, locale: nil, created_at: 10.days.ago) } + fab!(:new_topic) { Fabricate(:topic, locale: nil, created_at: 2.days.ago) } + + before do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once + + SiteSetting.automatic_translation_backfill_max_age_days = 5 + end + + it "only processes topics within the age limit" do + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once + DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once + + job.execute({}) + end + end end diff --git a/spec/jobs/translate_categories_spec.rb b/spec/jobs/translate_categories_spec.rb index dc50eeea..b83bbec0 100644 --- a/spec/jobs/translate_categories_spec.rb +++ b/spec/jobs/translate_categories_spec.rb @@ -131,4 +131,24 @@ def localize_all_categories(*locales) Jobs::TranslateCategories.send(:remove_const, :BATCH_SIZE) Jobs::TranslateCategories.const_set(:BATCH_SIZE, 50) end + + it "skips read-restricted categories when configured" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = true + + category1 = Fabricate(:category, name: "Public Category", read_restricted: false) + category2 = Fabricate(:category, name: "Private Category", read_restricted: true) + + DiscourseTranslator::CategoryTranslator.expects(:translate).at_least_once + + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category1, any_parameters) + .twice + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category2, any_parameters) + .never + + job.execute({}) + end end diff --git a/spec/jobs/translate_posts_spec.rb b/spec/jobs/translate_posts_spec.rb index 0c30df02..a44ba350 100644 --- a/spec/jobs/translate_posts_spec.rb +++ b/spec/jobs/translate_posts_spec.rb @@ -119,4 +119,71 @@ job.execute({}) end end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category) } + fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: "es") } + fab!(:public_post) { Fabricate(:post, locale: "es") } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes posts from public categories" do + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once + + DiscourseTranslator::PostTranslator + .expects(:translate) + .with(private_post, any_parameters) + .never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "de").once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_post) { Fabricate(:post, locale: "es", created_at: 10.days.ago) } + fab!(:new_post) { Fabricate(:post, locale: "es", created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes posts within the age limit" do + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, any_parameters).never + + job.execute({}) + end + + it "processes all posts when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once + + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "en").once + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "ja").once + DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "de").once + + job.execute({}) + end + end end diff --git a/spec/jobs/translate_topics_spec.rb b/spec/jobs/translate_topics_spec.rb index 318546b4..7135a909 100644 --- a/spec/jobs/translate_topics_spec.rb +++ b/spec/jobs/translate_topics_spec.rb @@ -119,4 +119,70 @@ job.execute({}) end end + + describe "with public content limitation" do + fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } + fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: "es") } + fab!(:public_topic) { Fabricate(:topic, locale: "es") } + + before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } + + it "only processes topics from public categories" do + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once + + DiscourseTranslator::TopicTranslator + .expects(:translate) + .with(private_topic, any_parameters) + .never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_limit_to_public_content = false + + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "de").once + + job.execute({}) + end + end + + describe "with max age limit" do + fab!(:old_topic) { Fabricate(:topic, locale: "es", created_at: 10.days.ago) } + fab!(:new_topic) { Fabricate(:topic, locale: "es", created_at: 2.days.ago) } + + before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } + + it "only processes topics within the age limit" do + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, any_parameters).never + + job.execute({}) + end + + it "processes all topics when setting is disabled" do + SiteSetting.automatic_translation_backfill_max_age_days = 0 + + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once + + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "en").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "ja").once + DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "de").once + + job.execute({}) + end + end end