FEATURE: Automatically detect and translate posts into Post and PostLocalization (#290)

With the introduction of Post.locale and PostLocalization in core, we want to use this plugin's translator provider to automatically detect and translate posts and save it into those tables.

Author: nattsw

app/jobs/regular/detect_translate_post.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslatePost < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+      return if args[:post_id].blank?
+
+      post = Post.find(args[:post_id])
+      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
+
+      detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseTranslator::PostTranslator.translate(post, locale)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to translate post #{post.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+    end
+  end
+end

app/jobs/regular/translate_posts.rb

@@ -14,6 +14,8 @@ def execute(args)
       locales = SiteSetting.automatic_translation_target_languages.split("|")
       return if locales.blank?
 
+      limit = args[:limit] || BATCH_SIZE
+
       locales.each do |locale|
         posts =
           Post
@@ -26,7 +28,7 @@ def execute(args)
             .where.not(locale: nil)
             .where.not(locale: locale)
             .where("pl.id IS NULL")
-            .limit(BATCH_SIZE)
+            .limit(limit)
 
         next if posts.empty?
 

app/jobs/scheduled/post_translation_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostTranslationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      return if SiteSetting.automatic_translation_target_languages.blank?
+      return if SiteSetting.automatic_translation_backfill_rate == 0
+
+      Jobs.enqueue(:translate_posts, limit: SiteSetting.automatic_translation_backfill_rate)
+    end
+  end
+end

lib/discourse_translator/automatic_translations.rb

@@ -7,6 +7,10 @@ def inject(plugin)
         if translatable?(post)
           Jobs.enqueue(:translate_translatable, type: "Post", translatable_id: post.id)
         end
+
+        if SiteSetting.experimental_content_localization
+          Jobs.enqueue(:detect_translate_post, post_id: post.id)
+        end
       end
 
       plugin.on(:topic_created) do |topic|

spec/jobs/detect_translate_post_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+describe Jobs::DetectTranslatePost do
+  fab!(:post)
+  subject(:job) { described_class.new }
+
+  let(:locales) { %w[en ja] }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 1
+    SiteSetting.automatic_translation_target_languages = locales.join("|")
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "detects locale" do
+    SiteSetting.translator_enabled = true
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+    DiscourseTranslator::PostTranslator.expects(:translate).twice
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "skips bot posts" do
+    post.update!(user: Discourse.system_user)
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "does not translate when no target languages are configured" do
+    SiteSetting.automatic_translation_target_languages = ""
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en")
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "skips translating to the post's language" do
+    post.update(locale: "en")
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en")
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once
+
+    job.execute({ post_id: post.id })
+  end
+
+  it "handles translation errors gracefully" do
+    post.update(locale: "en")
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en")
+    DiscourseTranslator::PostTranslator.expects(:translate).raises(StandardError.new("API error"))
+
+    expect { job.execute({ post_id: post.id }) }.not_to raise_error
+  end
+end

spec/jobs/post_translation_backfill_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+describe Jobs::PostTranslationBackfill do
+  before do
+    SiteSetting.automatic_translation_backfill_rate = 100
+    SiteSetting.automatic_translation_target_languages = "en"
+  end
+
+  it "does not enqueue post translation when translator disabled" do
+    SiteSetting.translator_enabled = false
+
+    described_class.new.execute({})
+
+    expect_not_enqueued_with(job: :translate_posts)
+  end
+
+  it "does not enqueue post translation when experimental translation disabled" do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = false
+
+    described_class.new.execute({})
+
+    expect_not_enqueued_with(job: :translate_posts)
+  end
+
+  it "does not enqueue psot translation if backfill languages are not set" do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_target_languages = ""
+
+    described_class.new.execute({})
+
+    expect_not_enqueued_with(job: :translate_posts)
+  end
+
+  it "does not enqueue psot translation if backfill limit is set to 0" do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 0
+
+    described_class.new.execute({})
+
+    expect_not_enqueued_with(job: :translate_posts)
+  end
+
+  it "enqueues post translation with correct limit" do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 10
+
+    described_class.new.execute({})
+
+    expect_job_enqueued(job: :translate_posts, args: { limit: 10 })
+  end
+end

spec/lib/automatic_translation_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+describe DiscourseTranslator::AutomaticTranslations do
+  before { SiteSetting.translator_enabled = true }
+
+  describe "upon post process cooked" do
+    it "enqueues detect post locale and translate post job" do
+      SiteSetting.experimental_content_localization = true
+      post = Fabricate(:post)
+      CookedPostProcessor.new(post).post_process
+
+      expect_job_enqueued(job: :detect_translate_post, args: { post_id: post.id })
+    end
+
+    it "does not enqueue if setting disabled" do
+      SiteSetting.experimental_content_localization = false
+      post = Fabricate(:post)
+      CookedPostProcessor.new(post).post_process
+
+      expect(job_enqueued?(job: :detect_translate_post, args: { post_id: post.id })).to eq false
+    end
+  end
+end