FEATURE: Translate topics on a schedule and on topic create (#291)

This PR adds the functionality to translate topics using the selected provider. We are keeping the topic translation feature separate from the post translation on purpose, despite many things being the same. This allows easy modification per model and less coupling.

Author: nattsw

app/jobs/regular/detect_translate_topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+      return if args[:topic_id].blank?
+
+      topic = Topic.find(args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseTranslator::TopicTranslator.translate(topic, locale)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+    end
+  end
+end

app/jobs/regular/translate_topics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TranslateTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+            .limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseTranslator::TopicTranslator.translate(topic, locale)
+          rescue => e
+            Rails.logger.error(
+              "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseTranslator::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end

app/jobs/scheduled/automatic_category_translation.rb renamed to app/jobs/scheduled/category_translation_backfill.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Jobs
-  class AutomaticCategoryTranslation < ::Jobs::Scheduled
+  class CategoryTranslationBackfill < ::Jobs::Scheduled
     every 12.hours
     cluster_concurrency 1
 

app/jobs/scheduled/topic_translation_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicTranslationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      return if SiteSetting.automatic_translation_target_languages.blank?
+      return if SiteSetting.automatic_translation_backfill_rate == 0
+
+      Jobs.enqueue(:translate_topics, limit: SiteSetting.automatic_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+      return if SiteSetting.automatic_translation_backfill_rate == 0
+
+      limit = SiteSetting.automatic_translation_backfill_rate
+      topics =
+        Topic
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("topics.user_id > 0")
+          .order(updated_at: :desc)
+          .limit(limit)
+      return if topics.empty?
+
+      topics.each do |topic|
+        begin
+          DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to detect topic #{topic.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseTranslator::VerboseLogger.log("Detected #{topics.size} topic locales")
+    end
+  end
+end

app/services/discourse_translator/topic_locale_detector.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicLocaleDetector
+    def self.detect_locale(topic)
+      return if topic.blank?
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      detected_locale = translator.detect!(topic)
+      locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
+      topic.update!(locale:)
+      locale
+    end
+  end
+end

app/services/discourse_translator/topic_translator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicTranslator
+    def self.translate(topic, target_locale = I18n.locale)
+      return if topic.blank? || target_locale.blank? || topic.locale == target_locale.to_s
+
+      target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      translated_title = translator.translate_topic!(topic, target_locale_sym)
+
+      localization =
+        TopicLocalization.find_or_initialize_by(topic_id: topic.id, locale: target_locale_sym.to_s)
+
+      localization.title = translated_title
+      localization.fancy_title = Topic.fancy_title(translated_title)
+      localization.localizer_user_id = Discourse.system_user.id
+      localization.save!
+      localization
+    end
+  end
+end

lib/discourse_translator/automatic_translations.rb

@@ -17,13 +17,23 @@ def inject(plugin)
         if translatable?(topic)
           Jobs.enqueue(:translate_translatable, type: "Topic", translatable_id: topic.id)
         end
+
+        if SiteSetting.experimental_content_localization
+          Jobs.enqueue(:detect_translate_topic, topic_id: topic.id)
+        end
       end
 
       plugin.on(:topic_edited) do |topic|
         if translatable?(topic)
           Jobs.enqueue(:translate_translatable, type: "Topic", translatable_id: topic.id)
         end
       end
+
+      plugin.on(:post_edited) do |post, topic_changed|
+        if SiteSetting.experimental_content_localization && topic_changed
+          Jobs.enqueue(:detect_translate_topic, topic_id: post.topic_id)
+        end
+      end
     end
 
     def translatable?(content)

spec/jobs/detect_translate_topic_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+describe Jobs::DetectTranslateTopic do
+  fab!(:topic)
+  subject(:job) { described_class.new }
+
+  let(:locales) { %w[en ja] }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 1
+    SiteSetting.automatic_translation_target_languages = locales.join("|")
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "detects locale" do
+    SiteSetting.translator_enabled = true
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once
+    DiscourseTranslator::TopicTranslator.expects(:translate).twice
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "skips bot topics" do
+    topic.update!(user: Discourse.system_user)
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "does not translate when no target languages are configured" do
+    SiteSetting.automatic_translation_target_languages = ""
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "skips translating to the topic's language" do
+    topic.update(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never
+    DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "handles translation errors gracefully" do
+    topic.update(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).raises(StandardError.new("API error"))
+
+    expect { job.execute({ topic_id: topic.id }) }.not_to raise_error
+  end
+end

spec/jobs/topics_locale_detection_backfill_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+describe Jobs::TopicsLocaleDetectionBackfill do
+  fab!(:topic) { Fabricate(:topic, locale: nil) }
+  subject(:job) { described_class.new }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 100
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when there are no topics to detect" do
+    Topic.update_all(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "detects locale for topics with nil locale" do
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once
+    job.execute({})
+  end
+
+  it "detects most recently updated topics first" do
+    topic_2 = Fabricate(:topic, locale: nil)
+    topic_3 = Fabricate(:topic, locale: nil)
+
+    topic.update!(updated_at: 3.days.ago)
+    topic_2.update!(updated_at: 2.day.ago)
+    topic_3.update!(updated_at: 4.day.ago)
+
+    SiteSetting.automatic_translation_backfill_rate = 1
+
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_2).once
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_3).never
+
+    job.execute({})
+  end
+
+  it "skips bot topics" do
+    topic.update!(user: Discourse.system_user)
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never
+
+    job.execute({})
+  end
+
+  it "handles detection errors gracefully" do
+    DiscourseTranslator::TopicLocaleDetector
+      .expects(:detect_locale)
+      .with(topic)
+      .raises(StandardError.new("jiboomz"))
+      .once
+
+    expect { job.execute({}) }.not_to raise_error
+  end
+
+  it "logs a summary after running" do
+    DiscourseTranslator::TopicLocaleDetector.stubs(:detect_locale)
+    DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 topic locales"))
+
+    job.execute({})
+  end
+end