FEATURE: Translate topics on a schedule and on topic create

Author: nattsw

app/jobs/regular/detect_translate_topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+      return if args[:topic_id].blank?
+
+      topic = Topic.find(args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseTranslator::TopicTranslator.translate(topic, locale)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+    end
+  end
+end

app/jobs/regular/translate_topics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TranslateTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+            .limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseTranslator::TopicTranslator.translate(topic, locale)
+          rescue => e
+            Rails.logger.error(
+              "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseTranslator::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end

app/jobs/scheduled/automatic_category_translation.rb renamed to app/jobs/scheduled/category_translation_backfill.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Jobs
-  class AutomaticCategoryTranslation < ::Jobs::Scheduled
+  class CategoryTranslationBackfill < ::Jobs::Scheduled
     every 12.hours
     cluster_concurrency 1
 

app/jobs/scheduled/topic_translation_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicTranslationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      return if SiteSetting.automatic_translation_target_languages.blank?
+      return if SiteSetting.automatic_translation_backfill_rate == 0
+
+      Jobs.enqueue(:translate_topics, limit: SiteSetting.automatic_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+      return if SiteSetting.automatic_translation_backfill_rate == 0
+
+      limit = SiteSetting.automatic_translation_backfill_rate
+      topics =
+        Topic
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("topics.user_id > 0")
+          .order(updated_at: :desc)
+          .limit(limit)
+      return if topics.empty?
+
+      topics.each do |topic|
+        begin
+          DiscourseTranslator::TopicLocaleDetector.detect_locale(topic)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to detect topic #{topic.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseTranslator::VerboseLogger.log("Detected #{topics.size} topic locales")
+    end
+  end
+end

app/services/discourse_translator/topic_locale_detector.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicLocaleDetector
+    def self.detect_locale(topic)
+      return if topic.blank?
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      detected_locale = translator.detect!(topic)
+      locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
+      topic.update!(locale:)
+      locale
+    end
+  end
+end

app/services/discourse_translator/topic_translator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicTranslator
+    def self.translate(topic, target_locale = I18n.locale)
+      return if topic.blank? || target_locale.blank? || topic.locale == target_locale.to_s
+
+      target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      translated_title = translator.translate_topic!(topic, target_locale_sym)
+
+      localization =
+        TopicLocalization.find_or_initialize_by(topic_id: topic.id, locale: target_locale_sym.to_s)
+
+      localization.title = translated_title
+      localization.fancy_title = Topic.fancy_title(translated_title)
+      localization.localizer_user_id = Discourse.system_user.id
+      localization.save!
+      localization
+    end
+  end
+end

lib/discourse_translator/automatic_translations.rb

@@ -17,13 +17,23 @@ def inject(plugin)
         if translatable?(topic)
           Jobs.enqueue(:translate_translatable, type: "Topic", translatable_id: topic.id)
         end
+
+        if SiteSetting.experimental_content_localization
+          Jobs.enqueue(:detect_translate_topic, topic_id: topic.id)
+        end
       end
 
       plugin.on(:topic_edited) do |topic|
         if translatable?(topic)
           Jobs.enqueue(:translate_translatable, type: "Topic", translatable_id: topic.id)
         end
       end
+
+      plugin.on(:post_edited) do |post, topic_changed|
+        if SiteSetting.experimental_content_localization && topic_changed
+          Jobs.enqueue(:detect_translate_topic, topic_id: post.topic_id)
+        end
+      end
     end
 
     def translatable?(content)

spec/jobs/detect_translate_topic_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+describe Jobs::DetectTranslateTopic do
+  fab!(:topic)
+  subject(:job) { described_class.new }
+
+  let(:locales) { %w[en ja] }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 1
+    SiteSetting.automatic_translation_target_languages = locales.join("|")
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "detects locale" do
+    SiteSetting.translator_enabled = true
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once
+    DiscourseTranslator::TopicTranslator.expects(:translate).twice
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "skips bot topics" do
+    topic.update!(user: Discourse.system_user)
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "does not translate when no target languages are configured" do
+    SiteSetting.automatic_translation_target_languages = ""
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).never
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "skips translating to the topic's language" do
+    topic.update(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never
+    DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once
+
+    job.execute({ topic_id: topic.id })
+  end
+
+  it "handles translation errors gracefully" do
+    topic.update(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en")
+    DiscourseTranslator::TopicTranslator.expects(:translate).raises(StandardError.new("API error"))
+
+    expect { job.execute({ topic_id: topic.id }) }.not_to raise_error
+  end
+end

spec/jobs/topics_locale_detection_backfill_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+describe Jobs::TopicsLocaleDetectionBackfill do
+  fab!(:topic) { Fabricate(:topic, locale: nil) }
+  subject(:job) { described_class.new }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 100
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when there are no topics to detect" do
+    Topic.update_all(locale: "en")
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "detects locale for topics with nil locale" do
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once
+    job.execute({})
+  end
+
+  it "detects most recently updated topics first" do
+    topic_2 = Fabricate(:topic, locale: nil)
+    topic_3 = Fabricate(:topic, locale: nil)
+
+    topic.update!(updated_at: 3.days.ago)
+    topic_2.update!(updated_at: 2.day.ago)
+    topic_3.update!(updated_at: 4.day.ago)
+
+    SiteSetting.automatic_translation_backfill_rate = 1
+
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_2).once
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_3).never
+
+    job.execute({})
+  end
+
+  it "skips bot topics" do
+    topic.update!(user: Discourse.system_user)
+    DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never
+
+    job.execute({})
+  end
+
+  it "handles detection errors gracefully" do
+    DiscourseTranslator::TopicLocaleDetector
+      .expects(:detect_locale)
+      .with(topic)
+      .raises(StandardError.new("jiboomz"))
+      .once
+
+    expect { job.execute({}) }.not_to raise_error
+  end
+
+  it "logs a summary after running" do
+    DiscourseTranslator::TopicLocaleDetector.stubs(:detect_locale)
+    DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 topic locales"))
+
+    job.execute({})
+  end
+end