FEATURE: Detect locale and translate posts from core table (#287)

This PR adds two jobs that fill in `post.locale` and adds `PostLocalization` for each defined locale in `SiteSetting.automatic_translation_target_languages`.

It also creates dedicated `DiscourseTranslator::PostTranslator` and `DiscourseTranslator::PostLocaleDetector` to serve this purpose. We should be moving away from dealing with `translatables` in the near future.

Author: nattsw

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta4-dev: 14ca3c07efa0a80712a4cbb8ca455c32a727adec
 < 3.5.0.beta2-dev: 5f24835801fdc7cb98e1bcf42d2ab2e49e609921
 < 3.5.0.beta1-dev: 7d411e458bdd449f8aead2bc07cedeb00b856798
 < 3.4.0.beta3-dev: b4cf3a065884816fa3f770248c2bf908ba65d8ac

app/jobs/regular/detect_posts_locale.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectPostsLocale < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      posts =
+        Post
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+          .order(updated_at: :desc)
+          .limit(BATCH_SIZE)
+      return if posts.empty?
+
+      posts.each do |post|
+        begin
+          DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to detect post #{post.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseTranslator::VerboseLogger.log("Detected #{posts.size} post locales")
+    end
+  end
+end

app/jobs/regular/translate_posts.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TranslatePosts < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("posts.user_id > 0")
+            .where.not(raw: [nil, ""])
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("pl.id IS NULL")
+            .limit(BATCH_SIZE)
+
+        next if posts.empty?
+
+        posts.each do |post|
+          begin
+            DiscourseTranslator::PostTranslator.translate(post, locale)
+          rescue => e
+            Rails.logger.error(
+              "Discourse Translator: Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseTranslator::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+      end
+    end
+  end
+end

app/services/discourse_translator/post_locale_detector.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostLocaleDetector
+    def self.detect_locale(post)
+      return if post.blank?
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      detected_locale = translator.detect!(post)
+      post.update!(locale: detected_locale)
+      detected_locale
+    end
+  end
+end

app/services/discourse_translator/post_translator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostTranslator
+    def self.translate(post, target_locale = I18n.locale)
+      return if post.blank? || target_locale.blank? || post.locale == target_locale.to_s
+
+      target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      translated_raw = translator.translate_post!(post, target_locale_sym)
+
+      localization =
+        PostLocalization.find_or_initialize_by(post_id: post.id, locale: target_locale_sym.to_s)
+
+      localization.raw = translated_raw
+      localization.cooked = PrettyText.cook(translated_raw)
+      localization.post_version = post.version
+      localization.localizer_user_id = Discourse.system_user.id
+      localization.save!
+      localization
+    end
+  end
+end

app/services/discourse_translator/provider/base_provider.rb

@@ -55,12 +55,7 @@ def self.translate(translatable, target_locale_sym = I18n.locale)
         [detected_lang, translated]
       end
 
-      # Subclasses must implement this method to translate the text of a
-      # post or topic and return only the translated text.
-      # Subclasses should use text_for_translation
-      # @param translatable [Post|Topic]
-      # @param target_locale_sym [Symbol]
-      # @return [String]
+      # TODO: Deprecate this in favour of translate_<model>
       def self.translate_translatable!(translatable, target_locale_sym = I18n.locale)
         raise "Not Implemented"
       end
@@ -69,6 +64,14 @@ def self.translate_text!(text, target_locale_sym = I18n.locale)
         raise "Not Implemented"
       end
 
+      def self.translate_post!(post, target_locale_sym = I18n.locale)
+        translate_translatable!(post, target_locale_sym)
+      end
+
+      def self.translate_topic!(topic, target_locale_sym = I18n.locale)
+        translate_translatable!(topic, target_locale_sym)
+      end
+
       # Returns the stored detected locale of a post or topic.
       # If the locale does not exist yet, it will be detected first via the API then stored.
       # @param translatable [Post|Topic]

app/services/discourse_translator/provider/discourse_ai.rb

@@ -16,25 +16,32 @@ def self.detect!(topic_or_post)
       end
 
       def self.translate_translatable!(translatable, target_locale_sym = I18n.locale)
+        if (translatable.class.name == "Post")
+          translate_post!(translatable, target_locale_sym)
+        elsif (translatable.class.name == "Topic")
+          translate_topic!(translatable, target_locale_sym)
+        end
+      end
+
+      def self.translate_post!(post, target_locale_sym = I18n.locale)
         validate_required_settings!
 
-        language = get_language_name(target_locale_sym)
+        text = text_for_translation(post, raw: true)
+        chunks = DiscourseTranslator::ContentSplitter.split(text)
         translated =
-          case translatable.class.name
-          when "Post"
-            text = text_for_translation(translatable, raw: true)
-            chunks = DiscourseTranslator::ContentSplitter.split(text)
-            chunks
-              .map { |chunk| ::DiscourseAi::PostTranslator.new(chunk, target_locale_sym).translate }
-              .join("")
-          when "Topic"
-            ::DiscourseAi::TopicTranslator.new(
-              text_for_translation(translatable),
-              language,
-            ).translate
-          end
+          chunks
+            .map { |chunk| ::DiscourseAi::PostTranslator.new(chunk, target_locale_sym).translate }
+            .join("")
+        DiscourseTranslator::TranslatedContentNormalizer.normalize(post, translated)
+      end
 
-        DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated)
+      def self.translate_topic!(topic, target_locale_sym = I18n.locale)
+        validate_required_settings!
+
+        language = get_language_name(target_locale_sym)
+        translated =
+          ::DiscourseAi::TopicTranslator.new(text_for_translation(topic), language).translate
+        DiscourseTranslator::TranslatedContentNormalizer.normalize(topic, translated)
       end
 
       def self.translate_text!(text, target_locale_sym = I18n.locale)

spec/jobs/detect_posts_locale_spec.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+describe Jobs::DetectPostsLocale do
+  fab!(:post) { Fabricate(:post, locale: nil) }
+  subject(:job) { described_class.new }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when there are no posts to detect" do
+    Post.update_all(locale: "en")
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "detects locale for posts with nil locale" do
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+    job.execute({})
+  end
+
+  it "detects most recently updated posts first" do
+    post_2 = Fabricate(:post, locale: nil)
+    post_3 = Fabricate(:post, locale: nil)
+
+    post.update!(updated_at: 3.days.ago)
+    post_2.update!(updated_at: 2.day.ago)
+    post_3.update!(updated_at: 4.day.ago)
+
+    original_batch = described_class::BATCH_SIZE
+    described_class.const_set(:BATCH_SIZE, 1)
+
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post_2).once
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post_3).never
+
+    job.execute({})
+  ensure
+    described_class.const_set(:BATCH_SIZE, original_batch)
+  end
+
+  it "skips bot posts" do
+    post.update!(user: Discourse.system_user)
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+
+    job.execute({})
+  end
+
+  it "handles detection errors gracefully" do
+    DiscourseTranslator::PostLocaleDetector
+      .expects(:detect_locale)
+      .with(post)
+      .raises(StandardError.new("jiboomz"))
+      .once
+
+    expect { job.execute({}) }.not_to raise_error
+  end
+
+  it "logs a summary after running" do
+    DiscourseTranslator::PostLocaleDetector.stubs(:detect_locale)
+    DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 post locales"))
+
+    job.execute({})
+  end
+end