FIX: Ignore captions and quotes when detecting locale

Author: nattsw

lib/translation/post_detection_text.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class PostDetectionText
+      def self.get_text(post)
+        return if post.blank?
+        cooked = post.cooked
+        return if cooked.blank?
+
+        doc = Nokogiri::HTML.fragment(cooked)
+        original = doc.text.strip
+
+        # quotes and blockquotes
+        doc.css("blockquote, aside.quote").remove
+        # image captions
+        doc.css(".lightbox-wrapper").remove
+
+        necessary = doc.text.strip
+
+        # oneboxes (external content)
+        doc.css("aside.onebox").remove
+        # code blocks
+        doc.css("code, pre").remove
+        # hashtags
+        doc.css("a.hashtag-cooked").remove
+        # emoji
+        doc.css("img.emoji").remove
+        # mentions
+        doc.css("a.mention").remove
+
+        preferred = doc.text.strip
+
+        return preferred if preferred.present?
+        return necessary if necessary.present?
+        original
+      end
+    end
+  end
+end

lib/translation/post_locale_detector.rb

@@ -6,7 +6,8 @@ class PostLocaleDetector
       def self.detect_locale(post)
         return if post.blank?
 
-        detected_locale = LanguageDetector.new(post.raw).detect
+        text = PostDetectionText.get_text(post)
+        detected_locale = LanguageDetector.new(text).detect
         locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
         post.update_column(:locale, locale)
         locale

lib/translation/topic_locale_detector.rb

@@ -6,10 +6,7 @@ class TopicLocaleDetector
       def self.detect_locale(topic)
         return if topic.blank?
 
-        text = topic.title.dup
-        text << " #{topic.first_post.raw}" if topic.first_post.raw
-
-        detected_locale = LanguageDetector.new(text).detect
+        detected_locale = LanguageDetector.new(topic.title.dup).detect
         locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
         topic.update_column(:locale, locale)
         locale

spec/lib/translation/post_detection_text_spec.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+describe DiscourseAi::Translation::PostDetectionText do
+  describe ".get_text" do
+    let(:post) { Fabricate.build(:post) }
+
+    it "returns nil when post is nil" do
+      expect(described_class.get_text(nil)).to be_nil
+    end
+
+    it "returns nil when post.cooked is nil" do
+      post.cooked = nil
+      expect(described_class.get_text(post)).to be_nil
+    end
+
+    it "handles simple text" do
+      post.cooked = "<p>Hello world</p>"
+      expect(described_class.get_text(post)).to eq("Hello world")
+    end
+
+    it "removes quotes" do
+      post.cooked = "<p>Hello </p><blockquote><p>Quote</p></blockquote><p>World</p>"
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes Discourse quotes" do
+      post.cooked = '<p>Hello </p><aside class="quote"><p>Quote</p></aside><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes image captions" do
+      post.cooked = '<p>Hello </p><div class="lightbox-wrapper">Caption text</div><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes oneboxes" do
+      post.cooked = '<p>Hello </p><aside class="onebox">Onebox content</aside><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes code blocks" do
+      post.cooked = "<p>Hello </p><pre><code>Code block</code></pre><p>World</p>"
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes hashtags" do
+      post.cooked = '<p>Hello </p><a class="hashtag-cooked">#hashtag</a><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes emoji" do
+      post.cooked = '<p>Hello </p><img class="emoji" alt=":smile:" title=":smile:"><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "removes mentions" do
+      post.cooked = '<p>Hello </p><a class="mention">@user</a><p>World</p>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+
+    it "falls back to necessary text when preferred is empty" do
+      post.cooked = '<aside class="quote">Quote</aside><a class="mention">@user</a>'
+      expect(described_class.get_text(post)).to eq("@user")
+    end
+
+    it "falls back to cooked when all filtering removes all content" do
+      post.cooked = "<blockquote>Quote</blockquote>"
+      expect(described_class.get_text(post)).to eq("Quote")
+    end
+
+    it "handles complex nested content correctly" do
+      post.cooked =
+        '<p>Hello </p><div class="lightbox-wrapper"><p>Image caption</p><img src="test.jpg"></div><blockquote><p>Quote text</p></blockquote><p>World</p><pre><code>Code block</code></pre><a class="mention">@user</a>'
+      expect(described_class.get_text(post)).to eq("Hello World")
+    end
+  end
+end

spec/lib/translation/post_locale_detector_spec.rb

@@ -2,7 +2,7 @@
 
 describe DiscourseAi::Translation::PostLocaleDetector do
   describe ".detect_locale" do
-    fab!(:post) { Fabricate(:post, raw: "Hello world", locale: nil) }
+    fab!(:post) { Fabricate(:post, cooked: "Hello world", locale: nil) }
 
     def language_detector_stub(opts)
       mock = instance_double(DiscourseAi::Translation::LanguageDetector)
@@ -17,16 +17,16 @@ def language_detector_stub(opts)
     end
 
     it "updates the post locale with the detected locale" do
-      language_detector_stub({ text: post.raw, locale: "zh_CN" })
+      language_detector_stub({ text: post.cooked, locale: "zh_CN" })
       expect { described_class.detect_locale(post) }.to change { post.reload.locale }.from(nil).to(
         "zh_CN",
       )
     end
 
     it "bypasses validations when updating locale" do
-      post.update_column(:raw, "A")
+      post.update_column(:cooked, "A")
 
-      language_detector_stub({ text: post.raw, locale: "zh_CN" })
+      language_detector_stub({ text: post.cooked, locale: "zh_CN" })
 
       described_class.detect_locale(post)
       expect(post.reload.locale).to eq("zh_CN")

spec/lib/translation/topic_locale_detector_spec.rb

@@ -3,7 +3,7 @@
 describe DiscourseAi::Translation::TopicLocaleDetector do
   describe ".detect_locale" do
     fab!(:topic) { Fabricate(:topic, title: "this is a cat topic", locale: nil) }
-    fab!(:post) { Fabricate(:post, raw: "and kittens", topic:) }
+    fab!(:post) { Fabricate(:post, topic:) }
 
     def language_detector_stub(opts)
       mock = instance_double(DiscourseAi::Translation::LanguageDetector)
@@ -18,7 +18,7 @@ def language_detector_stub(opts)
     end
 
     it "updates the topic locale with the detected locale" do
-      language_detector_stub({ text: "This is a cat topic and kittens", locale: "zh_CN" })
+      language_detector_stub({ text: "This is a cat topic", locale: "zh_CN" })
       expect { described_class.detect_locale(topic) }.to change { topic.reload.locale }.from(
         nil,
       ).to("zh_CN")
@@ -29,7 +29,7 @@ def language_detector_stub(opts)
       SiteSetting.min_topic_title_length = 15
       SiteSetting.max_topic_title_length = 16
 
-      language_detector_stub({ text: "A and kittens", locale: "zh_CN" })
+      language_detector_stub({ text: "A", locale: "zh_CN" })
 
       described_class.detect_locale(topic)
       expect(topic.reload.locale).to eq("zh_CN")