FIX: Normalize languages within Discourse AI translator (#194)

The criteria to allow a post to be translated is if the user's locale is different from the site's default locale.

For each translator (e.g. Google, Amazon, Azure), we maintain a list of mappings `SUPPORTED_LANG_MAPPING` for each locale. On top of that, this mapping is also used to define what language the post should be translated to. When using most translators available in the plugin, "en_GB" forums will translate text to "en" due to these mappings. This is not a problem for Discourse AI provider, since there are no restrictions on language.

This PR normalizes locales within Discourse AI translator, so that it sees "ar" to be the same as "ar_SA".

Author: nattsw

app/services/discourse_translator/discourse_ai.rb

@@ -6,8 +6,9 @@
 module DiscourseTranslator
   class DiscourseAi < Base
     MAX_DETECT_LOCALE_TEXT_LENGTH = 1000
-    def self.language_supported?(_)
-      true
+    def self.language_supported?(detected_lang)
+      locale_without_region = I18n.locale.to_s.split("_").first
+      detected_lang != locale_without_region
     end
 
     def self.detect(topic_or_post)

lib/discourse_translator/guardian_extension.rb

@@ -18,4 +18,21 @@ def can_detect_language?(post)
         post&.user&.in_any_groups?(SiteSetting.restrict_translation_by_poster_group_map)
     ) && post.raw.present? && post.post_type != Post.types[:small_action]
   end
+
+  def can_translate?(post)
+    return false if !user_group_allow_translate?
+
+    # we will deal with regionalized_strings (not syms) when comparing locales
+    # e.g. "en_GB"
+    #      not "en-GB"
+    #      nor :en_GB (I18n.locale)
+    detected_lang =
+      post.custom_fields[::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD].to_s.sub("-", "_")
+    return false if detected_lang.blank?
+
+    detected_lang != I18n.locale.to_s &&
+      "DiscourseTranslator::#{SiteSetting.translator}".constantize.language_supported?(
+        detected_lang,
+      )
+  end
 end

plugin.rb

@@ -44,14 +44,6 @@ module ::DiscourseTranslator
   end
 
   add_to_serializer :post, :can_translate do
-    return false if !scope.user_group_allow_translate?
-
-    detected_lang = post_custom_fields[::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD]
-    return false if detected_lang.blank?
-
-    detected_lang.to_sym != I18n.locale &&
-      "DiscourseTranslator::#{SiteSetting.translator}".constantize.language_supported?(
-        detected_lang,
-      )
+    scope.can_translate?(object)
   end
 end

spec/lib/guardian_extension_spec.rb

@@ -113,4 +113,64 @@
       end
     end
   end
+
+  describe "#can_translate?" do
+    fab!(:group)
+    fab!(:user) { Fabricate(:user, locale: "en", groups: [group]) }
+    fab!(:post)
+
+    let(:guardian) { Guardian.new(user) }
+
+    it "returns false when translator disabled" do
+      SiteSetting.translator_enabled = false
+
+      expect(guardian.can_translate?(post)).to eq(false)
+    end
+
+    describe "when translator enabled" do
+      before { SiteSetting.translator_enabled = true }
+
+      describe "anon user" do
+        before { SiteSetting.restrict_translation_by_group = "#{Group::AUTO_GROUPS[:everyone]}" }
+
+        it "cannot translate" do
+          expect(Guardian.new.can_translate?(post)).to eq(false)
+        end
+      end
+
+      describe "logged in user" do
+        it "cannot translate when user is not in restrict_translation_by_group" do
+          SiteSetting.restrict_translation_by_group = "#{group.id + 1}"
+
+          expect(guardian.can_translate?(post)).to eq(false)
+        end
+
+        describe "user is in restrict_translation_by_group" do
+          before { SiteSetting.restrict_translation_by_group = "#{group.id}" }
+
+          describe "locale is :xx" do
+            before { I18n.stubs(:locale).returns(:pt) }
+
+            it "cannot translate when post does not have DETECTED_LANG_CUSTOM_FIELD" do
+              expect(guardian.can_translate?(post)).to eq(false)
+            end
+
+            it "cannot translate when post has DETECTED_LANG_CUSTOM_FIELD matches locale" do
+              post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] = "pt"
+              post.save
+
+              expect(guardian.can_translate?(post)).to eq(false)
+            end
+
+            it "can translate when post has DETECTED_LANG_CUSTOM_FIELD does not match locale" do
+              post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] = "jp"
+              post.save
+
+              expect(guardian.can_translate?(post)).to eq(true)
+            end
+          end
+        end
+      end
+    end
+  end
 end

spec/services/discourse_ai_spec.rb

@@ -15,9 +15,15 @@
   end
 
   describe ".language_supported?" do
-    it "returns true for any language" do
+    it "returns true when detected language is different from i18n locale" do
+      I18n.stubs(:locale).returns(:xx)
       expect(described_class.language_supported?("any-language")).to eq(true)
     end
+
+    it "returns false when detected language is same base language as i18n locale" do
+      I18n.stubs(:locale).returns(:en_GB)
+      expect(described_class.language_supported?("en")).to eq(false)
+    end
   end
 
   describe ".detect" do