FIX: Allow admins to define locales that should be normalized

The criteria to allow a post to be translated is if the user's locale is different from the site's default locale.
For each translator (e.g. Google, Amazon, Azure), we maintain a list of mappings `SUPPORTED_LANG_MAPPING` for each locale. On top of that, th
is mapping is also used to define what language the post should be translated to. When using most translators available in the plugin, "en_GB" forums
 will translate text to "en" due to these mappings. This is not a problem for Discourse AI provider, since there are no restrictions on language.

This PR introduces a site setting `normalize_language_variants_map` so that admins can properly indicate if
A. A. They want `"pt" == "pt_BR" == "pt_PT"` or `"zh" == "zh_CN" == "zh_TW"` or
B. They would prefer that `"en"` is not the same as `"en_GB"`.

Author: nattsw

app/services/discourse_translator/base.rb

@@ -63,11 +63,12 @@ def self.get_text(topic_or_post)
       end
     end
 
-    def self.language_supported?(detected_lang)
+    def self.language_supported?(detected_lang, site_lang)
       raise NotImplementedError unless self.const_defined?(:SUPPORTED_LANG_MAPPING)
+      site_lang_sym = site_lang.to_sym
       supported_lang = const_get(:SUPPORTED_LANG_MAPPING)
-      return false if supported_lang[I18n.locale].nil?
-      detected_lang != supported_lang[I18n.locale]
+      return false if supported_lang[site_lang_sym].nil?
+      detected_lang.to_sym != supported_lang[site_lang_sym].to_sym
     end
 
     private

app/services/discourse_translator/discourse_ai.rb

@@ -6,7 +6,7 @@
 module DiscourseTranslator
   class DiscourseAi < Base
     MAX_DETECT_LOCALE_TEXT_LENGTH = 1000
-    def self.language_supported?(_)
+    def self.language_supported?(_, _)
       true
     end
 

config/settings.yml

@@ -94,6 +94,11 @@ discourse_translator:
     client: true
   max_translations_per_minute:
     default: 3
+  normalize_language_variants:
+    type: list
+    list_type: compact
+    default: "en"
+    client: true
   restrict_translation_by_group:
     default: "11" # default group trust_level_1
     client: true

lib/discourse_translator/guardian_extension.rb

@@ -18,4 +18,28 @@ def can_detect_language?(post)
         post&.user&.in_any_groups?(SiteSetting.restrict_translation_by_poster_group_map)
     ) && post.raw.present? && post.post_type != Post.types[:small_action]
   end
+
+  def can_translate?(post)
+    return false if !user_group_allow_translate?
+
+    # we will deal with strings (not syms) when comparing locales below
+    detected_lang =
+      post.custom_fields[::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD].to_s.sub("-", "_")
+    return false if detected_lang.blank?
+
+    locale_without_region = I18n.locale.to_s.split("_").first
+    site_locale =
+      (
+        if SiteSetting.normalize_language_variants_map.include?(locale_without_region)
+          locale_without_region
+        else
+          I18n.locale.to_s
+        end
+      )
+    detected_lang != site_locale &&
+      "DiscourseTranslator::#{SiteSetting.translator}".constantize.language_supported?(
+        detected_lang,
+        site_locale,
+      )
+  end
 end

plugin.rb

@@ -44,14 +44,6 @@ module ::DiscourseTranslator
   end
 
   add_to_serializer :post, :can_translate do
-    return false if !scope.user_group_allow_translate?
-
-    detected_lang = post_custom_fields[::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD]
-    return false if detected_lang.blank?
-
-    detected_lang.to_sym != I18n.locale &&
-      "DiscourseTranslator::#{SiteSetting.translator}".constantize.language_supported?(
-        detected_lang,
-      )
+    scope.can_translate?(object)
   end
 end

spec/lib/guardian_extension_spec.rb

@@ -113,4 +113,64 @@
       end
     end
   end
+
+  describe "#can_translate?" do
+    fab!(:group)
+    fab!(:user) { Fabricate(:user, locale: "en", groups: [group]) }
+    fab!(:post)
+
+    let(:guardian) { Guardian.new(user) }
+
+    it "returns false when translator disabled" do
+      SiteSetting.translator_enabled = false
+
+      expect(guardian.can_translate?(post)).to eq(false)
+    end
+
+    describe "when translator enabled" do
+      before { SiteSetting.translator_enabled = true }
+
+      describe "anon user" do
+        before { SiteSetting.restrict_translation_by_group = "#{Group::AUTO_GROUPS[:everyone]}" }
+
+        it "cannot translate" do
+          expect(Guardian.new.can_translate?(post)).to eq(false)
+        end
+      end
+
+      describe "logged in user" do
+        it "cannot translate when user is not in restrict_translation_by_group" do
+          SiteSetting.restrict_translation_by_group = "#{group.id + 1}"
+
+          expect(guardian.can_translate?(post)).to eq(false)
+        end
+
+        describe "user is in restrict_translation_by_group" do
+          before { SiteSetting.restrict_translation_by_group = "#{group.id}" }
+
+          describe "locale is :xx" do
+            before { I18n.stubs(:locale).returns(:pt) }
+
+            it "cannot translate when post does not have DETECTED_LANG_CUSTOM_FIELD" do
+              expect(guardian.can_translate?(post)).to eq(false)
+            end
+
+            it "cannot translate when post has DETECTED_LANG_CUSTOM_FIELD matches locale" do
+              post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] = "pt"
+              post.save
+
+              expect(guardian.can_translate?(post)).to eq(false)
+            end
+
+            it "can translate when post has DETECTED_LANG_CUSTOM_FIELD does not match locale" do
+              post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] = "jp"
+              post.save
+
+              expect(guardian.can_translate?(post)).to eq(true)
+            end
+          end
+        end
+      end
+    end
+  end
 end

spec/models/post_spec.rb

@@ -43,7 +43,11 @@
       post =
         PostCreator.new(
           user,
-          { title: "a topic about cats", raw: "tomtom is a cat", category: Fabricate(:category).id },
+          {
+            title: "a topic about cats",
+            raw: "tomtom is a cat",
+            category: Fabricate(:category).id,
+          },
         ).create
 
       expect(

spec/services/base_spec.rb

@@ -12,24 +12,21 @@ class EmptyTranslator < DiscourseTranslator::Base
 
   describe ".language_supported?" do
     it "raises an error when the method is not implemented" do
-      expect { EmptyTranslator.language_supported?("en") }.to raise_error(NotImplementedError)
+      expect { EmptyTranslator.language_supported?("en", "en") }.to raise_error(NotImplementedError)
     end
 
     it "returns false when the locale is not supported" do
-      I18n.stubs(:locale).returns(:xx)
-      expect(TestTranslator.language_supported?("en")).to eq(false)
+      expect(TestTranslator.language_supported?("en", "xx")).to eq(false)
     end
 
     it "returns true when the detected language is not the current locale" do
-      I18n.locale = :pt
-      expect(TestTranslator.language_supported?("en")).to eq(true)
-      expect(TestTranslator.language_supported?("ar")).to eq(true)
-      expect(TestTranslator.language_supported?("es-MX")).to eq(true)
+      expect(TestTranslator.language_supported?("en", "pt")).to eq(true)
+      expect(TestTranslator.language_supported?("ar", "pt")).to eq(true)
+      expect(TestTranslator.language_supported?("es-MX", "pt")).to eq(true)
     end
 
     it "returns false when the detected language is the detected locale" do
-      I18n.locale = :pt
-      expect(TestTranslator.language_supported?("pt")).to eq(false)
+      expect(TestTranslator.language_supported?("pt", "pt")).to eq(false)
     end
   end
 

spec/services/discourse_ai_spec.rb

@@ -16,7 +16,7 @@
 
   describe ".language_supported?" do
     it "returns true for any language" do
-      expect(described_class.language_supported?("any-language")).to eq(true)
+      expect(described_class.language_supported?("any-language", "??")).to eq(true)
     end
   end