FIX: Skip locale region for Google Translate API (#263)

There is some peculiar behaviour with Google Translate API (translation provider) right now that requires this change.

### Extra context: 

For each translation provider, we can make an API call to get the language code (locale) of a piece of text. This can be things like "en", "pt", "zh", "pt-PT", "zh-CN".

We use the result here to determine if the user is able to translate the post or not. If the user is in the same locale as the post, we do not allow them to translate since it is already in a readable language.

For some translation providers, we take this language code and tell the API "please translate this content from pt-PT to en". For some other providers, we can just do "please translate this content to en".

### This fix

Google Translate API is returning "bn-Latn" for some texts. However, 
1. they do not support translation from "bn-Latn" to "en", 
2. they do support translation from "bn" to "en", 
3. same piece of text can be directly translated to "en" without supplying the source locale ("bn-Latn") 🤯 

This fix does two things specific to the Google provider:
- Re-check if an unsupported language is supported, by removing the region part of the locale
- Send content for translation without indicating the source language

Author: nattsw

app/services/discourse_translator/google.rb

@@ -64,6 +64,7 @@ class Google < Base
       nb_NO: "no",
       fa_IR: "fa",
     }
+    CHINESE_LOCALE = "zh"
 
     def self.access_token_key
       "google-translator"
@@ -84,16 +85,22 @@ def self.detect!(topic_or_post)
 
     def self.translate_supported?(source, target)
       res = result(SUPPORT_URI, target: SUPPORTED_LANG_MAPPING[target])
-      res["languages"].any? { |obj| obj["language"] == source }
+      supported = res["languages"].any? { |obj| obj["language"] == source }
+      return true if supported
+
+      normalized_source = source.split("-").first
+      if (source.include?("-") && normalized_source != CHINESE_LOCALE)
+        res["languages"].any? { |obj| obj["language"] == normalized_source }
+      else
+        false
+      end
     end
 
     def self.translate!(translatable, target_locale_sym = I18n.locale)
-      detected_locale = detect(translatable)
       res =
         result(
           TRANSLATE_URI,
           q: text_for_translation(translatable),
-          source: detected_locale,
           target: SUPPORTED_LANG_MAPPING[target_locale_sym],
         )
       res["translations"][0]["translatedText"]

spec/services/google_spec.rb

@@ -73,39 +73,44 @@
 
   describe ".translate_supported?" do
     let(:topic) { Fabricate(:topic, title: "This title is in english") }
-    it "should equate source language to target" do
+
+    it "equates source language to target" do
       source = "en"
       target = "fr"
-      Excon.expects(:post).returns(
-        mock_response.new(200, %{ { "data": { "languages": [ { "language": "#{source}" }] } } }),
+      stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return(
+        status: 200,
+        body: %{ { "data": { "languages": [ { "language": "#{source}" }] } } },
       )
       expect(described_class.translate_supported?(source, target)).to be true
     end
 
-    it "should pass through strings already in target language" do
-      lang = I18n.locale
-      topic.set_detected_locale(lang)
-      expect(described_class.translate(topic)).to eq(["en", "This title is in english"])
+    it "checks again without -* when the source language is not supported" do
+      source = "en"
+      target = "fr"
+      stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return(
+        status: 200,
+        body: %{ { "data": { "languages": [ { "language": "#{source}" }] } } },
+      )
+
+      expect(described_class.translate_supported?("en-GB", target)).to be true
     end
   end
 
-  describe ".translate" do
+  describe ".translate!" do
     let(:post) { Fabricate(:post) }
 
-    it "raise an error and warns admin on failure" do
+    it "raises an error and warns admin on failure" do
       described_class.expects(:access_token).returns(api_key)
       described_class.expects(:detect).returns("__")
 
-      Excon.expects(:post).returns(
-        mock_response.new(
-          400,
-          {
-            error: {
-              code: "400",
-              message: "API key not valid. Please pass a valid API key.",
-            },
-          }.to_json,
-        ),
+      stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return(
+        status: 400,
+        body: {
+          error: {
+            code: "400",
+            message: "API key not valid. Please pass a valid API key.",
+          },
+        }.to_json,
       )
 
       ProblemCheckTracker[:translator_error].no_problem!
@@ -153,34 +158,27 @@
       post.set_detected_locale("de")
       body = {
         q: post.cooked.truncate(SiteSetting.max_characters_per_translation, omission: nil),
-        source: "de",
         target: "en",
         key: api_key,
       }
 
       translated_text = "hur dur hur dur"
-      Excon
-        .expects(:post)
-        .with(
-          DiscourseTranslator::Google::TRANSLATE_URI,
-          body: URI.encode_www_form(body),
-          headers: {
-            "Content-Type" => "application/x-www-form-urlencoded",
-            "Referer" => "http://test.localhost",
-          },
-        )
-        .returns(
-          mock_response.new(
-            200,
-            %{ { "data": { "translations": [ { "translatedText": "#{translated_text}" } ] } } },
-          ),
-        )
-        .once
-      Excon.expects(:post).returns(
-        mock_response.new(200, %{ { "data": { "languages": [ { "language": "de" }] } } }),
+      stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return(
+        status: 200,
+        body: %{ { "data": { "languages": [ { "language": "de" }] } } },
+      )
+      stub_request(:post, DiscourseTranslator::Google::TRANSLATE_URI).with(
+        body: URI.encode_www_form(body),
+        headers: {
+          "Content-Type" => "application/x-www-form-urlencoded",
+          "Referer" => "http://test.localhost",
+        },
+      ).to_return(
+        status: 200,
+        body: %{ { "data": { "translations": [ { "translatedText": "#{translated_text}" } ] } } },
       )
 
-      expect(described_class.translate(post)).to eq(["de", translated_text])
+      expect(described_class.translate!(post)).to eq(translated_text)
     end
   end
 end