FEATURE: Add DiscourseAi translator (#181)

This commit adds DiscourseAi as a provider for translation. It defines two DiscourseAi::Completions::Prompt, in translator and language_detector, that uses the set ai_helper_model.

Validations are in place when this new provider is selected but its requirements are not set up (SiteSetting.ai_helper_model, SiteSetting.ai_enabled)

Author: nattsw

about.json

@@ -0,0 +1,7 @@
+{
+  "tests": {
+    "requiredPlugins": [
+      "https://github.com/discourse/discourse-ai"
+    ]
+  }
+}

app/services/discourse_ai/language_detector.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  class LanguageDetector
+    PROMPT_TEXT = <<~TEXT
+      I want you to act as a language expert, determining the locale for a set of text.
+      The locale is a language identifier, such as "en" for English, "de" for German, etc,
+      and can also include a region identifier, such as "en-GB" for British English, or "zh-Hans" for Simplified Chinese.
+      I will provide you with text, and you will determine the locale of the text.
+      Include your locale between <language></language> XML tags.
+    TEXT
+
+    def initialize(text)
+      @text = text
+    end
+
+    def detect
+      prompt =
+        DiscourseAi::Completions::Prompt.new(
+          PROMPT_TEXT,
+          messages: [{ type: :user, content: @text, id: "user" }],
+        )
+
+      response =
+        DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model).generate(
+          prompt,
+          user: Discourse.system_user,
+          feature_name: "translator-language-detect",
+        )
+
+      (Nokogiri::HTML5.fragment(response).at("language")&.text || response)
+    end
+  end
+end

app/services/discourse_ai/translator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  class Translator
+    PROMPT_TEMPLATE = <<~TEXT.freeze
+      You are a highly skilled linguist and web programmer, with expertise in many languages, and very well versed in HTML.
+      Your task is to identify the language of the text I provide and accurately translate it into this language locale "%{target_language}" while preserving the meaning, tone, and nuance of the original text.
+      The text will contain html tags, which must absolutely be preserved in the translation.
+      Maintain proper grammar, spelling, and punctuation in the translated version.
+      Wrap the translated text in a <translation> tag.
+    TEXT
+
+    def initialize(text, target_language)
+      @text = text
+      @target_language = target_language
+    end
+
+    def translate
+      prompt =
+        DiscourseAi::Completions::Prompt.new(
+          build_prompt(@target_language),
+          messages: [{ type: :user, content: @text, id: "user" }],
+        )
+
+      llm_translation =
+        DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model).generate(
+          prompt,
+          user: Discourse.system_user,
+          feature_name: "translator-translate",
+        )
+
+      (Nokogiri::HTML5.fragment(llm_translation).at("translation")&.inner_html || llm_translation)
+    end
+
+    private
+
+    def build_prompt(target_language)
+      PROMPT_TEMPLATE % { target_language: target_language }
+    end
+  end
+end

app/services/discourse_translator/discourse_ai.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require "json"
+
+module DiscourseTranslator
+  class DiscourseAi < Base
+    MAX_DETECT_LOCALE_TEXT_LENGTH = 1000
+    def self.language_supported?(_)
+      true
+    end
+
+    def self.detect(topic_or_post)
+      return unless required_settings_enabled
+
+      topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin
+        ::DiscourseAi::LanguageDetector.new(text_for_detection(topic_or_post)).detect
+      end
+    rescue => e
+      Rails.logger.warn(
+        "#{::DiscourseTranslator::PLUGIN_NAME}: Failed to detect language for #{topic_or_post.class.name} #{topic_or_post.id}: #{e}",
+      )
+    end
+
+    def self.translate(topic_or_post)
+      return unless required_settings_enabled
+
+      detected_lang = detect(topic_or_post)
+      translated_text =
+        from_custom_fields(topic_or_post) do
+          ::DiscourseAi::Translator.new(text_for_translation(topic_or_post), I18n.locale).translate
+        end
+
+      [detected_lang, translated_text]
+    end
+
+    private
+
+    def self.required_settings_enabled
+      SiteSetting.translator_enabled && SiteSetting.translator == "DiscourseAi" &&
+        SiteSetting.discourse_ai_enabled && SiteSetting.ai_helper_enabled
+    end
+  end
+end

config/locales/server.en.yml

@@ -28,6 +28,10 @@ en:
     microsoft:
       missing_token: "The translator was unable to retrieve a valid token."
       missing_key: "No Azure Subscription Key provided."
+
+    discourse_ai:
+      not_installed: "You need to install the discourse-ai plugin to use this feature."
+      ai_helper_required: 'You need to <a href="%{base_url}/admin/site_settings/category/all_results?filter=plugin%3Adiscourse-ai%20helper">configure the ai helper</a> to use this feature.'
   not_in_group:
     user_not_in_group: "You don't belong to a group allowed to translate."
     poster_not_in_group: "Post wasn't made by an user in an allowed group."

config/settings.yml

@@ -7,11 +7,13 @@ discourse_translator:
     client: true
     type: enum
     choices:
+      - DiscourseAi
       - Microsoft
       - Google
       - Amazon
       - Yandex
       - LibreTranslate
+    validator: "DiscourseTranslator::TranslatorSelectionValidator"
   translator_azure_subscription_key:
     default: ''
   translator_azure_region:

lib/discourse_translator/translator_selection_validator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TranslatorSelectionValidator
+    def initialize(opts = {})
+      @opts = opts
+    end
+
+    def valid_value?(val)
+      return true if val.blank?
+
+      if val == "DiscourseAi"
+        return false if !defined?(::DiscourseAi)
+        return false if !SiteSetting.ai_helper_enabled
+      end
+
+      true
+    end
+
+    def error_message
+      return I18n.t("translator.discourse_ai.not_installed") if !defined?(::DiscourseAi)
+
+      I18n.t("translator.discourse_ai.ai_helper_required") if !SiteSetting.ai_helper_enabled
+    end
+  end
+end

plugin.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # name: discourse-translator
-# about: Translates posts on Discourse using Microsoft, Google, Yandex or LibreTranslate translation APIs.
+# about: Translates posts on Discourse using Microsoft, Google, Yandex, LibreTranslate, or Discourse AI translation APIs.
 # meta_topic_id: 32630
 # version: 0.3.0
 # authors: Alan Tan
@@ -31,7 +31,7 @@ module ::DiscourseTranslator
 
   topic_view_post_custom_fields_allowlister { [::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] }
 
-  reloadable_patch do |plugin|
+  reloadable_patch do
     Guardian.prepend(DiscourseTranslator::GuardianExtension)
     Post.prepend(DiscourseTranslator::PostExtension)
     Topic.prepend(DiscourseTranslator::TopicExtension)

spec/lib/translator_selection_validator_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+describe ::DiscourseTranslator::TranslatorSelectionValidator do
+  fab!(:llm_model)
+
+  describe "#valid_value?" do
+    context "when value is blank" do
+      it "returns true" do
+        expect(described_class.new.valid_value?(nil)).to eq(true)
+        expect(described_class.new.valid_value?("")).to eq(true)
+      end
+    end
+
+    context "when value is 'DiscourseAi'" do
+      context "when DiscourseAi is not defined" do
+        it "returns false" do
+          hide_const("DiscourseAi")
+          expect(described_class.new.valid_value?("DiscourseAi")).to eq(false)
+        end
+      end
+
+      context "when DiscourseAi is defined but ai_helper_enabled is false" do
+        it "returns false" do
+          SiteSetting.ai_helper_enabled = false
+          expect(described_class.new.valid_value?("DiscourseAi")).to eq(false)
+        end
+      end
+
+      context "when DiscourseAi is defined and ai_helper_enabled is true" do
+        it "returns true" do
+          DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do
+            SiteSetting.ai_helper_model = "custom:#{llm_model.id}"
+            SiteSetting.ai_helper_enabled = true
+          end
+          expect(described_class.new.valid_value?("DiscourseAi")).to eq(true)
+        end
+      end
+    end
+
+    context "when value is not 'DiscourseAi'" do
+      it "returns true" do
+        expect(described_class.new.valid_value?("googly")).to eq(true)
+        expect(described_class.new.valid_value?("poopy")).to eq(true)
+      end
+    end
+  end
+
+  describe "#error_message" do
+    context "when DiscourseAi is not defined" do
+      it "returns the not_installed error message" do
+        hide_const("DiscourseAi")
+        expect(described_class.new.error_message).to eq(
+          I18n.t("translator.discourse_ai.not_installed"),
+        )
+      end
+    end
+
+    context "when DiscourseAi is defined but ai_helper_enabled is false" do
+      it "returns the ai_helper_required error message" do
+        SiteSetting.ai_helper_enabled = false
+        expect(described_class.new.error_message).to eq(
+          I18n.t("translator.discourse_ai.ai_helper_required"),
+        )
+      end
+    end
+
+    context "when DiscourseAi is defined and ai_helper_enabled is true" do
+      it "returns nil" do
+        DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do
+          SiteSetting.ai_helper_model = "custom:#{llm_model.id}"
+          SiteSetting.ai_helper_enabled = true
+        end
+        expect(described_class.new.error_message).to be_nil
+      end
+    end
+  end
+end

spec/services/discourse_ai/language_detector_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+describe DiscourseAi::LanguageDetector do
+  before do
+    Fabricate(:fake_model).tap do |fake_llm|
+      SiteSetting.public_send("ai_helper_model=", "custom:#{fake_llm.id}")
+    end
+    SiteSetting.ai_helper_enabled = true
+  end
+
+  describe ".detect" do
+    it "creates the correct prompt" do
+      allow(DiscourseAi::Completions::Prompt).to receive(:new).with(
+        DiscourseAi::LanguageDetector::PROMPT_TEXT,
+        messages: [{ type: :user, content: "meow", id: "user" }],
+      ).and_call_original
+
+      described_class.new("meow").detect
+    end
+
+    it "sends the language detection prompt to the ai helper model" do
+      mock_prompt = instance_double(DiscourseAi::Completions::Prompt)
+      mock_llm = instance_double(DiscourseAi::Completions::Llm)
+
+      allow(DiscourseAi::Completions::Prompt).to receive(:new).and_return(mock_prompt)
+      allow(DiscourseAi::Completions::Llm).to receive(:proxy).with(
+        SiteSetting.ai_helper_model,
+      ).and_return(mock_llm)
+      allow(mock_llm).to receive(:generate).with(
+        mock_prompt,
+        user: Discourse.system_user,
+        feature_name: "translator-language-detect",
+      )
+
+      described_class.new("meow").detect
+    end
+
+    it "returns the language from the llm's response in the language tag" do
+      DiscourseAi::Completions::Llm.with_prepared_responses(["<language>de</language>"]) do
+        expect(described_class.new("meow").detect).to eq "de"
+      end
+    end
+  end
+end