diff --git a/about.json b/about.json new file mode 100644 index 00000000..ed7fddd3 --- /dev/null +++ b/about.json @@ -0,0 +1,7 @@ +{ + "tests": { + "requiredPlugins": [ + "https://github.com/discourse/discourse-ai" + ] + } +} diff --git a/app/services/discourse_ai/language_detector.rb b/app/services/discourse_ai/language_detector.rb new file mode 100644 index 00000000..6357fd0c --- /dev/null +++ b/app/services/discourse_ai/language_detector.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module DiscourseAi + class LanguageDetector + PROMPT_TEXT = <<~TEXT + I want you to act as a language expert, determining the locale for a set of text. + The locale is a language identifier, such as "en" for English, "de" for German, etc, + and can also include a region identifier, such as "en-GB" for British English, or "zh-Hans" for Simplified Chinese. + I will provide you with text, and you will determine the locale of the text. + Include your locale between XML tags. + TEXT + + def initialize(text) + @text = text + end + + def detect + prompt = + DiscourseAi::Completions::Prompt.new( + PROMPT_TEXT, + messages: [{ type: :user, content: @text, id: "user" }], + ) + + response = + DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model).generate( + prompt, + user: Discourse.system_user, + feature_name: "translator-language-detect", + ) + + (Nokogiri::HTML5.fragment(response).at("language")&.text || response) + end + end +end diff --git a/app/services/discourse_ai/translator.rb b/app/services/discourse_ai/translator.rb new file mode 100644 index 00000000..ff2cba1b --- /dev/null +++ b/app/services/discourse_ai/translator.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module DiscourseAi + class Translator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a highly skilled linguist and web programmer, with expertise in many languages, and very well versed in HTML. + Your task is to identify the language of the text I provide and accurately translate it into this language locale "%{target_language}" while preserving the meaning, tone, and nuance of the original text. + The text will contain html tags, which must absolutely be preserved in the translation. + Maintain proper grammar, spelling, and punctuation in the translated version. + Wrap the translated text in a tag. + TEXT + + def initialize(text, target_language) + @text = text + @target_language = target_language + end + + def translate + prompt = + DiscourseAi::Completions::Prompt.new( + build_prompt(@target_language), + messages: [{ type: :user, content: @text, id: "user" }], + ) + + llm_translation = + DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_helper_model).generate( + prompt, + user: Discourse.system_user, + feature_name: "translator-translate", + ) + + (Nokogiri::HTML5.fragment(llm_translation).at("translation")&.inner_html || llm_translation) + end + + private + + def build_prompt(target_language) + PROMPT_TEMPLATE % { target_language: target_language } + end + end +end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb new file mode 100644 index 00000000..93dc4be3 --- /dev/null +++ b/app/services/discourse_translator/discourse_ai.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "base" +require "json" + +module DiscourseTranslator + class DiscourseAi < Base + MAX_DETECT_LOCALE_TEXT_LENGTH = 1000 + def self.language_supported?(_) + true + end + + def self.detect(topic_or_post) + return unless required_settings_enabled + + topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin + ::DiscourseAi::LanguageDetector.new(text_for_detection(topic_or_post)).detect + end + rescue => e + Rails.logger.warn( + "#{::DiscourseTranslator::PLUGIN_NAME}: Failed to detect language for #{topic_or_post.class.name} #{topic_or_post.id}: #{e}", + ) + end + + def self.translate(topic_or_post) + return unless required_settings_enabled + + detected_lang = detect(topic_or_post) + translated_text = + from_custom_fields(topic_or_post) do + ::DiscourseAi::Translator.new(text_for_translation(topic_or_post), I18n.locale).translate + end + + [detected_lang, translated_text] + end + + private + + def self.required_settings_enabled + SiteSetting.translator_enabled && SiteSetting.translator == "DiscourseAi" && + SiteSetting.discourse_ai_enabled && SiteSetting.ai_helper_enabled + end + end +end diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 135e457d..eaa148db 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -28,6 +28,10 @@ en: microsoft: missing_token: "The translator was unable to retrieve a valid token." missing_key: "No Azure Subscription Key provided." + + discourse_ai: + not_installed: "You need to install the discourse-ai plugin to use this feature." + ai_helper_required: 'You need to configure the ai helper to use this feature.' not_in_group: user_not_in_group: "You don't belong to a group allowed to translate." poster_not_in_group: "Post wasn't made by an user in an allowed group." diff --git a/config/settings.yml b/config/settings.yml index b0912bec..6ddc6c8a 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,11 +7,13 @@ discourse_translator: client: true type: enum choices: + - DiscourseAi - Microsoft - Google - Amazon - Yandex - LibreTranslate + validator: "DiscourseTranslator::TranslatorSelectionValidator" translator_azure_subscription_key: default: '' translator_azure_region: diff --git a/lib/discourse_translator/translator_selection_validator.rb b/lib/discourse_translator/translator_selection_validator.rb new file mode 100644 index 00000000..68835b05 --- /dev/null +++ b/lib/discourse_translator/translator_selection_validator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TranslatorSelectionValidator + def initialize(opts = {}) + @opts = opts + end + + def valid_value?(val) + return true if val.blank? + + if val == "DiscourseAi" + return false if !defined?(::DiscourseAi) + return false if !SiteSetting.ai_helper_enabled + end + + true + end + + def error_message + return I18n.t("translator.discourse_ai.not_installed") if !defined?(::DiscourseAi) + + I18n.t("translator.discourse_ai.ai_helper_required") if !SiteSetting.ai_helper_enabled + end + end +end diff --git a/plugin.rb b/plugin.rb index b25de7fb..bc48b8cb 100644 --- a/plugin.rb +++ b/plugin.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # name: discourse-translator -# about: Translates posts on Discourse using Microsoft, Google, Yandex or LibreTranslate translation APIs. +# about: Translates posts on Discourse using Microsoft, Google, Yandex, LibreTranslate, or Discourse AI translation APIs. # meta_topic_id: 32630 # version: 0.3.0 # authors: Alan Tan @@ -31,7 +31,7 @@ module ::DiscourseTranslator topic_view_post_custom_fields_allowlister { [::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] } - reloadable_patch do |plugin| + reloadable_patch do Guardian.prepend(DiscourseTranslator::GuardianExtension) Post.prepend(DiscourseTranslator::PostExtension) Topic.prepend(DiscourseTranslator::TopicExtension) diff --git a/spec/lib/translator_selection_validator_spec.rb b/spec/lib/translator_selection_validator_spec.rb new file mode 100644 index 00000000..de7c898c --- /dev/null +++ b/spec/lib/translator_selection_validator_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe ::DiscourseTranslator::TranslatorSelectionValidator do + fab!(:llm_model) + + describe "#valid_value?" do + context "when value is blank" do + it "returns true" do + expect(described_class.new.valid_value?(nil)).to eq(true) + expect(described_class.new.valid_value?("")).to eq(true) + end + end + + context "when value is 'DiscourseAi'" do + context "when DiscourseAi is not defined" do + it "returns false" do + hide_const("DiscourseAi") + expect(described_class.new.valid_value?("DiscourseAi")).to eq(false) + end + end + + context "when DiscourseAi is defined but ai_helper_enabled is false" do + it "returns false" do + SiteSetting.ai_helper_enabled = false + expect(described_class.new.valid_value?("DiscourseAi")).to eq(false) + end + end + + context "when DiscourseAi is defined and ai_helper_enabled is true" do + it "returns true" do + DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do + SiteSetting.ai_helper_model = "custom:#{llm_model.id}" + SiteSetting.ai_helper_enabled = true + end + expect(described_class.new.valid_value?("DiscourseAi")).to eq(true) + end + end + end + + context "when value is not 'DiscourseAi'" do + it "returns true" do + expect(described_class.new.valid_value?("googly")).to eq(true) + expect(described_class.new.valid_value?("poopy")).to eq(true) + end + end + end + + describe "#error_message" do + context "when DiscourseAi is not defined" do + it "returns the not_installed error message" do + hide_const("DiscourseAi") + expect(described_class.new.error_message).to eq( + I18n.t("translator.discourse_ai.not_installed"), + ) + end + end + + context "when DiscourseAi is defined but ai_helper_enabled is false" do + it "returns the ai_helper_required error message" do + SiteSetting.ai_helper_enabled = false + expect(described_class.new.error_message).to eq( + I18n.t("translator.discourse_ai.ai_helper_required"), + ) + end + end + + context "when DiscourseAi is defined and ai_helper_enabled is true" do + it "returns nil" do + DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do + SiteSetting.ai_helper_model = "custom:#{llm_model.id}" + SiteSetting.ai_helper_enabled = true + end + expect(described_class.new.error_message).to be_nil + end + end + end +end diff --git a/spec/services/discourse_ai/language_detector_spec.rb b/spec/services/discourse_ai/language_detector_spec.rb new file mode 100644 index 00000000..2105ca36 --- /dev/null +++ b/spec/services/discourse_ai/language_detector_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseAi::LanguageDetector do + before do + Fabricate(:fake_model).tap do |fake_llm| + SiteSetting.public_send("ai_helper_model=", "custom:#{fake_llm.id}") + end + SiteSetting.ai_helper_enabled = true + end + + describe ".detect" do + it "creates the correct prompt" do + allow(DiscourseAi::Completions::Prompt).to receive(:new).with( + DiscourseAi::LanguageDetector::PROMPT_TEXT, + messages: [{ type: :user, content: "meow", id: "user" }], + ).and_call_original + + described_class.new("meow").detect + end + + it "sends the language detection prompt to the ai helper model" do + mock_prompt = instance_double(DiscourseAi::Completions::Prompt) + mock_llm = instance_double(DiscourseAi::Completions::Llm) + + allow(DiscourseAi::Completions::Prompt).to receive(:new).and_return(mock_prompt) + allow(DiscourseAi::Completions::Llm).to receive(:proxy).with( + SiteSetting.ai_helper_model, + ).and_return(mock_llm) + allow(mock_llm).to receive(:generate).with( + mock_prompt, + user: Discourse.system_user, + feature_name: "translator-language-detect", + ) + + described_class.new("meow").detect + end + + it "returns the language from the llm's response in the language tag" do + DiscourseAi::Completions::Llm.with_prepared_responses(["de"]) do + expect(described_class.new("meow").detect).to eq "de" + end + end + end +end diff --git a/spec/services/discourse_ai/translator_spec.rb b/spec/services/discourse_ai/translator_spec.rb new file mode 100644 index 00000000..d3f10c7d --- /dev/null +++ b/spec/services/discourse_ai/translator_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseAi::Translator do + before do + Fabricate(:fake_model).tap do |fake_llm| + SiteSetting.public_send("ai_helper_model=", "custom:#{fake_llm.id}") + end + SiteSetting.ai_helper_enabled = true + end + + describe ".translate" do + let(:text_to_translate) { "cats are great" } + let(:target_language) { "de" } + + it "creates the correct prompt" do + allow(DiscourseAi::Completions::Prompt).to receive(:new).with( + <<~TEXT, + You are a highly skilled linguist and web programmer, with expertise in many languages, and very well versed in HTML. + Your task is to identify the language of the text I provide and accurately translate it into this language locale "de" while preserving the meaning, tone, and nuance of the original text. + The text will contain html tags, which must absolutely be preserved in the translation. + Maintain proper grammar, spelling, and punctuation in the translated version. + Wrap the translated text in a tag. + TEXT + messages: [{ type: :user, content: text_to_translate, id: "user" }], + ).and_call_original + + described_class.new(text_to_translate, target_language).translate + end + + it "sends the translation prompt to the selected ai helper model" do + mock_prompt = instance_double(DiscourseAi::Completions::Prompt) + mock_llm = instance_double(DiscourseAi::Completions::Llm) + + allow(DiscourseAi::Completions::Prompt).to receive(:new).and_return(mock_prompt) + allow(DiscourseAi::Completions::Llm).to receive(:proxy).with( + SiteSetting.ai_helper_model, + ).and_return(mock_llm) + allow(mock_llm).to receive(:generate).with( + mock_prompt, + user: Discourse.system_user, + feature_name: "translator-translate", + ) + + described_class.new(text_to_translate, target_language).translate + end + + it "returns the translation from the llm's response in the translation tag" do + DiscourseAi::Completions::Llm.with_prepared_responses( + ["hur dur hur dur!"], + ) do + expect( + described_class.new(text_to_translate, target_language).translate, + ).to eq "hur dur hur dur!" + end + end + + it "returns the raw response if the translation tag is not present" do + DiscourseAi::Completions::Llm.with_prepared_responses(["raw response."]) do + expect( + described_class.new(text_to_translate, target_language).translate, + ).to eq "raw response." + end + end + end +end diff --git a/spec/services/discourse_ai_spec.rb b/spec/services/discourse_ai_spec.rb new file mode 100644 index 00000000..37bcefc2 --- /dev/null +++ b/spec/services/discourse_ai_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseTranslator::DiscourseAi do + fab!(:post) + + before do + Fabricate(:fake_model).tap do |fake_llm| + SiteSetting.public_send("ai_helper_model=", "custom:#{fake_llm.id}") + end + SiteSetting.ai_helper_enabled = true + SiteSetting.translator_enabled = true + SiteSetting.translator = "DiscourseAi" + end + + describe ".language_supported?" do + it "returns true for any language" do + expect(described_class.language_supported?("any-language")).to eq(true) + end + end + + describe ".detect" do + it "stores the detected language in a custom field" do + locale = "de" + DiscourseAi::Completions::Llm.with_prepared_responses(["de"]) do + DiscourseTranslator::DiscourseAi.detect(post) + post.save_custom_fields + end + + expect(post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD]).to eq locale + end + end + + describe ".translate" do + before do + post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] = "de" + post.save_custom_fields + end + + it "translates the post and returns [locale, translated_text]" do + DiscourseAi::Completions::Llm.with_prepared_responses( + ["some translated text"], + ) do + locale, translated_text = DiscourseTranslator::DiscourseAi.translate(post) + expect(locale).to eq "de" + expect(translated_text).to eq "some translated text" + end + end + end +end