Skip to content

DEV: Move saving translations outside of the API providers #244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 6, 2025
Merged

DEV: Move saving translations outside of the API providers #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6272c77
DEV: Move saving into the base class so providers only return api res…
nattsw Mar 6, 2025
0babb90
Update method as it was only available locally
nattsw Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 24 additions & 28 deletions app/services/discourse_translator/amazon.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,40 +108,36 @@ def self.access_token_key
end

def self.detect!(topic_or_post)
save_detected_locale(topic_or_post) do
begin
client.translate_text(
{
text: truncate(text_for_detection(topic_or_post)),
source_language_code: "auto",
target_language_code: SUPPORTED_LANG_MAPPING[I18n.locale],
},
)&.source_language_code
rescue Aws::Errors::MissingCredentialsError
raise I18n.t("translator.amazon.invalid_credentials")
end
begin
client.translate_text(
{
text: truncate(text_for_detection(topic_or_post)),
source_language_code: "auto",
target_language_code: SUPPORTED_LANG_MAPPING[I18n.locale],
},
)&.source_language_code
rescue Aws::Errors::MissingCredentialsError
raise I18n.t("translator.amazon.invalid_credentials")
Comment on lines -111 to +120
Copy link
Contributor Author

@nattsw nattsw Mar 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bulk of the change is just removing save_... out of the providers.

end
end

def self.translate!(translatable, target_locale_sym = I18n.locale)
detected_lang = detect(translatable)

save_translation(translatable, target_locale_sym) do
begin
client.translate_text(
{
text: truncate(text_for_translation(translatable)),
source_language_code: "auto",
target_language_code: SUPPORTED_LANG_MAPPING[target_locale_sym],
},
)
rescue Aws::Translate::Errors::UnsupportedLanguagePairException
raise I18n.t(
"translator.failed",
source_locale: detected_lang,
target_locale: target_locale_sym,
)
end
begin
client.translate_text(
{
text: truncate(text_for_translation(translatable)),
source_language_code: "auto",
target_language_code: SUPPORTED_LANG_MAPPING[target_locale_sym],
},
)
rescue Aws::Translate::Errors::UnsupportedLanguagePairException
raise I18n.t(
"translator.failed",
source_locale: detected_lang,
target_locale: target_locale_sym,
)
end
end

Expand Down
49 changes: 31 additions & 18 deletions app/services/discourse_translator/base.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def self.translate(translatable, target_locale_sym = I18n.locale)
detected_lang = detect(translatable)

if translatable.locale_matches?(target_locale_sym)
return detected_lang, get_text(translatable)
return detected_lang, get_untranslated(translatable)
end

translation = translatable.translation_for(target_locale_sym)
Expand All @@ -48,12 +48,18 @@ def self.translate(translatable, target_locale_sym = I18n.locale)
),
)
end
[detected_lang, translate!(translatable, target_locale_sym)]

translated = translate!(translatable, target_locale_sym)
save_translation(translatable, target_locale_sym) { translated }
[detected_lang, translated]
end

# Subclasses must implement this method to translate the text of a post or topic
# then use the save_translation method to store the translated text.
# Subclasses must implement this method to translate the text of a
# post or topic and return only the translated text.
# Subclasses should use text_for_translation
# @param translatable [Post|Topic]
# @param target_locale_sym [Symbol]
# @return [String]
def self.translate!(translatable, target_locale_sym = I18n.locale)
raise "Not Implemented"
end
Expand All @@ -63,13 +69,16 @@ def self.translate!(translatable, target_locale_sym = I18n.locale)
# @param translatable [Post|Topic]
def self.detect(translatable)
return if text_for_detection(translatable).blank?
get_detected_locale(translatable) || detect!(translatable)
get_detected_locale(translatable) ||
save_detected_locale(translatable) { detect!(translatable) }
end

# Subclasses must implement this method to detect the text of a post or topic
# then use the save_detected_locale method to store the detected locale.
# and return only the detected locale.
# Subclasses should use text_for_detection
# @param translatable [Post|Topic]
def self.detect!(post)
# @return [String]
def self.detect!(translatable)
raise "Not Implemented"
end

Expand Down Expand Up @@ -100,15 +109,6 @@ def self.save_detected_locale(translatable)
detected_locale
end

def self.get_text(translatable)
case translatable.class.name
when "Post"
translatable.cooked
when "Topic"
translatable.title
end
end

def self.language_supported?(detected_lang)
raise NotImplementedError unless self.const_defined?(:SUPPORTED_LANG_MAPPING)
supported_lang = const_get(:SUPPORTED_LANG_MAPPING)
Expand All @@ -129,11 +129,24 @@ def self.strip_tags_for_detection(detection_text)
end

def self.text_for_detection(translatable)
strip_tags_for_detection(get_text(translatable)).truncate(DETECTION_CHAR_LIMIT, omission: nil)
strip_tags_for_detection(get_untranslated(translatable)).truncate(
DETECTION_CHAR_LIMIT,
omission: nil,
)
end

def self.text_for_translation(translatable)
get_text(translatable).truncate(SiteSetting.max_characters_per_translation, omission: nil)
max_char = SiteSetting.max_characters_per_translation
get_untranslated(translatable).truncate(max_char, omission: nil)
end

def self.get_untranslated(translatable)
case translatable.class.name
when "Post"
translatable.cooked
when "Topic"
translatable.title
end
end
end
end
27 changes: 17 additions & 10 deletions app/services/discourse_translator/discourse_ai.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,28 @@ def self.language_supported?(detected_lang)
end

def self.detect!(topic_or_post)
return unless required_settings_enabled

save_detected_locale(topic_or_post) do
::DiscourseAi::LanguageDetector.new(text_for_detection(topic_or_post)).detect
unless required_settings_enabled
raise TranslatorError.new(
I18n.t(
"translator.discourse_ai.ai_helper_required",
{ base_url: Discourse.base_url },
),
)
end

::DiscourseAi::LanguageDetector.new(text_for_detection(topic_or_post)).detect
end

def self.translate!(translatable, target_locale_sym = I18n.locale)
return unless required_settings_enabled
save_translation(translatable, target_locale_sym) do
::DiscourseAi::Translator.new(
text_for_translation(translatable),
target_locale_sym,
).translate
unless required_settings_enabled
raise TranslatorError.new(
I18n.t(
"translator.discourse_ai.ai_helper_required",
{ base_url: Discourse.base_url },
),
)
end
::DiscourseAi::Translator.new(text_for_translation(translatable), target_locale_sym).translate
end

private
Expand Down
30 changes: 13 additions & 17 deletions app/services/discourse_translator/google.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,11 @@ def self.access_token
end

def self.detect!(topic_or_post)
save_detected_locale(topic_or_post) do
result(DETECT_URI, q: text_for_detection(topic_or_post))["detections"][0].max do |a, b|
a.confidence <=> b.confidence
end[
"language"
]
end
result(DETECT_URI, q: text_for_detection(topic_or_post))["detections"][0].max do |a, b|
a.confidence <=> b.confidence
end[
"language"
]
end

def self.translate_supported?(source, target)
Expand All @@ -91,16 +89,14 @@ def self.translate_supported?(source, target)

def self.translate!(translatable, target_locale_sym = I18n.locale)
detected_locale = detect(translatable)
save_translation(translatable, target_locale_sym) do
res =
result(
TRANSLATE_URI,
q: text_for_translation(translatable),
source: detected_locale,
target: SUPPORTED_LANG_MAPPING[target_locale_sym],
)
res["translations"][0]["translatedText"]
end
res =
result(
TRANSLATE_URI,
q: text_for_translation(translatable),
source: detected_locale,
target: SUPPORTED_LANG_MAPPING[target_locale_sym],
)
res["translations"][0]["translatedText"]
end

def self.result(url, body)
Expand Down
34 changes: 15 additions & 19 deletions app/services/discourse_translator/libre_translate.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,12 @@ def self.access_token
end

def self.detect!(topic_or_post)
save_detected_locale(topic_or_post) do
res =
result(
detect_uri,
q: ActionController::Base.helpers.strip_tags(text_for_detection(topic_or_post)),
)
!res.empty? ? res[0]["language"] : "en"
end
res =
result(
detect_uri,
q: ActionController::Base.helpers.strip_tags(text_for_detection(topic_or_post)),
)
!res.empty? ? res[0]["language"] : "en"
end

def self.translate_supported?(source, target)
Expand All @@ -98,17 +96,15 @@ def self.translate_supported?(source, target)
def self.translate!(translatable, target_locale_sym = I18n.locale)
detected_lang = detect(translatable)

save_translation(translatable, target_locale_sym) do
res =
result(
translate_uri,
q: text_for_translation(translatable),
source: detected_lang,
target: SUPPORTED_LANG_MAPPING[target_locale_sym],
format: "html",
)
res["translatedText"]
end
res =
result(
translate_uri,
q: text_for_translation(translatable),
source: detected_lang,
target: SUPPORTED_LANG_MAPPING[target_locale_sym],
format: "html",
)
res["translatedText"]
end

def self.get(url)
Expand Down
27 changes: 10 additions & 17 deletions app/services/discourse_translator/microsoft.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,10 @@ def self.access_token_key
end

def self.detect!(topic_or_post)
save_detected_locale(topic_or_post) do
body = [{ "Text" => text_for_detection(topic_or_post) }].to_json
uri = URI(detect_endpoint)
uri.query = URI.encode_www_form(self.default_query)
result(uri.to_s, body, default_headers).first["language"]
end
body = [{ "Text" => text_for_detection(topic_or_post) }].to_json
uri = URI(detect_endpoint)
uri.query = URI.encode_www_form(self.default_query)
result(uri.to_s, body, default_headers).first["language"]
end

def self.translate!(translatable, target_locale_sym = I18n.locale)
Expand All @@ -166,17 +164,12 @@ def self.translate!(translatable, target_locale_sym = I18n.locale)
locale =
SUPPORTED_LANG_MAPPING[target_locale_sym] || (raise I18n.t("translator.not_supported"))

save_translation(translatable, target_locale_sym) do
query = default_query.merge("from" => detected_lang, "to" => locale, "textType" => "html")

body = [{ "Text" => text_for_translation(translatable) }].to_json

uri = URI(translate_endpoint)
uri.query = URI.encode_www_form(query)

response_body = result(uri.to_s, body, default_headers)
response_body.first["translations"].first["text"]
end
query = default_query.merge("from" => detected_lang, "to" => locale, "textType" => "html")
body = [{ "Text" => text_for_translation(translatable) }].to_json
uri = URI(translate_endpoint)
uri.query = URI.encode_www_form(query)
response_body = result(uri.to_s, body, default_headers)
response_body.first["translations"].first["text"]
end

def self.translate_supported?(detected_lang, target_lang)
Expand Down
32 changes: 14 additions & 18 deletions app/services/discourse_translator/yandex.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,33 +124,29 @@ def self.access_token
end

def self.detect!(topic_or_post)
save_detected_locale(topic_or_post) do
query = default_query.merge("text" => text_for_detection(topic_or_post))
uri = URI(DETECT_URI)
uri.query = URI.encode_www_form(query)
result(uri.to_s, "", default_headers)["lang"]
end
query = default_query.merge("text" => text_for_detection(topic_or_post))
uri = URI(DETECT_URI)
uri.query = URI.encode_www_form(query)
result(uri.to_s, "", default_headers)["lang"]
end

def self.translate!(translatable, target_locale_sym = I18n.locale)
detected_lang = detect(translatable)
locale =
SUPPORTED_LANG_MAPPING[target_locale_sym] || (raise I18n.t("translator.not_supported"))

save_translation(translatable, target_locale_sym) do
query =
default_query.merge(
"lang" => "#{detected_lang}-#{locale}",
"text" => text_for_translation(translatable),
"format" => "html",
)
query =
default_query.merge(
"lang" => "#{detected_lang}-#{locale}",
"text" => text_for_translation(translatable),
"format" => "html",
)

uri = URI(TRANSLATE_URI)
uri.query = URI.encode_www_form(query)
uri = URI(TRANSLATE_URI)
uri.query = URI.encode_www_form(query)

response_body = result(uri.to_s, "", default_headers)
response_body["text"][0]
end
response_body = result(uri.to_s, "", default_headers)
response_body["text"][0]
end

def self.translate_supported?(detected_lang, target_lang)
Expand Down
6 changes: 2 additions & 4 deletions spec/services/discourse_ai_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,11 @@
end

describe ".detect!" do
it "stores the detected language" do
it "returns the detected language" do
locale = "de"
DiscourseAi::Completions::Llm.with_prepared_responses(["<language>de</language>"]) do
DiscourseTranslator::DiscourseAi.detect!(post)
expect(DiscourseTranslator::DiscourseAi.detect!(post)).to eq locale
end

expect(post.detected_locale).to eq locale
end
end

Expand Down
Loading