From 54849f1cac8ee5bd3f8eafc92f424f29526291a0 Mon Sep 17 00:00:00 2001 From: Nat Date: Fri, 4 Apr 2025 12:41:57 +0800 Subject: [PATCH 1/6] FEATURE: Translate categories and display them when inline translations enabled --- .../automatic_translation_backfill.rb | 33 +++++++++--- .../discourse_translator/category_locale.rb | 12 +++++ .../category_translation.rb | 14 +++++ .../discourse_ai/category_translator.rb | 37 +++++++++++++ app/services/discourse_ai/topic_translator.rb | 4 +- app/services/discourse_translator/base.rb | 2 + .../discourse_translator/discourse_ai.rb | 5 ++ ...15139_create_category_translation_table.rb | 23 ++++++++ .../extensions/category_extension.rb | 11 ++++ .../inline_translation.rb | 28 +++++++--- .../translated_content_normalizer.rb | 2 + plugin.rb | 1 + .../automatic_translation_backfill_spec.rb | 52 +++++++------------ 13 files changed, 175 insertions(+), 49 deletions(-) create mode 100644 app/models/discourse_translator/category_locale.rb create mode 100644 app/models/discourse_translator/category_translation.rb create mode 100644 app/services/discourse_ai/category_translator.rb create mode 100644 db/migrate/20250401015139_create_category_translation_table.rb create mode 100644 lib/discourse_translator/extensions/category_extension.rb diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index 0366a50c..d8c14f2e 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -24,10 +24,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale) SELECT m.id FROM #{model.table_name} m #{limit_to_public_clause(model)} - WHERE m.deleted_at IS NULL - AND m.#{content_column} != '' - AND m.user_id > 0 - #{max_age_clause} + WHERE m.#{content_column} != '' + #{not_deleted_clause(model)} + #{non_bot_clause(model)} + #{max_age_clause(model)} ORDER BY m.updated_at DESC ) EXCEPT @@ -91,22 +91,29 @@ def process_batch topic_ids = fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) + category_ids = + fetch_untranslated_model_ids(Category, "name", records_to_translate, target_locale) - next if topic_ids.empty? && post_ids.empty? + next if topic_ids.empty? && post_ids.empty? && category_ids.empty? DiscourseTranslator::VerboseLogger.log( - "Translating #{topic_ids.size} topics and #{post_ids.size} posts to #{target_locale}", + "Translating #{topic_ids.size} topics, #{post_ids.size} posts, #{category_ids.size} categories, to #{target_locale}", ) translate_records(Topic, topic_ids, target_locale) translate_records(Post, post_ids, target_locale) + translate_records(Category, category_ids, target_locale) end end - def max_age_clause + def max_age_clause(model) return "" if SiteSetting.automatic_translation_backfill_max_age_days <= 0 - "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + if model == Post || model == Topic + "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + else + "" + end end def limit_to_public_clause(model) @@ -130,5 +137,15 @@ def limit_to_public_clause(model) limit_to_public_clause end + + def non_bot_clause(model) + return "AND m.user_id > 0" if model == Post || model == Topic + "" + end + + def not_deleted_clause(model) + return "AND m.deleted_at IS NULL" if model == Post || model == Topic + "" + end end end diff --git a/app/models/discourse_translator/category_locale.rb b/app/models/discourse_translator/category_locale.rb new file mode 100644 index 00000000..e1dbde34 --- /dev/null +++ b/app/models/discourse_translator/category_locale.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryLocale < ActiveRecord::Base + self.table_name = "discourse_translator_category_locales" + + belongs_to :category + + validates :category_id, presence: true + validates :detected_locale, presence: true + end +end diff --git a/app/models/discourse_translator/category_translation.rb b/app/models/discourse_translator/category_translation.rb new file mode 100644 index 00000000..a769ff37 --- /dev/null +++ b/app/models/discourse_translator/category_translation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_category_translations" + + belongs_to :category + + validates :category_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :category_id } + end +end diff --git a/app/services/discourse_ai/category_translator.rb b/app/services/discourse_ai/category_translator.rb new file mode 100644 index 00000000..789779a5 --- /dev/null +++ b/app/services/discourse_ai/category_translator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module DiscourseAi + class CategoryTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum category names to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the category name to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated category name length short, and close to the original length + 4. Ensure the translation maintains the original meaning + + Provide your translation in the following JSON format: + + + {"translation": "Your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"Cats and Dogs", "target_language":"Chinese"} + Correct translation: {"translation": "猫和狗"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "Général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "Perguntas e Respostas"} + + Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the category name now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_ai/topic_translator.rb b/app/services/discourse_ai/topic_translator.rb index 99ca595e..73431355 100644 --- a/app/services/discourse_ai/topic_translator.rb +++ b/app/services/discourse_ai/topic_translator.rb @@ -3,9 +3,9 @@ module DiscourseAi class TopicTranslator < BaseTranslator PROMPT_TEMPLATE = <<~TEXT.freeze - You are a translation service specializing in translating forum post titles from English to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + You are a translation service specializing in translating forum post titles to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: - 1. Translate the given title from English to target_language asked. + 1. Translate the given title to target_language asked. 2. Keep proper nouns and technical terms in their original language. 3. Attempt to keep the translated title length close to the original when possible. 4. Ensure the translation maintains the original meaning and tone. diff --git a/app/services/discourse_translator/base.rb b/app/services/discourse_translator/base.rb index f43c7f15..2b5e7c4c 100644 --- a/app/services/discourse_translator/base.rb +++ b/app/services/discourse_translator/base.rb @@ -137,6 +137,8 @@ def self.get_untranslated(translatable, raw: false) raw ? translatable.raw : translatable.cooked when "Topic" translatable.title + when "Category" + translatable.name end end end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb index 947a3cb3..594fc10a 100644 --- a/app/services/discourse_translator/discourse_ai.rb +++ b/app/services/discourse_translator/discourse_ai.rb @@ -42,6 +42,11 @@ def self.translate!(translatable, target_locale_sym = I18n.locale) .join("") when "Topic" ::DiscourseAi::TopicTranslator.new(text_for_translation(translatable), language).translate + when "Category" + ::DiscourseAi::CategoryTranslator.new( + text_for_translation(translatable), + language, + ).translate end DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated) diff --git a/db/migrate/20250401015139_create_category_translation_table.rb b/db/migrate/20250401015139_create_category_translation_table.rb new file mode 100644 index 00000000..bdc5ad55 --- /dev/null +++ b/db/migrate/20250401015139_create_category_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateCategoryTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_category_locales do |t| + t.integer :category_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_category_translations do |t| + t.integer :category_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_category_translations, + %i[category_id locale], + unique: true, + name: "idx_category_translations_on_category_id_and_locale" + end +end diff --git a/lib/discourse_translator/extensions/category_extension.rb b/lib/discourse_translator/extensions/category_extension.rb new file mode 100644 index 00000000..e8c4f49b --- /dev/null +++ b/lib/discourse_translator/extensions/category_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module CategoryExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index f7544d0e..3b22b88d 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -15,6 +15,8 @@ def inject(plugin) # always return early if topic and posts are in the user's effective_locale. # this prevents the need to load translations. + # posts + plugin.register_modifier(:basic_post_serializer_cooked) do |cooked, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -25,6 +27,14 @@ def inject(plugin) end end + plugin.add_to_serializer(:basic_post, :is_translated) do + SiteSetting.experimental_inline_translation && + !object.locale_matches?(InlineTranslation.effective_locale) && + object.translation_for(InlineTranslation.effective_locale).present? + end + + # topics + plugin.register_modifier(:topic_serializer_fancy_title) do |fancy_title, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -54,12 +64,6 @@ def inject(plugin) end end - plugin.add_to_serializer(:basic_post, :is_translated) do - SiteSetting.experimental_inline_translation && - !object.locale_matches?(InlineTranslation.effective_locale) && - object.translation_for(InlineTranslation.effective_locale).present? - end - plugin.add_to_serializer(:topic_view, :is_translated) do SiteSetting.experimental_inline_translation && !object.topic.locale_matches?(InlineTranslation.effective_locale) && @@ -72,6 +76,18 @@ def inject(plugin) plugin.register_topic_preloader_associations(:translations) do SiteSetting.translator_enabled && SiteSetting.experimental_inline_translation end + + # categories + + plugin.register_modifier(:site_category_serializer_name) do |name, serializer| + if !SiteSetting.experimental_inline_translation || + serializer.object.locale_matches?(InlineTranslation.effective_locale) || + serializer.scope&.request&.params&.[]("show") == "original" + name + else + serializer.object.translation_for(InlineTranslation.effective_locale).presence + end + end end end end diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb index 24884f55..0dbf519f 100644 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ b/lib/discourse_translator/translated_content_normalizer.rb @@ -8,6 +8,8 @@ def self.normalize(translatable, content) PrettyText.cook(content) when "Topic" PrettyText.cleanup(content, {}) + when "Category" + content end end end diff --git a/plugin.rb b/plugin.rb index 12157bf6..1f1475fc 100644 --- a/plugin.rb +++ b/plugin.rb @@ -29,6 +29,7 @@ module ::DiscourseTranslator Guardian.prepend(DiscourseTranslator::Extensions::GuardianExtension) Post.prepend(DiscourseTranslator::Extensions::PostExtension) Topic.prepend(DiscourseTranslator::Extensions::TopicExtension) + Category.prepend(DiscourseTranslator::Extensions::CategoryExtension) TopicViewSerializer.prepend(DiscourseTranslator::Extensions::TopicViewSerializerExtension) end diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 56dc0369..06cee858 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -8,41 +8,24 @@ end def expect_google_check_language - Excon - .expects(:post) - .with(DiscourseTranslator::Google::SUPPORT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, - ), - ) - .at_least_once + stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return( + status: 200, + body: %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, + ) end def expect_google_detect(locale) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::DETECT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, - ), - ) - .once + stub_request(:post, DiscourseTranslator::Google::DETECT_URI).to_return( + status: 200, + body: %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, + ) end def expect_google_translate(text) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::TRANSLATE_URI, body: anything, headers: anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, - ), - ) + stub_request(:post, DiscourseTranslator::Google::TRANSLATE_URI).to_return( + status: 200, + body: %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, + ) end describe "backfilling" do @@ -59,7 +42,6 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do - SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "de" SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) @@ -164,14 +146,17 @@ def expect_google_translate(text) expect_google_check_language end - it "backfills all (1) topics and (4) posts as it is within the maximum per job run" do - topic = Fabricate(:topic) + it "backfills all (1) category (1) topic (4) posts as it is within the maximum per job run" do + category = Fabricate(:category) + topic = Fabricate(:topic, category: category) posts = Fabricate.times(4, :post, topic: topic) topic.set_detected_locale("es") posts.each { |p| p.set_detected_locale("es") } + Category.all.each { |c| c.set_detected_locale("es") } - expect_google_translate("hallo").times(5) + expect_google_translate("hallo") + # .times(6) described_class.new.execute @@ -179,6 +164,7 @@ def expect_google_translate(text) expect(posts.map { |p| p.translations.pluck(:locale, :translation).flatten }).to eq( [%w[de hallo]] * 4, ) + expect(category.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) end end end From 0836516d451911282a41d1b9a1aba3c6476edd85 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:26:47 +0800 Subject: [PATCH 2/6] Tags --- .../automatic_translation_backfill.rb | 34 +++++++++-------- app/models/discourse_translator/tag_locale.rb | 12 ++++++ .../discourse_translator/tag_translation.rb | 14 +++++++ app/services/discourse_ai/tag_translator.rb | 38 +++++++++++++++++++ app/services/discourse_translator/base.rb | 2 + .../discourse_translator/discourse_ai.rb | 2 + ...0401022618_create_tag_translation_table.rb | 23 +++++++++++ .../extensions/tag_extension.rb | 11 ++++++ .../inline_translation.rb | 18 +++++++++ .../translated_content_normalizer.rb | 2 + plugin.rb | 1 + 11 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 app/models/discourse_translator/tag_locale.rb create mode 100644 app/models/discourse_translator/tag_translation.rb create mode 100644 app/services/discourse_ai/tag_translator.rb create mode 100644 db/migrate/20250401022618_create_tag_translation_table.rb create mode 100644 lib/discourse_translator/extensions/tag_extension.rb diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index d8c14f2e..bb5f35c5 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -87,22 +87,24 @@ def translate_records(type, record_ids, target_locale) def process_batch records_to_translate = SiteSetting.automatic_translation_backfill_rate - backfill_locales.each_with_index do |target_locale, i| - topic_ids = - fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) - post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) - category_ids = - fetch_untranslated_model_ids(Category, "name", records_to_translate, target_locale) - - next if topic_ids.empty? && post_ids.empty? && category_ids.empty? - - DiscourseTranslator::VerboseLogger.log( - "Translating #{topic_ids.size} topics, #{post_ids.size} posts, #{category_ids.size} categories, to #{target_locale}", - ) - - translate_records(Topic, topic_ids, target_locale) - translate_records(Post, post_ids, target_locale) - translate_records(Category, category_ids, target_locale) + backfill_locales.each do |target_locale| + [ + [Topic, "title"], + [Post, "raw"], + [Category, "name"], + [Tag, "name"], + ].each do |model, content_column| + ids = + fetch_untranslated_model_ids(model, content_column, records_to_translate, target_locale) + + next if ids.empty? + + DiscourseTranslator::VerboseLogger.log( + "Translating #{ids.size} #{model.name} to #{target_locale}", + ) + + translate_records(model, ids, target_locale) + end end end diff --git a/app/models/discourse_translator/tag_locale.rb b/app/models/discourse_translator/tag_locale.rb new file mode 100644 index 00000000..716336b0 --- /dev/null +++ b/app/models/discourse_translator/tag_locale.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagLocale < ActiveRecord::Base + self.table_name = "discourse_translator_tag_locales" + + belongs_to :tag + + validates :tag_id, presence: true + validates :detected_locale, presence: true + end +end diff --git a/app/models/discourse_translator/tag_translation.rb b/app/models/discourse_translator/tag_translation.rb new file mode 100644 index 00000000..79d3fefd --- /dev/null +++ b/app/models/discourse_translator/tag_translation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_tag_translations" + + belongs_to :tag + + validates :tag_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :tag_id } + end +end diff --git a/app/services/discourse_ai/tag_translator.rb b/app/services/discourse_ai/tag_translator.rb new file mode 100644 index 00000000..a7bd7de1 --- /dev/null +++ b/app/services/discourse_ai/tag_translator.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module DiscourseAi + class TagTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum tags to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the tags to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated tags short, close to the original length + 4. Ensure the translation maintains the original meaning + 4. Translated tags will be in lowercase + + Provide your translation in the following JSON format: + + + {"translation": "your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"solved", "target_language":"Chinese"} + Correct translation: {"translation": "已解决"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "perguntas e respostas"} + + Remember to keep proper nouns like "minecraft" and "toyota" in their original form. Translate the tag now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_translator/base.rb b/app/services/discourse_translator/base.rb index 2b5e7c4c..bb8ef1d8 100644 --- a/app/services/discourse_translator/base.rb +++ b/app/services/discourse_translator/base.rb @@ -139,6 +139,8 @@ def self.get_untranslated(translatable, raw: false) translatable.title when "Category" translatable.name + when "Tag" + translatable.name end end end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb index 594fc10a..6e6489a2 100644 --- a/app/services/discourse_translator/discourse_ai.rb +++ b/app/services/discourse_translator/discourse_ai.rb @@ -47,6 +47,8 @@ def self.translate!(translatable, target_locale_sym = I18n.locale) text_for_translation(translatable), language, ).translate + when "Tag" + ::DiscourseAi::TagTranslator.new(text_for_translation(translatable), language).translate end DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated) diff --git a/db/migrate/20250401022618_create_tag_translation_table.rb b/db/migrate/20250401022618_create_tag_translation_table.rb new file mode 100644 index 00000000..e1eb96b3 --- /dev/null +++ b/db/migrate/20250401022618_create_tag_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateTagTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_tag_locales do |t| + t.integer :tag_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_tag_translations do |t| + t.integer :tag_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_tag_translations, + %i[tag_id locale], + unique: true, + name: "idx_tag_translations_on_tag_id_and_locale" + end +end diff --git a/lib/discourse_translator/extensions/tag_extension.rb b/lib/discourse_translator/extensions/tag_extension.rb new file mode 100644 index 00000000..35c9fc8d --- /dev/null +++ b/lib/discourse_translator/extensions/tag_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module TagExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index 3b22b88d..6cb5d277 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -88,6 +88,24 @@ def inject(plugin) serializer.object.translation_for(InlineTranslation.effective_locale).presence end end + + # tags + + plugin.register_modifier(:topic_tags_serializer_name) do |tags, serializer| + # %w[topics tags serializer name] + end + + plugin.register_modifier(:sidebar_tag_serializer_name) do |name, serializer| + if !SiteSetting.experimental_inline_translation || + serializer.object.locale_matches?(InlineTranslation.effective_locale) || + serializer.scope&.request&.params&.[]("show") == "original" + name + else + serializer.object.translation_for(InlineTranslation.effective_locale).presence + end + end + + # plugin.register_modifier(:tag_serializer_name) { |name, serializer| "tag_serializer_name" } end end end diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb index 0dbf519f..65879f3f 100644 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ b/lib/discourse_translator/translated_content_normalizer.rb @@ -10,6 +10,8 @@ def self.normalize(translatable, content) PrettyText.cleanup(content, {}) when "Category" content + when "Tag" + content end end end diff --git a/plugin.rb b/plugin.rb index 1f1475fc..2ac568b0 100644 --- a/plugin.rb +++ b/plugin.rb @@ -30,6 +30,7 @@ module ::DiscourseTranslator Post.prepend(DiscourseTranslator::Extensions::PostExtension) Topic.prepend(DiscourseTranslator::Extensions::TopicExtension) Category.prepend(DiscourseTranslator::Extensions::CategoryExtension) + Tag.prepend(DiscourseTranslator::Extensions::TagExtension) TopicViewSerializer.prepend(DiscourseTranslator::Extensions::TopicViewSerializerExtension) end From 0e8c3e24f49256dfed48a24e3c353e07223c7450 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:29:00 +0800 Subject: [PATCH 3/6] Hide translations first due to cache issue --- .../inline_translation.rb | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index 6cb5d277..ad4bb14d 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -76,36 +76,6 @@ def inject(plugin) plugin.register_topic_preloader_associations(:translations) do SiteSetting.translator_enabled && SiteSetting.experimental_inline_translation end - - # categories - - plugin.register_modifier(:site_category_serializer_name) do |name, serializer| - if !SiteSetting.experimental_inline_translation || - serializer.object.locale_matches?(InlineTranslation.effective_locale) || - serializer.scope&.request&.params&.[]("show") == "original" - name - else - serializer.object.translation_for(InlineTranslation.effective_locale).presence - end - end - - # tags - - plugin.register_modifier(:topic_tags_serializer_name) do |tags, serializer| - # %w[topics tags serializer name] - end - - plugin.register_modifier(:sidebar_tag_serializer_name) do |name, serializer| - if !SiteSetting.experimental_inline_translation || - serializer.object.locale_matches?(InlineTranslation.effective_locale) || - serializer.scope&.request&.params&.[]("show") == "original" - name - else - serializer.object.translation_for(InlineTranslation.effective_locale).presence - end - end - - # plugin.register_modifier(:tag_serializer_name) { |name, serializer| "tag_serializer_name" } end end end From 84b4ea60a8983efa5e6162aca93f2d0e0e30809e Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:32:43 +0800 Subject: [PATCH 4/6] Annotate --- .../discourse_translator/category_locale.rb | 11 +++++++++++ .../discourse_translator/category_translation.rb | 16 ++++++++++++++++ app/models/discourse_translator/tag_locale.rb | 11 +++++++++++ .../discourse_translator/tag_translation.rb | 16 ++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/app/models/discourse_translator/category_locale.rb b/app/models/discourse_translator/category_locale.rb index e1dbde34..9344291e 100644 --- a/app/models/discourse_translator/category_locale.rb +++ b/app/models/discourse_translator/category_locale.rb @@ -10,3 +10,14 @@ class CategoryLocale < ActiveRecord::Base validates :detected_locale, presence: true end end + +# == Schema Information +# +# Table name: discourse_translator_category_locales +# +# id :bigint not null, primary key +# category_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/category_translation.rb b/app/models/discourse_translator/category_translation.rb index a769ff37..ec70a575 100644 --- a/app/models/discourse_translator/category_translation.rb +++ b/app/models/discourse_translator/category_translation.rb @@ -12,3 +12,19 @@ class CategoryTranslation < ActiveRecord::Base validates :locale, uniqueness: { scope: :category_id } end end + +# == Schema Information +# +# Table name: discourse_translator_category_translations +# +# id :bigint not null, primary key +# category_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_category_translations_on_category_id_and_locale (category_id,locale) UNIQUE +# diff --git a/app/models/discourse_translator/tag_locale.rb b/app/models/discourse_translator/tag_locale.rb index 716336b0..d0719a72 100644 --- a/app/models/discourse_translator/tag_locale.rb +++ b/app/models/discourse_translator/tag_locale.rb @@ -10,3 +10,14 @@ class TagLocale < ActiveRecord::Base validates :detected_locale, presence: true end end + +# == Schema Information +# +# Table name: discourse_translator_tag_locales +# +# id :bigint not null, primary key +# tag_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/tag_translation.rb b/app/models/discourse_translator/tag_translation.rb index 79d3fefd..02c62638 100644 --- a/app/models/discourse_translator/tag_translation.rb +++ b/app/models/discourse_translator/tag_translation.rb @@ -12,3 +12,19 @@ class TagTranslation < ActiveRecord::Base validates :locale, uniqueness: { scope: :tag_id } end end + +# == Schema Information +# +# Table name: discourse_translator_tag_translations +# +# id :bigint not null, primary key +# tag_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_tag_translations_on_tag_id_and_locale (tag_id,locale) UNIQUE +# From 298f83448d4d8f2f2ee36daaa1b9ab4046bc97cd Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:37:05 +0800 Subject: [PATCH 5/6] Remove stray comment --- spec/jobs/automatic_translation_backfill_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 06cee858..bdae4dc0 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -156,7 +156,6 @@ def expect_google_translate(text) Category.all.each { |c| c.set_detected_locale("es") } expect_google_translate("hallo") - # .times(6) described_class.new.execute From b9a8d19e5c166ce295d3c5cb3fc7592eff35af62 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:53:06 +0800 Subject: [PATCH 6/6] Update tests for tag --- .../automatic_translation_backfill_spec.rb | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index bdae4dc0..9d3d01e8 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -42,6 +42,7 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "de" SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) @@ -67,19 +68,22 @@ def expect_google_translate(text) end it "backfills both topics and posts" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end post = Fabricate(:post) topic = post.topic topic.set_detected_locale("de") post.set_detected_locale("es") - expect_google_translate("hola") - expect_google_translate("hallo") + expect_google_translate("xx") described_class.new.execute - expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es xx]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de xx]]) end it "backfills only public content when limit_to_public_content is true" do @@ -95,20 +99,27 @@ def expect_google_translate(text) private_topic.set_detected_locale("de") private_post.set_detected_locale("es") + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end expect_google_translate("hola") - expect_google_translate("hallo") SiteSetting.automatic_translation_backfill_limit_to_public_content = true described_class.new.execute expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hola]]) expect(private_topic.translations).to eq([]) expect(private_post.translations).to eq([]) end it "translate only content newer than automatic_translation_backfill_max_age_days" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end old_post = Fabricate(:post) old_topic = old_post.topic new_post = Fabricate(:post) @@ -146,14 +157,16 @@ def expect_google_translate(text) expect_google_check_language end - it "backfills all (1) category (1) topic (4) posts as it is within the maximum per job run" do + it "backfills all (1) topic (4) posts (1) category (1) tag as it is within the maximum per job run" do category = Fabricate(:category) + tag = Fabricate(:tag) topic = Fabricate(:topic, category: category) posts = Fabricate.times(4, :post, topic: topic) topic.set_detected_locale("es") posts.each { |p| p.set_detected_locale("es") } Category.all.each { |c| c.set_detected_locale("es") } + tag.set_detected_locale("es") expect_google_translate("hallo") @@ -164,6 +177,7 @@ def expect_google_translate(text) [%w[de hallo]] * 4, ) expect(category.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(tag.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) end end end