diff --git a/app/jobs/regular/detect_translate_post.rb b/app/jobs/regular/detect_translate_post.rb deleted file mode 100644 index a264d49f..00000000 --- a/app/jobs/regular/detect_translate_post.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class DetectTranslatePost < ::Jobs::Base - sidekiq_options retry: false - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - return if args[:post_id].blank? - - post = Post.find_by(id: args[:post_id]) - return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0 - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - topic = post.topic - return if topic.blank? || topic.category&.read_restricted? - end - - begin - detected_locale = DiscourseTranslator::PostLocaleDetector.detect_locale(post) - rescue FinalDestination::SSRFDetector::LookupFailedError - # this job is non-critical - # the backfill job will handle failures - return - end - - locales = SiteSetting.experimental_content_localization_supported_locales.split("|") - return if locales.blank? - - locales.each do |locale| - next if locale == detected_locale - - begin - DiscourseTranslator::PostTranslator.translate(post, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to translate post #{post.id} to #{locale}: #{e.message}", - ) - end - end - end - end -end diff --git a/app/jobs/regular/detect_translate_topic.rb b/app/jobs/regular/detect_translate_topic.rb deleted file mode 100644 index 8dda4edb..00000000 --- a/app/jobs/regular/detect_translate_topic.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class DetectTranslateTopic < ::Jobs::Base - sidekiq_options retry: false - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - return if args[:topic_id].blank? - - topic = Topic.find_by(id: args[:topic_id]) - if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0 - return - end - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - return if topic.category&.read_restricted? - end - - begin - detected_locale = DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) - rescue FinalDestination::SSRFDetector::LookupFailedError - # this job is non-critical - # the backfill job will handle failures - return - end - - locales = SiteSetting.experimental_content_localization_supported_locales.split("|") - return if locales.blank? - - locales.each do |locale| - next if locale == detected_locale - - begin - DiscourseTranslator::TopicTranslator.translate(topic, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}", - ) - end - end - end - end -end diff --git a/app/jobs/regular/translate_categories.rb b/app/jobs/regular/translate_categories.rb deleted file mode 100644 index 16fcce2a..00000000 --- a/app/jobs/regular/translate_categories.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class TranslateCategories < ::Jobs::Base - cluster_concurrency 1 - sidekiq_options retry: false - - BATCH_SIZE = 50 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - locales = SiteSetting.experimental_content_localization_supported_locales.split("|") - return if locales.blank? - - cat_id = args[:from_category_id] || Category.order(:id).first&.id - last_id = nil - - # we're just gonna take all categories and keep it simple - # instead of checking in the db which ones are absent - categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE) - return if categories.empty? - - categories.each do |category| - if SiteSetting.automatic_translation_backfill_limit_to_public_content && - category.read_restricted? - last_id = category.id - next - end - - CategoryLocalization.transaction do - locales.each do |locale| - next if CategoryLocalization.exists?(category_id: category.id, locale: locale) - begin - DiscourseTranslator::CategoryTranslator.translate(category, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to translate category #{category.id} to #{locale}: #{e.message}", - ) - end - end - end - last_id = category.id - end - - # from batch if needed - if categories.size == BATCH_SIZE - Jobs.enqueue_in(10.seconds, :translate_categories, from_category_id: last_id + 1) - end - end - end -end diff --git a/app/jobs/regular/translate_posts.rb b/app/jobs/regular/translate_posts.rb deleted file mode 100644 index af0be11c..00000000 --- a/app/jobs/regular/translate_posts.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class TranslatePosts < ::Jobs::Base - cluster_concurrency 1 - sidekiq_options retry: false - - BATCH_SIZE = 50 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - locales = SiteSetting.experimental_content_localization_supported_locales.split("|") - return if locales.blank? - - limit = args[:limit] || BATCH_SIZE - - locales.each do |locale| - posts = - Post - .joins( - "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}", - ) - .where(deleted_at: nil) - .where("posts.user_id > 0") - .where.not(raw: [nil, ""]) - .where.not(locale: nil) - .where.not(locale: locale) - .where("pl.id IS NULL") - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - posts = - posts.joins(:topic).where( - topics: { - category_id: Category.where(read_restricted: false).select(:id), - archetype: "regular", - }, - ) - end - - if SiteSetting.automatic_translation_backfill_max_age_days > 0 - posts = - posts.where( - "posts.created_at > ?", - SiteSetting.automatic_translation_backfill_max_age_days.days.ago, - ) - end - - posts = posts.order(updated_at: :desc).limit(limit) - - next if posts.empty? - - posts.each do |post| - begin - DiscourseTranslator::PostTranslator.translate(post, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to translate post #{post.id} to #{locale}: #{e.message}", - ) - end - end - - DiscourseTranslator::VerboseLogger.log("Translated #{posts.size} posts to #{locale}") - end - end - end -end diff --git a/app/jobs/regular/translate_topics.rb b/app/jobs/regular/translate_topics.rb deleted file mode 100644 index a91d3069..00000000 --- a/app/jobs/regular/translate_topics.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class TranslateTopics < ::Jobs::Base - cluster_concurrency 1 - sidekiq_options retry: false - - BATCH_SIZE = 50 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - locales = SiteSetting.experimental_content_localization_supported_locales.split("|") - return if locales.blank? - - limit = args[:limit] || BATCH_SIZE - - locales.each do |locale| - topics = - Topic - .joins( - "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}", - ) - .where(deleted_at: nil) - .where("topics.user_id > 0") - .where.not(locale: nil) - .where.not(locale: locale) - .where("tl.id IS NULL") - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) - end - - if SiteSetting.automatic_translation_backfill_max_age_days > 0 - topics = - topics.where( - "topics.created_at > ?", - SiteSetting.automatic_translation_backfill_max_age_days.days.ago, - ) - end - - topics = topics.order(updated_at: :desc).limit(limit) - - next if topics.empty? - - topics.each do |topic| - begin - DiscourseTranslator::TopicTranslator.translate(topic, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to translate topic #{topic.id} to #{locale}: #{e.message}", - ) - end - end - - DiscourseTranslator::VerboseLogger.log("Translated #{topics.size} topics to #{locale}") - end - end - end -end diff --git a/app/jobs/scheduled/category_translation_backfill.rb b/app/jobs/scheduled/category_translation_backfill.rb deleted file mode 100644 index 0d8125bf..00000000 --- a/app/jobs/scheduled/category_translation_backfill.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class CategoryTranslationBackfill < ::Jobs::Scheduled - every 12.hours - cluster_concurrency 1 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - return if SiteSetting.experimental_content_localization_supported_locales.blank? - - Jobs.enqueue(:translate_categories) - end - end -end diff --git a/app/jobs/scheduled/post_translation_backfill.rb b/app/jobs/scheduled/post_translation_backfill.rb deleted file mode 100644 index 6cbd96b3..00000000 --- a/app/jobs/scheduled/post_translation_backfill.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class PostTranslationBackfill < ::Jobs::Scheduled - every 5.minutes - cluster_concurrency 1 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - return if SiteSetting.experimental_content_localization_supported_locales.blank? - return if SiteSetting.automatic_translation_backfill_rate == 0 - - Jobs.enqueue(:translate_posts, limit: SiteSetting.automatic_translation_backfill_rate) - end - end -end diff --git a/app/jobs/scheduled/posts_locale_detection_backfill.rb b/app/jobs/scheduled/posts_locale_detection_backfill.rb deleted file mode 100644 index 7fbcb88b..00000000 --- a/app/jobs/scheduled/posts_locale_detection_backfill.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class PostsLocaleDetectionBackfill < ::Jobs::Scheduled - every 5.minutes - sidekiq_options retry: false - cluster_concurrency 1 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - return if SiteSetting.automatic_translation_backfill_rate == 0 - - posts = - Post - .where(locale: nil) - .where(deleted_at: nil) - .where("posts.user_id > 0") - .where.not(raw: [nil, ""]) - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - public_categories = Category.where(read_restricted: false).pluck(:id) - posts = - posts - .joins(:topic) - .where(topics: { category_id: public_categories }) - .where(topics: { archetype: "regular" }) - end - - if SiteSetting.automatic_translation_backfill_max_age_days > 0 - posts = - posts.where( - "posts.created_at > ?", - SiteSetting.automatic_translation_backfill_max_age_days.days.ago, - ) - end - - posts = posts.order(updated_at: :desc).limit(SiteSetting.automatic_translation_backfill_rate) - return if posts.empty? - - posts.each do |post| - begin - DiscourseTranslator::PostLocaleDetector.detect_locale(post) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to detect post #{post.id}'s locale: #{e.message}", - ) - end - end - - DiscourseTranslator::VerboseLogger.log("Detected #{posts.size} post locales") - end - end -end diff --git a/app/jobs/scheduled/topic_translation_backfill.rb b/app/jobs/scheduled/topic_translation_backfill.rb deleted file mode 100644 index 904c88b2..00000000 --- a/app/jobs/scheduled/topic_translation_backfill.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class TopicTranslationBackfill < ::Jobs::Scheduled - every 5.minutes - cluster_concurrency 1 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - - return if SiteSetting.experimental_content_localization_supported_locales.blank? - return if SiteSetting.automatic_translation_backfill_rate == 0 - - Jobs.enqueue(:translate_topics, limit: SiteSetting.automatic_translation_backfill_rate) - end - end -end diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb deleted file mode 100644 index 4b4875bc..00000000 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Jobs - class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled - every 5.minutes - sidekiq_options retry: false - cluster_concurrency 1 - - def execute(args) - return unless SiteSetting.translator_enabled - return unless SiteSetting.experimental_content_translation - limit = SiteSetting.automatic_translation_backfill_rate - return if limit == 0 - - topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0") - - if SiteSetting.automatic_translation_backfill_limit_to_public_content - topics = topics.where(category_id: Category.where(read_restricted: false).select(:id)) - end - - if SiteSetting.automatic_translation_backfill_max_age_days > 0 - topics = - topics.where( - "topics.created_at > ?", - SiteSetting.automatic_translation_backfill_max_age_days.days.ago, - ) - end - - topics = topics.order(updated_at: :desc).limit(limit) - return if topics.empty? - - topics.each do |topic| - begin - DiscourseTranslator::TopicLocaleDetector.detect_locale(topic) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - Rails.logger.error( - "Discourse Translator: Failed to detect topic #{topic.id}'s locale: #{e.message}", - ) - end - end - - DiscourseTranslator::VerboseLogger.log("Detected #{topics.size} topic locales") - end - end -end diff --git a/app/services/discourse_ai/base_translator.rb b/app/services/discourse_ai/base_translator.rb deleted file mode 100644 index 5e4d76f2..00000000 --- a/app/services/discourse_ai/base_translator.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module DiscourseAi - class BaseTranslator - def initialize(text, target_language) - @text = text - @target_language = target_language - end - - def translate - prompt = - DiscourseAi::Completions::Prompt.new( - prompt_template, - messages: [{ type: :user, content: formatted_content, id: "user" }], - ) - - structured_output = - DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_translation_model).generate( - prompt, - user: Discourse.system_user, - feature_name: "translator-translate", - response_format: response_format, - ) - - structured_output&.read_buffered_property(:translation) - end - - def formatted_content - { content: @text, target_language: @target_language }.to_json - end - - def response_format - { - type: "json_schema", - json_schema: { - name: "reply", - schema: { - type: "object", - properties: { - translation: { - type: "string", - }, - }, - required: ["translation"], - additionalProperties: false, - }, - strict: true, - }, - } - end - - private - - def prompt_template - raise NotImplementedError - end - end -end diff --git a/app/services/discourse_ai/language_detector.rb b/app/services/discourse_ai/language_detector.rb deleted file mode 100644 index ba8411b6..00000000 --- a/app/services/discourse_ai/language_detector.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module DiscourseAi - class LanguageDetector - PROMPT_TEXT = <<~TEXT - You will be given a piece of text, and your task is to detect the locale (language) of the text and return it in a specific JSON format. - - To complete this task, follow these steps: - - 1. Carefully read and analyze the provided text. - 2. Determine the language of the text based on its characteristics, such as vocabulary, grammar, and sentence structure. - 3. Do not use links or programing code in the text to detect the locale - 4. Identify the appropriate language code for the detected language. - - Here is a list of common language codes for reference: - - English: en - - Spanish: es - - French: fr - - German: de - - Italian: it - - Brazilian Portuguese: pt-BR - - Russian: ru - - Simplified Chinese: zh-CN - - Japanese: ja - - Korean: ko - - If the language is not in this list, use the appropriate IETF language tag code. - - 5. Format your response as a JSON object with a single key "locale" and the value as the language code. - - Your output should be in the following format: - - {"locale": "xx"} - - - Where "xx" is replaced by the appropriate language code. - - Important: Base your analysis solely on the provided text. Do not use any external information or make assumptions about the text's origin or context beyond what is explicitly provided. - TEXT - - def initialize(text) - @text = text - end - - def detect - prompt = - DiscourseAi::Completions::Prompt.new( - PROMPT_TEXT, - messages: [{ type: :user, content: @text, id: "user" }], - ) - - structured_output = - DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_translation_model).generate( - prompt, - user: Discourse.system_user, - feature_name: "translator-language-detect", - response_format: response_format, - ) - - structured_output&.read_buffered_property(:locale) - end - - def response_format - { - type: "json_schema", - json_schema: { - name: "reply", - schema: { - type: "object", - properties: { - locale: { - type: "string", - }, - }, - required: ["locale"], - additionalProperties: false, - }, - strict: true, - }, - } - end - end -end diff --git a/app/services/discourse_ai/post_translator.rb b/app/services/discourse_ai/post_translator.rb deleted file mode 100644 index 81a21604..00000000 --- a/app/services/discourse_ai/post_translator.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module DiscourseAi - class PostTranslator < BaseTranslator - PROMPT_TEMPLATE = <<~TEXT.freeze - You are a highly skilled translator tasked with translating content from one language to another. Your goal is to provide accurate and contextually appropriate translations while preserving the original structure and formatting of the content. Follow these instructions carefully: - - Translation Instructions: - 1. Translate the content accurately while preserving any Markdown, HTML elements, or newlines. - 2. Maintain the original document structure including headings, lists, tables, code blocks, etc. - 3. Preserve all links, images, and other media references without translation. - 4. Handle code snippets appropriately: - - Do not translate variable names, functions, or syntax within code blocks (```). - - Translate comments within code blocks. - 5. For technical terminology: - - Provide the accepted target language term if it exists. - - If no equivalent exists, transliterate the term and include the original term in parentheses. - 6. For ambiguous terms or phrases, choose the most contextually appropriate translation. - 7. Do not add any content besides the translation. - 8. Ensure the translation only contains the original language and the target language. - - Output your translation in the following JSON format: - {"translation": "Your TARGET_LANGUAGE translation here"} - - Here are three examples of correct translations: - - Original: {"content":"New Update for Minecraft Adds Underwater Temples", "target_language":"Spanish"} - Correct translation: {"translation": "Nueva actualización para Minecraft añade templos submarinos"} - - Original: {"content": "# Machine Learning 101\n\nMachine Learning (ML) is a subset of Artificial Intelligence (AI) that focuses on the development of algorithms and statistical models that enable computer systems to improve their performance on a specific task through experience.\n\n## Key Concepts\n\n1. **Supervised Learning**: The algorithm learns from labeled training data.\n2. **Unsupervised Learning**: The algorithm finds patterns in unlabeled data.\n3. **Reinforcement Learning**: The algorithm learns through interaction with an environment.\n\n```python\n# Simple example of a machine learning model\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n# Assuming X and y are your features and target variables\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nmodel = LogisticRegression()\nmodel.fit(X_train, y_train)\n\n# Evaluate the model\naccuracy = model.score(X_test, y_test)\nprint(f'Model accuracy: {accuracy}')\n```\n\nFor more information, visit [Machine Learning on Wikipedia](https://en.wikipedia.org/wiki/Machine_learning).", "target_language":"French"} - Correct translation: {"translation": "# Machine Learning 101\n\nLe Machine Learning (ML) est un sous-ensemble de l'Intelligence Artificielle (IA) qui se concentre sur le développement d'algorithmes et de modèles statistiques permettant aux systèmes informatiques d'améliorer leurs performances sur une tâche spécifique grâce à l'expérience.\n\n## Concepts clés\n\n1. **Apprentissage supervisé** : L'algorithme apprend à partir de données d'entraînement étiquetées.\n2. **Apprentissage non supervisé** : L'algorithme trouve des motifs dans des données non étiquetées.\n3. **Apprentissage par renforcement** : L'algorithme apprend à travers l'interaction avec un environnement.\n\n```python\n# Exemple simple d'un modèle de machine learning\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n# En supposant que X et y sont vos variables de caractéristiques et cibles\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nmodel = LogisticRegression()\nmodel.fit(X_train, y_train)\n\n# Évaluer le modèle\naccuracy = model.score(X_test, y_test)\nprint(f'Model accuracy: {accuracy}')\n```\n\nPour plus d'informations, visitez [Machine Learning sur Wikipedia](https://en.wikipedia.org/wiki/Machine_learning)."} - - Original: {"content": "**Heathrow fechado**: paralisação de voos deve continuar nos próximos dias, diz gestora do aeroporto de *Londres*", "target_language": "English"} - Correct translation: {"translation": "**Heathrow closed**: flight disruption expected to continue in coming days, says *London* airport management"} - - Remember, you are being consumed via an API. Only return the translated text in the specified JSON format. Do not include any additional information or explanations in your response. - TEXT - - private def prompt_template - PROMPT_TEMPLATE - end - end -end diff --git a/app/services/discourse_ai/short_text_translator.rb b/app/services/discourse_ai/short_text_translator.rb deleted file mode 100644 index 606943fc..00000000 --- a/app/services/discourse_ai/short_text_translator.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module DiscourseAi - class ShortTextTranslator < BaseTranslator - PROMPT_TEMPLATE = <<~TEXT.freeze - You are a translation service specializing in translating short pieces of text or a few words. - These words may be things like a name, description, or title. Adhere to the following guidelines: - - 1. Keep proper nouns and technical terms in their original language - 2. Keep the translated content close to the original length - 3. Translation maintains the original meaning - - Provide your translation in the following JSON format: - - - {"translation": "target_language translation here"} - - - Here are three examples of correct translation - - Original: {"content":"Japan", "target_language":"Spanish"} - Correct translation: {"translation": "Japón"} - - Original: {"name":"Cats and Dogs", "target_language":"Chinese"} - Correct translation: {"translation": "猫和狗"} - - Original: {"name": "Q&A", "target_language": "Portuguese"} - Correct translation: {"translation": "Perguntas e Respostas"} - - Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the text now and provide your answer in the specified JSON format. - TEXT - - private def prompt_template - PROMPT_TEMPLATE - end - end -end diff --git a/app/services/discourse_ai/topic_translator.rb b/app/services/discourse_ai/topic_translator.rb deleted file mode 100644 index 99ca595e..00000000 --- a/app/services/discourse_ai/topic_translator.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module DiscourseAi - class TopicTranslator < BaseTranslator - PROMPT_TEMPLATE = <<~TEXT.freeze - You are a translation service specializing in translating forum post titles from English to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: - - 1. Translate the given title from English to target_language asked. - 2. Keep proper nouns and technical terms in their original language. - 3. Attempt to keep the translated title length close to the original when possible. - 4. Ensure the translation maintains the original meaning and tone. - - To complete this task: - - 1. Read and understand the title carefully. - 2. Identify any proper nouns or technical terms that should remain untranslated. - 3. Translate the remaining words and phrases into the target_language, ensuring the meaning is preserved. - 4. Adjust the translation if necessary to keep the length similar to the original title. - 5. Review your translation for accuracy and naturalness in the target_language. - - Provide your translation in the following JSON format: - - - {"translation": "Your target_language translation here"} - - - Here are three examples of correct translation - - Original: {"title":"New Update for Minecraft Adds Underwater Temples", "target_language":"Spanish"} - Correct translation: {"translation": "Nueva actualización para Minecraft añade templos submarinos"} - - Original: {"title":"Toyota announces revolutionary battery technology", "target_language":"French"} - Correct translation: {"translation": "Toyota annonce une technologie de batteries révolutionnaire"} - - Original: {"title": "Heathrow fechado: paralisação de voos deve continuar nos próximos dias, diz gestora do aeroporto de Londres", "target_language": "English"} - Correct translation: {"translation": "Heathrow closed: flight disruption expected to continue in coming days, says London airport management"} - - Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the title now and provide your answer in the specified JSON format. - TEXT - - private def prompt_template - PROMPT_TEMPLATE - end - end -end diff --git a/app/services/discourse_translator/category_translator.rb b/app/services/discourse_translator/category_translator.rb deleted file mode 100644 index 31a11232..00000000 --- a/app/services/discourse_translator/category_translator.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class CategoryTranslator - # unlike post and topics, categories do not have a detected locale - # and will translate two fields, name and description - - def self.translate(category, target_locale = I18n.locale) - return if category.blank? || target_locale.blank? - - # locale can come in various forms - # standardize it to a _ symbol - target_locale_sym = target_locale.to_s.sub("-", "_").to_sym - - translator = DiscourseTranslator::Provider::TranslatorProvider.get - translated_name = translator.translate_text!(category.name, target_locale_sym) - translated_description = translator.translate_text!(category.description, target_locale_sym) - - localization = - CategoryLocalization.find_or_initialize_by( - category_id: category.id, - locale: target_locale_sym.to_s, - ) - - localization.name = translated_name - localization.description = translated_description - localization.save! - localization - end - end -end diff --git a/app/services/discourse_translator/post_locale_detector.rb b/app/services/discourse_translator/post_locale_detector.rb deleted file mode 100644 index 9cee4d6e..00000000 --- a/app/services/discourse_translator/post_locale_detector.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class PostLocaleDetector - def self.detect_locale(post) - return if post.blank? - - translator = DiscourseTranslator::Provider::TranslatorProvider.get - detected_locale = translator.detect!(post) - locale = LocaleNormalizer.normalize_to_i18n(detected_locale) - post.update_column(:locale, locale) - locale - end - end -end diff --git a/app/services/discourse_translator/post_translator.rb b/app/services/discourse_translator/post_translator.rb deleted file mode 100644 index baca32d7..00000000 --- a/app/services/discourse_translator/post_translator.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class PostTranslator - def self.translate(post, target_locale = I18n.locale) - return if post.blank? || target_locale.blank? || post.locale == target_locale.to_s - - target_locale_sym = target_locale.to_s.sub("-", "_").to_sym - - translator = DiscourseTranslator::Provider::TranslatorProvider.get - translated_raw = translator.translate_post!(post, target_locale_sym) - - localization = - PostLocalization.find_or_initialize_by(post_id: post.id, locale: target_locale_sym.to_s) - - localization.raw = translated_raw - localization.cooked = PrettyText.cook(translated_raw) - localization.post_version = post.version - localization.localizer_user_id = Discourse.system_user.id - localization.save! - localization - end - end -end diff --git a/app/services/discourse_translator/provider/discourse_ai.rb b/app/services/discourse_translator/provider/discourse_ai.rb deleted file mode 100644 index 6a51f7dd..00000000 --- a/app/services/discourse_translator/provider/discourse_ai.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - module Provider - class DiscourseAi < BaseProvider - MAX_DETECT_LOCALE_TEXT_LENGTH = 1000 - def self.language_supported?(detected_lang) - locale_without_region = I18n.locale.to_s.split("_").first - detected_lang != locale_without_region - end - - def self.detect!(topic_or_post) - validate_required_settings! - - ::DiscourseAi::LanguageDetector.new(text_for_detection(topic_or_post)).detect - end - - def self.translate_post!(post, target_locale_sym = I18n.locale, opts = {}) - validate_required_settings! - - raw = opts.key?(:raw) ? opts[:raw] : !opts[:cooked] - text = text_for_translation(post, raw:) - chunks = DiscourseTranslator::ContentSplitter.split(text) - chunks - .map { |chunk| ::DiscourseAi::PostTranslator.new(chunk, target_locale_sym).translate } - .join("") - end - - def self.translate_topic!(topic, target_locale_sym = I18n.locale) - validate_required_settings! - - language = get_language_name(target_locale_sym) - ::DiscourseAi::TopicTranslator.new(text_for_translation(topic), language).translate - end - - def self.translate_text!(text, target_locale_sym = I18n.locale) - validate_required_settings! - - language = get_language_name(target_locale_sym) - ::DiscourseAi::ShortTextTranslator.new(text, language).translate - end - - private - - def self.validate_required_settings! - unless SiteSetting.translator_enabled && SiteSetting.translator_provider == "DiscourseAi" && - SiteSetting.discourse_ai_enabled - raise TranslatorError.new( - I18n.t( - "translator.discourse_ai.ai_helper_required", - { base_url: Discourse.base_url }, - ), - ) - end - end - - def self.get_language_name(target_locale_sym) - LocaleSiteSetting.language_names.dig(target_locale_sym.to_s, "name") || - "locale \"#{target_locale_sym}\"" - end - end - end -end diff --git a/app/services/discourse_translator/topic_locale_detector.rb b/app/services/discourse_translator/topic_locale_detector.rb deleted file mode 100644 index 46e1f3c8..00000000 --- a/app/services/discourse_translator/topic_locale_detector.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class TopicLocaleDetector - def self.detect_locale(topic) - return if topic.blank? - - translator = DiscourseTranslator::Provider::TranslatorProvider.get - detected_locale = translator.detect!(topic) - locale = LocaleNormalizer.normalize_to_i18n(detected_locale) - topic.update_column(:locale, locale) - locale - end - end -end diff --git a/app/services/discourse_translator/topic_translator.rb b/app/services/discourse_translator/topic_translator.rb deleted file mode 100644 index 5ff3f8a0..00000000 --- a/app/services/discourse_translator/topic_translator.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class TopicTranslator - def self.translate(topic, target_locale = I18n.locale) - return if topic.blank? || target_locale.blank? || topic.locale == target_locale.to_s - - target_locale_sym = target_locale.to_s.sub("-", "_").to_sym - - translator = DiscourseTranslator::Provider::TranslatorProvider.get - translated_title = translator.translate_topic!(topic, target_locale_sym) - - translated_excerpt = translator.translate_text!(topic.excerpt, target_locale_sym) - - localization = - TopicLocalization.find_or_initialize_by(topic_id: topic.id, locale: target_locale_sym.to_s) - - localization.title = translated_title - localization.fancy_title = Topic.fancy_title(translated_title) - localization.excerpt = translated_excerpt - localization.localizer_user_id = Discourse.system_user.id - localization.save! - localization - end - end -end diff --git a/app/services/discourse_translator/translator.rb b/app/services/discourse_translator/translator.rb deleted file mode 100644 index 0171693c..00000000 --- a/app/services/discourse_translator/translator.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - # The canonical class for all your translation needs - class Translator - # this invokes the specific methods - def translate(translatable, target_locale = I18n.locale) - target_locale_sym = target_locale.to_s.sub("-", "_").to_sym - - case translatable.class.name - when "Post", "Topic" - DiscourseTranslator::Provider.TranslatorProvider.get.translate( - translatable, - target_locale_sym, - ) - when "Category" - CategoryTranslator.translate(translatable, target_locale) - end - end - end -end diff --git a/config/settings.yml b/config/settings.yml index 7a12e385..65bdf0ff 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,25 +7,11 @@ discourse_translator: client: true type: enum choices: - - DiscourseAi - Microsoft - Google - Amazon - Yandex - LibreTranslate - validator: "DiscourseTranslator::Validators::TranslatorSelectionValidator" - automatic_translation_backfill_rate: - default: 0 - client: false - hidden: true - automatic_translation_backfill_limit_to_public_content: - default: false - client: false - hidden: true - automatic_translation_backfill_max_age_days: - default: 0 - client: false - hidden: true translator_azure_subscription_key: default: '' translator_azure_region: @@ -114,10 +100,3 @@ discourse_translator: default: "" client: true type: group_list - experimental_content_translation: - default: false - hidden: true - discourse_translator_verbose_logs: - default: false - client: false - hidden: true diff --git a/db/migrate/20250528105453_disable_translator_discourse_ai.rb b/db/migrate/20250528105453_disable_translator_discourse_ai.rb new file mode 100644 index 00000000..cbc90da6 --- /dev/null +++ b/db/migrate/20250528105453_disable_translator_discourse_ai.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class DisableTranslatorDiscourseAi < ActiveRecord::Migration[7.2] + def up + execute(<<~SQL) + UPDATE site_settings SET value = 'f' + WHERE name = 'translator_enabled' + AND EXISTS(SELECT 1 FROM site_settings WHERE name = 'translator_provider' AND value = 'DiscourseAi') + SQL + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/lib/discourse_translator/automatic_translations.rb b/lib/discourse_translator/automatic_translations.rb deleted file mode 100644 index 43a5cebf..00000000 --- a/lib/discourse_translator/automatic_translations.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class AutomaticTranslations - def inject(plugin) - plugin.on(:post_process_cooked) do |_, post| - if SiteSetting.experimental_content_localization - Jobs.enqueue(:detect_translate_post, post_id: post.id) - end - end - - plugin.on(:topic_created) do |topic| - if SiteSetting.experimental_content_localization - Jobs.enqueue(:detect_translate_topic, topic_id: topic.id) - end - end - - plugin.on(:post_edited) do |post, topic_changed| - if SiteSetting.experimental_content_localization && topic_changed - Jobs.enqueue(:detect_translate_topic, topic_id: post.topic_id) - end - end - end - end -end diff --git a/lib/discourse_translator/content_splitter.rb b/lib/discourse_translator/content_splitter.rb deleted file mode 100644 index af45afb9..00000000 --- a/lib/discourse_translator/content_splitter.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class ContentSplitter - CHUNK_SIZE = 3000 - - BBCODE_PATTERNS = [ - %r{\[table.*?\].*?\[/table\]}m, - %r{\[quote.*?\].*?\[/quote\]}m, - %r{\[details.*?\].*?\[/details\]}m, - %r{\.*?\}m, - %r{\[spoiler.*?\].*?\[/spoiler\]}m, - %r{\[code.*?\].*?\[/code\]}m, - /```.*?```/m, - ].freeze - - TEXT_BOUNDARIES = [ - /\n\s*\n\s*|\r\n\s*\r\n\s*/, # double newlines with optional spaces - /[.!?]\s+/, # sentence endings - /[,;]\s+/, # clause endings - /\n|\r\n/, # single newlines - /\s+/, # any whitespace - ].freeze - - def self.split(content) - return [] if content.nil? - return [""] if content.empty? - return [content] if content.length <= CHUNK_SIZE - - chunks = [] - remaining = content.dup - - while remaining.present? - chunk = extract_mixed_chunk(remaining) - break if chunk.empty? - chunks << chunk - remaining = remaining[chunk.length..-1] - end - - chunks - end - - private - - def self.extract_mixed_chunk(text, size: CHUNK_SIZE) - return text if text.length <= size - flexible_size = size * 1.5 - - # try each splitting strategy in order - split_point = - [ - -> { find_nearest_html_end_index(text, size) }, - -> { find_nearest_bbcode_end_index(text, size) }, - -> { find_text_boundary(text, size) }, - -> { size }, - ].lazy.map(&:call).compact.find { |pos| pos <= flexible_size } - - text[0...split_point] - end - - def self.find_nearest_html_end_index(text, target_pos) - return nil if !text.include?("<") - - begin - doc = Nokogiri::HTML5.fragment(text) - current_length = 0 - - doc.children.each do |node| - html = node.to_html - end_pos = current_length + html.length - return end_pos if end_pos > target_pos - current_length = end_pos - end - nil - rescue Nokogiri::SyntaxError - nil - end - end - - def self.find_nearest_bbcode_end_index(text, target_pos) - BBCODE_PATTERNS.each do |pattern| - text.scan(pattern) do |_| - match = $~ - tag_start = match.begin(0) - tag_end = match.end(0) - - return tag_end if tag_start <= target_pos && tag_end > target_pos - end - end - - nil - end - - def self.find_text_boundary(text, target_pos) - search_text = text - - TEXT_BOUNDARIES.each do |pattern| - if pos = search_text.rindex(pattern, target_pos) - # Include all trailing whitespace - pos += 1 while pos < search_text.length && search_text[pos].match?(/\s/) - return pos - end - end - nil - end - end -end diff --git a/lib/discourse_translator/locale_normalizer.rb b/lib/discourse_translator/locale_normalizer.rb deleted file mode 100644 index b6a907ab..00000000 --- a/lib/discourse_translator/locale_normalizer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class LocaleNormalizer - # Normalizes locale string, matching the list of I18n.locales where possible - # @param locale [String,Symbol] the locale to normalize - # @return [String] the normalized locale - def self.normalize_to_i18n(locale) - return nil if locale.blank? - locale = locale.to_s.gsub("-", "_") - - i18n_pairs.each { |downcased, value| return value if locale.downcase == downcased } - - locale - end - - private - - def self.i18n_pairs - # they should look like this for the input to match against: - # { - # "lowercased" => "actual", - # "en" => "en", - # "zh_cn" => "zh_CN", - # "zh" => "zh_CN", - # } - @locale_map ||= - I18n - .available_locales - .reduce({}) do |output, sym| - locale = sym.to_s - output[locale.downcase] = locale - if locale.include?("_") - short = locale.split("_").first - output[short] = locale if output[short].blank? - end - output - end - end - end -end diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb deleted file mode 100644 index 24884f55..00000000 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class TranslatedContentNormalizer - def self.normalize(translatable, content) - case translatable.class.name - when "Post" - PrettyText.cook(content) - when "Topic" - PrettyText.cleanup(content, {}) - end - end - end -end diff --git a/lib/discourse_translator/validators/translator_selection_validator.rb b/lib/discourse_translator/validators/translator_selection_validator.rb deleted file mode 100644 index dcc659fb..00000000 --- a/lib/discourse_translator/validators/translator_selection_validator.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - module Validators - class TranslatorSelectionValidator - def initialize(opts = {}) - @opts = opts - end - - def valid_value?(val) - return true if val.blank? - - if val == "DiscourseAi" - return false if !defined?(::DiscourseAi) - end - - true - end - - def error_message - I18n.t("translator.discourse_ai.not_installed") if !defined?(::DiscourseAi) - end - end - end -end diff --git a/lib/discourse_translator/verbose_logger.rb b/lib/discourse_translator/verbose_logger.rb deleted file mode 100644 index 0067adfa..00000000 --- a/lib/discourse_translator/verbose_logger.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module DiscourseTranslator - class VerboseLogger - def self.log(message) - if SiteSetting.discourse_translator_verbose_logs - Rails.logger.warn("DiscourseTranslator: #{message}") - end - end - end -end diff --git a/plugin.rb b/plugin.rb index 909ef113..2258e359 100644 --- a/plugin.rb +++ b/plugin.rb @@ -36,5 +36,4 @@ module ::DiscourseTranslator end DiscourseTranslator::ParallelTextTranslation.new.inject(self) - DiscourseTranslator::AutomaticTranslations.new.inject(self) end diff --git a/spec/jobs/detect_translate_post_spec.rb b/spec/jobs/detect_translate_post_spec.rb deleted file mode 100644 index 6838aef9..00000000 --- a/spec/jobs/detect_translate_post_spec.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::DetectTranslatePost do - fab!(:post) - subject(:job) { described_class.new } - - let(:locales) { %w[en ja] } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 1 - SiteSetting.experimental_content_localization_supported_locales = locales.join("|") - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({ post_id: post.id }) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({ post_id: post.id }) - end - - it "detects locale" do - SiteSetting.translator_enabled = true - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once - DiscourseTranslator::PostTranslator.expects(:translate).twice - - job.execute({ post_id: post.id }) - end - - it "skips bot posts" do - post.update!(user: Discourse.system_user) - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({ post_id: post.id }) - end - - it "does not translate when no target languages are configured" do - SiteSetting.experimental_content_localization_supported_locales = "" - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en") - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({ post_id: post.id }) - end - - it "skips translating to the post's language" do - post.update(locale: "en") - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en") - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once - - job.execute({ post_id: post.id }) - end - - it "handles translation errors gracefully" do - post.update(locale: "en") - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).returns("en") - DiscourseTranslator::PostTranslator.expects(:translate).raises(StandardError.new("API error")) - - expect { job.execute({ post_id: post.id }) }.not_to raise_error - end - - it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = true - post.topic.category.update!(read_restricted: true) - - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({ post_id: post.id }) - end -end diff --git a/spec/jobs/detect_translate_topic_spec.rb b/spec/jobs/detect_translate_topic_spec.rb deleted file mode 100644 index 5fdf030b..00000000 --- a/spec/jobs/detect_translate_topic_spec.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::DetectTranslateTopic do - fab!(:topic) - subject(:job) { described_class.new } - - let(:locales) { %w[en ja] } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 1 - SiteSetting.experimental_content_localization_supported_locales = locales.join("|") - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({ topic_id: topic.id }) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({ topic_id: topic.id }) - end - - it "detects locale" do - SiteSetting.translator_enabled = true - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once - DiscourseTranslator::TopicTranslator.expects(:translate).twice - - job.execute({ topic_id: topic.id }) - end - - it "skips bot topics" do - topic.update!(user: Discourse.system_user) - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({ topic_id: topic.id }) - end - - it "does not translate when no target languages are configured" do - SiteSetting.experimental_content_localization_supported_locales = "" - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en") - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({ topic_id: topic.id }) - end - - it "skips translating to the topic's language" do - topic.update(locale: "en") - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en") - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once - - job.execute({ topic_id: topic.id }) - end - - it "handles translation errors gracefully" do - topic.update(locale: "en") - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).returns("en") - DiscourseTranslator::TopicTranslator.expects(:translate).raises(StandardError.new("API error")) - - expect { job.execute({ topic_id: topic.id }) }.not_to raise_error - end - - it "skips public content when `automatic_translation_backfill_limit_to_public_content ` site setting is enabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = true - topic.category.update!(read_restricted: true) - - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({ topic_id: topic.id }) - end -end diff --git a/spec/jobs/post_translation_backfill_spec.rb b/spec/jobs/post_translation_backfill_spec.rb deleted file mode 100644 index e7dba829..00000000 --- a/spec/jobs/post_translation_backfill_spec.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::PostTranslationBackfill do - before do - SiteSetting.automatic_translation_backfill_rate = 100 - SiteSetting.experimental_content_localization_supported_locales = "en" - end - - it "does not enqueue post translation when translator disabled" do - SiteSetting.translator_enabled = false - - described_class.new.execute({}) - - expect_not_enqueued_with(job: :translate_posts) - end - - it "does not enqueue post translation when experimental translation disabled" do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = false - - described_class.new.execute({}) - - expect_not_enqueued_with(job: :translate_posts) - end - - it "does not enqueue psot translation if backfill languages are not set" do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.experimental_content_localization_supported_locales = "" - - described_class.new.execute({}) - - expect_not_enqueued_with(job: :translate_posts) - end - - it "does not enqueue psot translation if backfill limit is set to 0" do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 0 - - described_class.new.execute({}) - - expect_not_enqueued_with(job: :translate_posts) - end - - it "enqueues post translation with correct limit" do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 10 - - described_class.new.execute({}) - - expect_job_enqueued(job: :translate_posts, args: { limit: 10 }) - end -end diff --git a/spec/jobs/posts_locale_detection_backfill_spec.rb b/spec/jobs/posts_locale_detection_backfill_spec.rb deleted file mode 100644 index 50de56b0..00000000 --- a/spec/jobs/posts_locale_detection_backfill_spec.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::PostsLocaleDetectionBackfill do - fab!(:post) { Fabricate(:post, locale: nil) } - subject(:job) { described_class.new } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 100 - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "does nothing when there are no posts to detect" do - Post.update_all(locale: "en") - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "detects locale for posts with nil locale" do - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once - job.execute({}) - end - - it "detects most recently updated posts first" do - post_2 = Fabricate(:post, locale: nil) - post_3 = Fabricate(:post, locale: nil) - - post.update!(updated_at: 3.days.ago) - post_2.update!(updated_at: 2.day.ago) - post_3.update!(updated_at: 4.day.ago) - - SiteSetting.automatic_translation_backfill_rate = 1 - - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post_2).once - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post_3).never - - job.execute({}) - end - - it "skips bot posts" do - post.update!(user: Discourse.system_user) - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never - - job.execute({}) - end - - it "handles detection errors gracefully" do - DiscourseTranslator::PostLocaleDetector - .expects(:detect_locale) - .with(post) - .raises(StandardError.new("jiboomz")) - .once - - expect { job.execute({}) }.not_to raise_error - end - - it "logs a summary after running" do - DiscourseTranslator::PostLocaleDetector.stubs(:detect_locale) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 post locales")) - - job.execute({}) - end - - describe "with public content limitation" do - fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } - fab!(:private_topic) { Fabricate(:topic, category: private_category) } - fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: nil) } - - before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } - - it "only processes posts from public categories" do - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).never - - job.execute({}) - end - - it "processes all posts when setting is disabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = false - - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(private_post).once - - job.execute({}) - end - end - - describe "with max age limit" do - fab!(:old_post) { Fabricate(:post, locale: nil, created_at: 10.days.ago) } - fab!(:new_post) { Fabricate(:post, locale: nil, created_at: 2.days.ago) } - - before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } - - it "only processes posts within the age limit" do - # other posts - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once - - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).never - - job.execute({}) - end - - it "processes all posts when setting is disabled" do - SiteSetting.automatic_translation_backfill_max_age_days = 0 - - # other posts - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).at_least_once - - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(new_post).once - DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(old_post).once - - job.execute({}) - end - end -end diff --git a/spec/jobs/topics_locale_detection_backfill_spec.rb b/spec/jobs/topics_locale_detection_backfill_spec.rb deleted file mode 100644 index 2d3d35b7..00000000 --- a/spec/jobs/topics_locale_detection_backfill_spec.rb +++ /dev/null @@ -1,134 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::TopicsLocaleDetectionBackfill do - fab!(:topic) { Fabricate(:topic, locale: nil) } - subject(:job) { described_class.new } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 100 - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "does nothing when there are no topics to detect" do - Topic.update_all(locale: "en") - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).never - - job.execute({}) - end - - it "detects locale for topics with nil locale" do - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).once - job.execute({}) - end - - it "detects most recently updated topics first" do - topic_2 = Fabricate(:topic, locale: nil) - topic_3 = Fabricate(:topic, locale: nil) - - topic.update!(updated_at: 3.days.ago) - topic_2.update!(updated_at: 2.day.ago) - topic_3.update!(updated_at: 4.day.ago) - - SiteSetting.automatic_translation_backfill_rate = 1 - - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_2).once - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic_3).never - - job.execute({}) - end - - it "skips bot topics" do - topic.update!(user: Discourse.system_user) - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(topic).never - - job.execute({}) - end - - it "handles detection errors gracefully" do - DiscourseTranslator::TopicLocaleDetector - .expects(:detect_locale) - .with(topic) - .raises(StandardError.new("jiboomz")) - .once - - expect { job.execute({}) }.not_to raise_error - end - - it "logs a summary after running" do - DiscourseTranslator::TopicLocaleDetector.stubs(:detect_locale) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 topic locales")) - - job.execute({}) - end - - describe "with public content limitation" do - fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } - fab!(:public_topic) { Fabricate(:topic, locale: nil) } - fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: nil) } - - before do - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once - - SiteSetting.automatic_translation_backfill_limit_to_public_content = true - end - - it "only processes topics from public categories" do - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).never - - job.execute({}) - end - - it "processes all topics when setting is disabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = false - - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once - - job.execute({}) - end - end - - describe "with max age limit" do - fab!(:old_topic) { Fabricate(:topic, locale: nil, created_at: 10.days.ago) } - fab!(:new_topic) { Fabricate(:topic, locale: nil, created_at: 2.days.ago) } - - before do - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).at_least_once - - SiteSetting.automatic_translation_backfill_max_age_days = 5 - end - - it "only processes topics within the age limit" do - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never - - job.execute({}) - end - - it "processes all topics when setting is disabled" do - SiteSetting.automatic_translation_backfill_max_age_days = 0 - - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once - DiscourseTranslator::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once - - job.execute({}) - end - end -end diff --git a/spec/jobs/translate_categories_spec.rb b/spec/jobs/translate_categories_spec.rb deleted file mode 100644 index f2891641..00000000 --- a/spec/jobs/translate_categories_spec.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::TranslateCategories do - subject(:job) { described_class.new } - - let(:translator) { mock } - - def localize_all_categories(*locales) - Category.all.each do |category| - locales.each { |locale| Fabricate(:category_localization, category:, locale:, name: "x") } - end - end - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 100 - SiteSetting.experimental_content_localization_supported_locales = "pt|zh_CN" - - DiscourseTranslator::Provider.stubs(:get).returns(translator) - Jobs.run_immediately! - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - - translator.expects(:translate_text!).never - - job.execute({}) - end - - it "does nothing when experimental_content_translation is disabled" do - SiteSetting.experimental_content_translation = false - - translator.expects(:translate_text!).never - - job.execute({}) - end - - it "does nothing when no target languages are configured" do - SiteSetting.experimental_content_localization_supported_locales = "" - - translator.expects(:translate_text!).never - - job.execute({}) - end - - it "does nothing when no categories exist" do - Category.destroy_all - - translator.expects(:translate_text!).never - - job.execute({}) - end - - it "translates categories to the configured locales" do - number_of_categories = Category.count - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(is_a(Category), "pt") - .times(number_of_categories) - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(is_a(Category), "zh_CN") - .times(number_of_categories) - - job.execute({}) - end - - it "skips categories that already have localizations" do - localize_all_categories("pt", "zh_CN") - - category1 = - Fabricate(:category, name: "First Category", description: "First category description") - Fabricate(:category_localization, category: category1, locale: "pt", name: "Primeira Categoria") - - # It should only translate to Chinese, not Portuguese - DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "pt").never - DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "zh_CN").once - - job.execute({}) - end - - it "continues from a specified category ID" do - category1 = Fabricate(:category, name: "First", description: "First description") - category2 = Fabricate(:category, name: "Second", description: "Second description") - - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(category1, any_parameters) - .never - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(category2, any_parameters) - .twice - - job.execute(from_category_id: category2.id) - end - - it "handles translation errors gracefully" do - localize_all_categories("pt", "zh_CN") - - category1 = Fabricate(:category, name: "First", description: "First description") - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(category1, "pt") - .raises(StandardError.new("API error")) - DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "zh_CN").once - - expect { job.execute({}) }.not_to raise_error - end - - it "enqueues the next batch when there are more categories" do - Jobs.run_later! - freeze_time - Jobs::TranslateCategories.const_set(:BATCH_SIZE, 1) - - job.execute({}) - - Category.all.each do |category| - puts category.id - expect_job_enqueued( - job: :translate_categories, - args: { - from_category_id: category.id + 1, - }, - at: 10.seconds.from_now, - ) - end - - Jobs::TranslateCategories.send(:remove_const, :BATCH_SIZE) - Jobs::TranslateCategories.const_set(:BATCH_SIZE, 50) - end - - it "skips read-restricted categories when configured" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = true - - category1 = Fabricate(:category, name: "Public Category", read_restricted: false) - category2 = Fabricate(:category, name: "Private Category", read_restricted: true) - - DiscourseTranslator::CategoryTranslator.expects(:translate).at_least_once - - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(category1, any_parameters) - .twice - DiscourseTranslator::CategoryTranslator - .expects(:translate) - .with(category2, any_parameters) - .never - - job.execute({}) - end -end diff --git a/spec/jobs/translate_posts_spec.rb b/spec/jobs/translate_posts_spec.rb deleted file mode 100644 index 3c542475..00000000 --- a/spec/jobs/translate_posts_spec.rb +++ /dev/null @@ -1,189 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::TranslatePosts do - fab!(:post) - subject(:job) { described_class.new } - - let(:locales) { %w[en ja de] } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 1 - SiteSetting.experimental_content_localization_supported_locales = locales.join("|") - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when no target languages are configured" do - SiteSetting.experimental_content_localization_supported_locales = "" - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when there are no posts to translate" do - Post.destroy_all - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "skips posts that already have localizations" do - Post.all.each do |post| - Fabricate(:post_localization, post:, locale: "en") - Fabricate(:post_localization, post:, locale: "ja") - end - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "skips bot posts" do - post.update!(user: Discourse.system_user) - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").never - - job.execute({}) - end - - it "handles translation errors gracefully" do - post.update(locale: "es") - DiscourseTranslator::PostTranslator - .expects(:translate) - .with(post, "en") - .raises(StandardError.new("API error")) - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "de").once - - expect { job.execute({}) }.not_to raise_error - end - - it "logs a summary after translation" do - post.update(locale: "es") - DiscourseTranslator::PostTranslator.stubs(:translate) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 posts to en")) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 posts to ja")) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 posts to de")) - - job.execute({}) - end - - context "for translation scenarios" do - it "scenario 1: skips post when locale is not set" do - DiscourseTranslator::PostTranslator.expects(:translate).never - - job.execute({}) - end - - it "scenario 2: returns post with locale 'es' if localizations for en/ja/de do not exist" do - post = Fabricate(:post, locale: "es") - - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "de").once - - job.execute({}) - end - - it "scenario 3: returns post with locale 'en' if ja/de localization does not exist" do - post = Fabricate(:post, locale: "en") - - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "de").once - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never - - job.execute({}) - end - - it "scenario 4: skips post with locale 'en' if 'ja' localization already exists" do - post = Fabricate(:post, locale: "en") - Fabricate(:post_localization, post: post, locale: "ja") - - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").never - DiscourseTranslator::PostTranslator.expects(:translate).with(post, "de").once - - job.execute({}) - end - end - - describe "with public content limitation" do - fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } - fab!(:private_topic) { Fabricate(:topic, category: private_category) } - fab!(:private_post) { Fabricate(:post, topic: private_topic, locale: "es") } - fab!(:public_post) { Fabricate(:post, locale: "es") } - - before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } - - it "only processes posts from public categories" do - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once - - DiscourseTranslator::PostTranslator - .expects(:translate) - .with(private_post, any_parameters) - .never - - job.execute({}) - end - - it "processes all posts when setting is disabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = false - - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(public_post, "de").once - - DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(private_post, "de").once - - job.execute({}) - end - end - - describe "with max age limit" do - fab!(:old_post) { Fabricate(:post, locale: "es", created_at: 10.days.ago) } - fab!(:new_post) { Fabricate(:post, locale: "es", created_at: 2.days.ago) } - - before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } - - it "only processes posts within the age limit" do - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once - - DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, any_parameters).never - - job.execute({}) - end - - it "processes all posts when setting is disabled" do - SiteSetting.automatic_translation_backfill_max_age_days = 0 - - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(new_post, "de").once - - DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "en").once - DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "ja").once - DiscourseTranslator::PostTranslator.expects(:translate).with(old_post, "de").once - - job.execute({}) - end - end -end diff --git a/spec/jobs/translate_topics_spec.rb b/spec/jobs/translate_topics_spec.rb deleted file mode 100644 index d8b3ca0a..00000000 --- a/spec/jobs/translate_topics_spec.rb +++ /dev/null @@ -1,188 +0,0 @@ -# frozen_string_literal: true - -describe Jobs::TranslateTopics do - fab!(:topic) - subject(:job) { described_class.new } - - let(:locales) { %w[en ja de] } - - before do - SiteSetting.translator_enabled = true - SiteSetting.experimental_content_translation = true - SiteSetting.automatic_translation_backfill_rate = 1 - SiteSetting.experimental_content_localization_supported_locales = locales.join("|") - end - - it "does nothing when translator is disabled" do - SiteSetting.translator_enabled = false - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when content translation is disabled" do - SiteSetting.experimental_content_translation = false - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when no target languages are configured" do - SiteSetting.experimental_content_localization_supported_locales = "" - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "does nothing when there are no topics to translate" do - Topic.destroy_all - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "skips topics that already have localizations" do - Topic.all.each do |topic| - Fabricate(:topic_localization, topic:, locale: "en") - Fabricate(:topic_localization, topic:, locale: "ja") - end - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "skips bot topics" do - topic.update!(user: Discourse.system_user) - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").never - - job.execute({}) - end - - it "handles translation errors gracefully" do - topic.update(locale: "es") - DiscourseTranslator::TopicTranslator - .expects(:translate) - .with(topic, "en") - .raises(StandardError.new("API error")) - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "de").once - - expect { job.execute({}) }.not_to raise_error - end - - it "logs a summary after translation" do - topic.update(locale: "es") - DiscourseTranslator::TopicTranslator.stubs(:translate) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 topics to en")) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 topics to ja")) - DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 topics to de")) - - job.execute({}) - end - - context "for translation scenarios" do - it "scenario 1: skips topic when locale is not set" do - DiscourseTranslator::TopicTranslator.expects(:translate).never - - job.execute({}) - end - - it "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do - topic = Fabricate(:topic, locale: "es") - - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "de").once - - job.execute({}) - end - - it "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do - topic = Fabricate(:topic, locale: "en") - - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "de").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never - - job.execute({}) - end - - it "scenario 4: skips topic with locale 'en' if 'ja' localization already exists" do - topic = Fabricate(:topic, locale: "en") - Fabricate(:topic_localization, topic: topic, locale: "ja") - - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "en").never - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "ja").never - DiscourseTranslator::TopicTranslator.expects(:translate).with(topic, "de").once - - job.execute({}) - end - end - - describe "with public content limitation" do - fab!(:private_category) { Fabricate(:private_category, group: Group[:staff]) } - fab!(:private_topic) { Fabricate(:topic, category: private_category, locale: "es") } - fab!(:public_topic) { Fabricate(:topic, locale: "es") } - - before { SiteSetting.automatic_translation_backfill_limit_to_public_content = true } - - it "only processes topics from public categories" do - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once - - DiscourseTranslator::TopicTranslator - .expects(:translate) - .with(private_topic, any_parameters) - .never - - job.execute({}) - end - - it "processes all topics when setting is disabled" do - SiteSetting.automatic_translation_backfill_limit_to_public_content = false - - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(public_topic, "de").once - - DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(private_topic, "de").once - - job.execute({}) - end - end - - describe "with max age limit" do - fab!(:old_topic) { Fabricate(:topic, locale: "es", created_at: 10.days.ago) } - fab!(:new_topic) { Fabricate(:topic, locale: "es", created_at: 2.days.ago) } - - before { SiteSetting.automatic_translation_backfill_max_age_days = 5 } - - it "only processes topics within the age limit" do - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once - - DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, any_parameters).never - - job.execute({}) - end - - it "processes all topics when setting is disabled" do - SiteSetting.automatic_translation_backfill_max_age_days = 0 - - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(new_topic, "de").once - - DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "en").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "ja").once - DiscourseTranslator::TopicTranslator.expects(:translate).with(old_topic, "de").once - - job.execute({}) - end - end -end diff --git a/spec/lib/automatic_translation_spec.rb b/spec/lib/automatic_translation_spec.rb deleted file mode 100644 index 0e97f7fe..00000000 --- a/spec/lib/automatic_translation_spec.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::AutomaticTranslations do - before { SiteSetting.translator_enabled = true } - - describe "upon post process cooked" do - it "enqueues detect post locale and translate post job" do - SiteSetting.experimental_content_localization = true - post = Fabricate(:post) - CookedPostProcessor.new(post).post_process - - expect_job_enqueued(job: :detect_translate_post, args: { post_id: post.id }) - end - - it "does not enqueue if setting disabled" do - SiteSetting.experimental_content_localization = false - post = Fabricate(:post) - CookedPostProcessor.new(post).post_process - - expect(job_enqueued?(job: :detect_translate_post, args: { post_id: post.id })).to eq false - end - end - - describe "upon topic created" do - it "enqueues detect topic locale and translate topic job" do - SiteSetting.experimental_content_localization = true - topic = - PostCreator.create!( - Fabricate(:admin), - raw: "post", - title: "topic", - skip_validations: true, - ).topic - - expect_job_enqueued(job: :detect_translate_topic, args: { topic_id: topic.id }) - end - - it "does not enqueue if setting disabled" do - SiteSetting.experimental_content_localization = false - topic = - PostCreator.create!( - Fabricate(:admin), - raw: "post", - title: "topic", - skip_validations: true, - ).topic - - expect(job_enqueued?(job: :detect_translate_topic, args: { topic_id: topic.id })).to eq false - end - end - - describe "upon first post (topic) edited" do - fab!(:post) { Fabricate(:post, post_number: 1) } - fab!(:non_first_post) { Fabricate(:post, post_number: 2) } - - it "enqueues detect topic locale and translate topic job" do - SiteSetting.experimental_content_localization = true - topic = post.topic - revisor = PostRevisor.new(post, topic) - revisor.revise!( - post.user, - { title: "A whole new hole" }, - { validate_post: false, bypass_bump: false }, - ) - revisor.post_process_post - - expect_job_enqueued(job: :detect_translate_topic, args: { topic_id: topic.id }) - end - - it "does not enqueue if setting disabled" do - SiteSetting.experimental_content_localization = false - - expect( - job_enqueued?(job: :detect_translate_topic, args: { topic_id: post.topic_id }), - ).to eq false - end - end -end diff --git a/spec/lib/content_splitter_spec.rb b/spec/lib/content_splitter_spec.rb deleted file mode 100644 index 3d1d9f2f..00000000 --- a/spec/lib/content_splitter_spec.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe DiscourseTranslator::ContentSplitter do - let(:original_limit) { 4000 } - - after { described_class.const_set(:CHUNK_SIZE, original_limit) } - - def set_limit(value) - described_class.const_set(:CHUNK_SIZE, value) - end - - it "returns empty array for empty input" do - expect(described_class.split("")).to eq([""]) - end - - it "handles content with only spaces" do - expect(described_class.split(" ")).to eq([" "]) - expect(described_class.split(" ")).to eq([" "]) - end - - it "handles nil input" do - expect(described_class.split(nil)).to eq([]) - end - - it "doesn't split content under limit" do - text = "hello world" - expect(described_class.split(text)).to eq([text]) - end - - it "preserves HTML tags" do - set_limit(10) - text = "

hello

meow

" - expect(described_class.split(text)).to eq(%w[

hello

meow

]) - - set_limit(35) - text = "
hello
jurassic

world

" - expect(described_class.split(text)).to eq( - ["
hello
jurassic
", "

world

"], - ) - end - - it "preserves BBCode tags" do - set_limit(20) - text = "[quote]hello[/quote][details]world[/details]" - expect(described_class.split(text)).to eq(["[quote]hello[/quote]", "[details]world[/details]"]) - end - - it "doesn't split in middle of words" do - set_limit(10) - text = "my kitty best in the world" - expect(described_class.split(text)).to eq(["my kitty ", "best in ", "the world"]) - end - - it "handles nested tags properly" do - set_limit(25) - text = "
hello

cat

world

meow

" - expect(described_class.split(text)).to eq(%w[
hello

cat

world

meow

]) - end - - it "handles mixed HTML and BBCode" do - set_limit(15) - text = "
hello
[quote]world[/quote]

beautiful

" - expect(described_class.split(text)).to eq( - ["
hello
", "[quote]world[/quote]", "

beautiful

"], - ) - end - - it "preserves newlines in sensible places" do - set_limit(10) - text = "hello\nbeautiful\nworld\n" - expect(described_class.split(text)).to eq(["hello\n", "beautiful\n", "world\n"]) - end - - it "handles email content properly" do - set_limit(20) - text = "From: test@test.com\nTo: other@test.com\nSubject: Hello\n\nContent here" - expect(described_class.split(text)).to eq( - ["From: test@test.com\n", "To: other@test.com\n", "Subject: Hello\n\n", "Content here"], - ) - end - - it "keeps code blocks intact" do - set_limit(30) - text = "Text\n```\ncode block\nhere\n```\nmore text" - expect(described_class.split(text)).to eq(["Text\n```\ncode block\nhere\n```\n", "more text"]) - end - - context "with multiple details tags" do - it "splits correctly between details tags" do - set_limit(30) - text = "
first content
second content
" - expect(described_class.split(text)).to eq( - ["
first content
", "
second content
"], - ) - end - end -end diff --git a/spec/lib/locale_normalizer_spec.rb b/spec/lib/locale_normalizer_spec.rb deleted file mode 100644 index e76686fa..00000000 --- a/spec/lib/locale_normalizer_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::LocaleNormalizer do - it "matches input locales to i18n locales" do - expect(described_class.normalize_to_i18n("en-GB")).to eq("en_GB") - expect(described_class.normalize_to_i18n("en")).to eq("en") - expect(described_class.normalize_to_i18n("zh")).to eq("zh_CN") - expect(described_class.normalize_to_i18n("tr")).to eq("tr_TR") - end - - it "converts dashes to underscores" do - expect(described_class.normalize_to_i18n("a-b")).to eq("a_b") - end -end diff --git a/spec/lib/translated_content_normalizer_spec.rb b/spec/lib/translated_content_normalizer_spec.rb deleted file mode 100644 index b22eec19..00000000 --- a/spec/lib/translated_content_normalizer_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::TranslatedContentNormalizer do - fab!(:post) - fab!(:topic) - - it "normalizes the content" do - expect( - DiscourseTranslator::TranslatedContentNormalizer.normalize( - post, - "

Testing

This is a test post

", - ), - ).to eq("

Testing

This is a test post

") - - expect( - DiscourseTranslator::TranslatedContentNormalizer.normalize( - topic, - "

Testing

This is a test post

", - ), - ).to eq("

Testing

This is a test post

") - end -end diff --git a/spec/lib/translator_selection_validator_spec.rb b/spec/lib/translator_selection_validator_spec.rb deleted file mode 100644 index 338c7daf..00000000 --- a/spec/lib/translator_selection_validator_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe ::DiscourseTranslator::Validators::TranslatorSelectionValidator do - fab!(:llm_model) - - describe "#valid_value?" do - context "when value is blank" do - it "returns true" do - expect(described_class.new.valid_value?(nil)).to eq(true) - expect(described_class.new.valid_value?("")).to eq(true) - end - end - - context "when value is 'DiscourseAi'" do - context "when DiscourseAi is not defined" do - it "returns false" do - hide_const("DiscourseAi") - expect(described_class.new.valid_value?("DiscourseAi")).to eq(false) - end - end - - context "when DiscourseAi is defined" do - it "returns true" do - DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do - SiteSetting.ai_translation_model = "custom:#{llm_model.id}" - end - expect(described_class.new.valid_value?("DiscourseAi")).to eq(true) - end - end - end - - context "when value is not 'DiscourseAi'" do - it "returns true" do - expect(described_class.new.valid_value?("googly")).to eq(true) - expect(described_class.new.valid_value?("poopy")).to eq(true) - end - end - end - - describe "#error_message" do - context "when DiscourseAi is not defined" do - it "returns the not_installed error message" do - hide_const("DiscourseAi") - expect(described_class.new.error_message).to eq( - I18n.t("translator.discourse_ai.not_installed"), - ) - end - end - - context "when DiscourseAi is defined" do - it "returns nil" do - DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) do - SiteSetting.ai_translation_model = "custom:#{llm_model.id}" - end - expect(described_class.new.error_message).to be_nil - end - end - end -end diff --git a/spec/services/category_translator_spec.rb b/spec/services/category_translator_spec.rb deleted file mode 100644 index b9aec656..00000000 --- a/spec/services/category_translator_spec.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::CategoryTranslator do - fab!(:category) do - Fabricate(:category, name: "Test Category", description: "This is a test category") - end - - describe ".translate" do - let(:target_locale) { :fr } - let(:translator) { mock } - - before { DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) } - - it "translates the category name and description" do - translator - .expects(:translate_text!) - .with(category.name, target_locale) - .returns("Catégorie de Test") - translator - .expects(:translate_text!) - .with(category.description, target_locale) - .returns("C'est une catégorie de test") - - res = DiscourseTranslator::CategoryTranslator.translate(category, target_locale) - - expect(res.name).to eq("Catégorie de Test") - expect(res.description).to eq("C'est une catégorie de test") - end - - it "translates the category name and description" do - localized = - Fabricate( - :category_localization, - category: category, - locale: target_locale, - name: "X", - description: "Y", - ) - translator - .expects(:translate_text!) - .with(category.name, target_locale) - .returns("Catégorie de Test") - translator - .expects(:translate_text!) - .with(category.description, target_locale) - .returns("C'est une catégorie de test") - - DiscourseTranslator::CategoryTranslator.translate(category, target_locale) - - localized.reload - expect(localized.name).to eq("Catégorie de Test") - expect(localized.description).to eq("C'est une catégorie de test") - end - - it "handles locale format standardization" do - translator.expects(:translate_text!).with(category.name, :fr).returns("Catégorie de Test") - translator - .expects(:translate_text!) - .with(category.description, :fr) - .returns("C'est une catégorie de test") - - res = DiscourseTranslator::CategoryTranslator.translate(category, "fr") - - expect(res.name).to eq("Catégorie de Test") - expect(res.description).to eq("C'est une catégorie de test") - end - - it "returns nil if category is blank" do - expect(DiscourseTranslator::CategoryTranslator.translate(nil)).to be_nil - end - - it "returns nil if target locale is blank" do - expect(DiscourseTranslator::CategoryTranslator.translate(category, nil)).to be_nil - end - - it "uses I18n.locale as default when no target locale is provided" do - I18n.locale = :es - translator.expects(:translate_text!).with(category.name, :es).returns("Categoría de Prueba") - translator - .expects(:translate_text!) - .with(category.description, :es) - .returns("Esta es una categoría de prueba") - - res = DiscourseTranslator::CategoryTranslator.translate(category) - - expect(res.name).to eq("Categoría de Prueba") - expect(res.description).to eq("Esta es una categoría de prueba") - expect(res.locale).to eq("es") - end - end -end diff --git a/spec/services/discourse_ai/base_translator_spec.rb b/spec/services/discourse_ai/base_translator_spec.rb deleted file mode 100644 index cbae367d..00000000 --- a/spec/services/discourse_ai/base_translator_spec.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe DiscourseAi::BaseTranslator do - before do - Fabricate(:fake_model).tap do |fake_llm| - SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") - end - end - - describe ".translate" do - let(:text_to_translate) { "cats are great" } - let(:target_language) { "de" } - let(:llm_response) { "hur dur hur dur!" } - - it "creates the correct prompt" do - post_translator = DiscourseAi::PostTranslator.new(text_to_translate, target_language) - allow(DiscourseAi::Completions::Prompt).to receive(:new).with( - DiscourseAi::PostTranslator::PROMPT_TEMPLATE, - messages: [{ type: :user, content: post_translator.formatted_content, id: "user" }], - ).and_call_original - - DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do - post_translator.translate - end - end - - it "sends the translation prompt to the selected ai helper model" do - mock_prompt = instance_double(DiscourseAi::Completions::Prompt) - mock_llm = instance_double(DiscourseAi::Completions::Llm) - post_translator = DiscourseAi::PostTranslator.new(text_to_translate, target_language) - - structured_output = - DiscourseAi::Completions::StructuredOutput.new({ translation: { type: "string" } }) - structured_output << { translation: llm_response }.to_json - - allow(DiscourseAi::Completions::Prompt).to receive(:new).and_return(mock_prompt) - allow(DiscourseAi::Completions::Llm).to receive(:proxy).with( - SiteSetting.ai_translation_model, - ).and_return(mock_llm) - allow(mock_llm).to receive(:generate).with( - mock_prompt, - user: Discourse.system_user, - feature_name: "translator-translate", - response_format: post_translator.response_format, - ).and_return(structured_output) - - post_translator.translate - end - - it "returns the translation from the llm's response" do - DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do - expect( - DiscourseAi::PostTranslator.new(text_to_translate, target_language).translate, - ).to eq "hur dur hur dur!" - end - end - end -end diff --git a/spec/services/discourse_ai/language_detector_spec.rb b/spec/services/discourse_ai/language_detector_spec.rb deleted file mode 100644 index 3bd2c7e5..00000000 --- a/spec/services/discourse_ai/language_detector_spec.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe DiscourseAi::LanguageDetector do - before do - Fabricate(:fake_model).tap do |fake_llm| - SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") - end - end - - describe ".detect" do - let(:locale_detector) { described_class.new("meow") } - let(:llm_response) { "hur dur hur dur!" } - - it "creates the correct prompt" do - allow(DiscourseAi::Completions::Prompt).to receive(:new).with( - DiscourseAi::LanguageDetector::PROMPT_TEXT, - messages: [{ type: :user, content: "meow", id: "user" }], - ).and_call_original - - DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do - locale_detector.detect - end - end - - it "sends the language detection prompt to the ai helper model" do - mock_prompt = instance_double(DiscourseAi::Completions::Prompt) - mock_llm = instance_double(DiscourseAi::Completions::Llm) - - structured_output = - DiscourseAi::Completions::StructuredOutput.new({ locale: { type: "string" } }) - structured_output << { locale: llm_response }.to_json - - allow(DiscourseAi::Completions::Prompt).to receive(:new).and_return(mock_prompt) - allow(DiscourseAi::Completions::Llm).to receive(:proxy).with( - SiteSetting.ai_translation_model, - ).and_return(mock_llm) - allow(mock_llm).to receive(:generate).with( - mock_prompt, - user: Discourse.system_user, - feature_name: "translator-language-detect", - response_format: locale_detector.response_format, - ).and_return(structured_output) - - locale_detector.detect - end - - it "returns the language from the llm's response in the language tag" do - DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do - locale_detector.detect - end - end - end -end diff --git a/spec/services/discourse_ai_spec.rb b/spec/services/discourse_ai_spec.rb deleted file mode 100644 index fe605270..00000000 --- a/spec/services/discourse_ai_spec.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::Provider::DiscourseAi do - fab!(:post) - fab!(:topic) - - before do - Fabricate(:fake_model).tap do |fake_llm| - SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") - end - SiteSetting.translator_enabled = true - SiteSetting.translator_provider = "DiscourseAi" - end - - describe ".language_supported?" do - it "returns true when detected language is different from i18n locale" do - I18n.stubs(:locale).returns(:xx) - expect(described_class.language_supported?("any-language")).to eq(true) - end - - it "returns false when detected language is same base language as i18n locale" do - I18n.stubs(:locale).returns(:en_GB) - expect(described_class.language_supported?("en")).to eq(false) - end - end - - describe ".detect!" do - it "returns the detected language" do - locale = "de" - DiscourseAi::Completions::Llm.with_prepared_responses([locale]) do - expect(DiscourseTranslator::Provider::DiscourseAi.detect!(post)).to eq locale - end - end - end - - describe ".translate_post!" do - before do - post.set_detected_locale("de") - topic.set_detected_locale("de") - end - - it "translates the post and returns [locale, translated_text]" do - DiscourseAi::Completions::Llm.with_prepared_responses(["some translated text"]) do - translated_text = DiscourseTranslator::Provider::DiscourseAi.translate_post!(post) - expect(translated_text).to eq "some translated text" - end - end - - it "sends the content for splitting and the split content for translation" do - post.update(raw: "#{"a" * 3000} #{"b" * 3000}") - DiscourseAi::Completions::Llm.with_prepared_responses(%w[lol wut]) do - expect(DiscourseTranslator::Provider::DiscourseAi.translate_post!(post)).to eq "lolwut" - end - end - end - - describe ".translate_topic!" do - it "translates the topic" do - allow(::DiscourseAi::TopicTranslator).to receive(:new).and_call_original - DiscourseAi::Completions::Llm.with_prepared_responses(["some translated text"]) do - translated_text = DiscourseTranslator::Provider::DiscourseAi.translate_topic!(topic) - expect(translated_text).to eq "some translated text" - end - end - end - - describe ".translate_text!" do - it "returns the translated text" do - DiscourseAi::Completions::Llm.with_prepared_responses(["some translated text"]) do - translated_text = DiscourseTranslator::Provider::DiscourseAi.translate_text!("derp") - expect(translated_text).to eq "some translated text" - end - end - end -end diff --git a/spec/services/post_locale_detector_spec.rb b/spec/services/post_locale_detector_spec.rb deleted file mode 100644 index 69470fc3..00000000 --- a/spec/services/post_locale_detector_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::PostLocaleDetector do - describe ".detect_locale" do - fab!(:post) { Fabricate(:post, raw: "Hello world", locale: nil) } - - let(:translator) { mock } - - before { DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) } - - it "returns nil if post is blank" do - expect(described_class.detect_locale(nil)).to eq(nil) - end - - it "calls detect! on the provider with the post" do - translator.expects(:detect!).with(post).returns("zh") - expect(described_class.detect_locale(post)).to eq("zh_CN") - end - - it "updates the post locale with the detected locale" do - translator.stubs(:detect!).with(post).returns("zh") - expect { described_class.detect_locale(post) }.to change { post.reload.locale }.from(nil).to( - "zh_CN", - ) - end - - it "bypasses validations when updating locale" do - post.update_column(:raw, "A") - - translator.stubs(:detect!).with(post).returns("zh_CN") - - described_class.detect_locale(post) - expect(post.reload.locale).to eq("zh_CN") - end - end -end diff --git a/spec/services/post_translator_spec.rb b/spec/services/post_translator_spec.rb deleted file mode 100644 index 88891b5e..00000000 --- a/spec/services/post_translator_spec.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::PostTranslator do - describe ".translate" do - fab!(:post) { Fabricate(:post, raw: "Hello world", version: 1) } - let(:translator) { mock } - let(:translated_raw) { "こんにちは世界" } - let(:cooked) { "

こんにちは世界

" } - let(:target_locale) { "ja" } - - before do - DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) - translator.stubs(:translate_post!).with(post, :ja).returns(translated_raw) - end - - it "returns nil if post is blank" do - expect(described_class.translate(nil, "ja")).to eq(nil) - end - - it "returns nil if target_locale is blank" do - expect(described_class.translate(post, nil)).to eq(nil) - expect(described_class.translate(post, "")).to eq(nil) - end - - it "returns nil if target_locale is same as post locale" do - post.locale = "en" - - expect(described_class.translate(post, "en")).to eq(nil) - end - - it "translates with post and locale" do - translator.expects(:translate_post!).with(post, :ja).returns(translated_raw) - - described_class.translate(post, "ja") - end - - it "normalizes dashes to underscores and symbol type for locale" do - translator.expects(:translate_post!).with(post, :zh_CN).returns("你好，世界") - - described_class.translate(post, "zh-CN") - end - - it "finds or creates a PostLocalization and sets its fields" do - expect { - res = described_class.translate(post, target_locale) - expect(res).to be_a(PostLocalization) - expect(res).to have_attributes( - post_id: post.id, - locale: target_locale, - raw: translated_raw, - cooked: cooked, - post_version: post.version, - localizer_user_id: Discourse.system_user.id, - ) - }.to change { PostLocalization.count }.by(1) - end - - it "updates an existing PostLocalization if present" do - localization = - Fabricate(:post_localization, post: post, locale: "ja", raw: "old", cooked: "old_cooked") - expect { - out = described_class.translate(post, "ja") - expect(out.id).to eq(localization.id) - expect(out.raw).to eq(translated_raw) - expect(out.cooked).to eq(cooked) - }.to_not change { PostLocalization.count } - end - end -end diff --git a/spec/services/topic_locale_detector_spec.rb b/spec/services/topic_locale_detector_spec.rb deleted file mode 100644 index 4038a555..00000000 --- a/spec/services/topic_locale_detector_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::TopicLocaleDetector do - describe ".detect_locale" do - fab!(:topic) { Fabricate(:topic, title: "this is a cat topic", locale: nil) } - - let(:translator) { mock } - - before { DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) } - - it "returns nil if topic is blank" do - expect(described_class.detect_locale(nil)).to eq(nil) - end - - it "calls detect! on the provider with the topic" do - translator.expects(:detect!).with(topic).returns("zh") - expect(described_class.detect_locale(topic)).to eq("zh_CN") - end - - it "updates the topic locale with the detected locale" do - translator.stubs(:detect!).with(topic).returns("zh") - expect { described_class.detect_locale(topic) }.to change { topic.reload.locale }.from( - nil, - ).to("zh_CN") - end - - it "bypasses validations when updating locale" do - topic.update_column(:title, "A") - SiteSetting.min_topic_title_length = 15 - SiteSetting.max_topic_title_length = 16 - - translator.stubs(:detect!).with(topic).returns("zh") - - described_class.detect_locale(topic) - expect(topic.reload.locale).to eq("zh_CN") - end - end -end diff --git a/spec/services/topic_translator_spec.rb b/spec/services/topic_translator_spec.rb deleted file mode 100644 index 8ac55ca2..00000000 --- a/spec/services/topic_translator_spec.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -describe DiscourseTranslator::TopicTranslator do - describe ".translate" do - fab!(:topic) do - Fabricate( - :topic, - title: "this is a cat topic :)", - excerpt: "cats are great. how many do you have?", - ) - end - let(:translator) { mock } - let(:translated_title) { "これは猫の話題です :)" } - let(:translated_excerpt) { "猫は素晴らしいですね。何匹飼っていますか？" } - let(:fancy_title) { "これは猫の話題です :slight_smile:" } - let(:target_locale) { "ja" } - - before do - DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) - translator.stubs(:translate_topic!).with(topic, :ja).returns(translated_title) - translator.stubs(:translate_text!).with(topic.excerpt, :ja).returns(translated_excerpt) - end - - it "returns nil if topic is blank" do - expect(described_class.translate(nil, "ja")).to eq(nil) - end - - it "returns nil if target_locale is blank" do - expect(described_class.translate(topic, nil)).to eq(nil) - expect(described_class.translate(topic, "")).to eq(nil) - end - - it "returns nil if target_locale is same as topic locale" do - topic.locale = "en" - - expect(described_class.translate(topic, "en")).to eq(nil) - end - - it "translates with topic and locale" do - translator.expects(:translate_topic!).with(topic, :ja).returns(translated_title) - translator.expects(:translate_text!).with(topic.excerpt, :ja).returns(translated_title) - - described_class.translate(topic, "ja") - end - - it "normalizes dashes to underscores and symbol type for locale" do - translator.expects(:translate_topic!).with(topic, :zh_CN).returns("这是一个猫主题 :)") - translator.expects(:translate_text!).with(topic.excerpt, :zh_CN).returns("这是一个猫主题 :)") - - described_class.translate(topic, "zh-CN") - end - - it "finds or creates a TopicLocalization and sets its fields" do - expect { - res = described_class.translate(topic, target_locale) - expect(res).to be_a(TopicLocalization) - expect(res).to have_attributes( - topic_id: topic.id, - locale: target_locale, - title: translated_title, - excerpt: translated_excerpt, - fancy_title: fancy_title, - localizer_user_id: Discourse.system_user.id, - ) - }.to change { TopicLocalization.count }.by(1) - end - - it "updates an existing TopicLocalization if present" do - localization = - Fabricate( - :topic_localization, - topic:, - locale: "ja", - title: "old title", - excerpt: "old excerpt", - fancy_title: "old_fancy_title", - ) - expect { - expect(described_class.translate(topic, "ja")).to have_attributes( - id: localization.id, - title: translated_title, - fancy_title: fancy_title, - excerpt: translated_excerpt, - ) - expect(localization.reload).to have_attributes( - title: translated_title, - fancy_title: fancy_title, - excerpt: translated_excerpt, - ) - }.to_not change { TopicLocalization.count } - end - end -end