DEV: Normalize locales that are similar (e.g. en and en_GB) so they do not get translated (#1495)

This commit
- normalizes locales like en_GB and variants to en. With this, the feature will not translate en_GB posts to en (or similarly pt_BR to pt_PT)
- consolidates whether the feature is enabled in `DiscourseAi::Translation.enabled?`
- similarly for backfill in  `DiscourseAi::Translation.backfill_enabled?`
  - turns off backfill if `ai_translation_backfill_max_age_days` is 0 to keep true to what it says. Set it to a high number to backfill everything

Author: nattsw

app/jobs/regular/detect_translate_post.rb

@@ -6,8 +6,7 @@ class DetectTranslatePost < ::Jobs::Base
     sidekiq_options retry: false
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
       return if args[:post_id].blank?
 
       post = Post.find_by(id: args[:post_id])
@@ -36,12 +35,14 @@ def execute(args)
         end
       end
 
+      return if detected_locale.blank?
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if locale == detected_locale
-        next if post.post_localizations.exists?(locale:)
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 
         begin
           DiscourseAi::Translation::PostLocalizer.localize(post, locale)

app/jobs/regular/detect_translate_topic.rb

@@ -6,8 +6,7 @@ class DetectTranslateTopic < ::Jobs::Base
     sidekiq_options retry: false
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
       return if args[:topic_id].blank?
 
       topic = Topic.find_by(id: args[:topic_id])
@@ -34,12 +33,14 @@ def execute(args)
         end
       end
 
+      return if detected_locale.blank?
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if locale == detected_locale
-        next if topic.topic_localizations.exists?(locale:)
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 
         begin
           DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)

app/jobs/regular/localize_categories.rb

@@ -6,21 +6,17 @@ class LocalizeCategories < ::Jobs::Base
     sidekiq_options retry: false
 
     def execute(args)
+      return if !DiscourseAi::Translation.enabled?
+
       limit = args[:limit]
       raise Discourse::InvalidParameters.new(:limit) if limit.nil?
       return if limit <= 0
-
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
       locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
 
       categories = Category.where("locale IS NOT NULL")
-
       if SiteSetting.ai_translation_backfill_limit_to_public_content
         categories = categories.where(read_restricted: false)
       end
-
       categories = categories.order(:id).limit(limit)
       return if categories.empty?
 
@@ -33,6 +29,7 @@ def execute(args)
         missing_locales = locales - existing_locales - [category.locale]
         missing_locales.each do |locale|
           break if remaining_limit <= 0
+          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
 
           begin
             DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)

app/jobs/regular/localize_posts.rb

@@ -9,23 +9,25 @@ def execute(args)
       limit = args[:limit]
       raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
 
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
       locales.each do |locale|
+        base_locale = locale.split("_").first
         posts =
           Post
             .joins(
-              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
             )
             .where(deleted_at: nil)
             .where("posts.user_id > 0")
             .where.not(raw: [nil, ""])
             .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("posts.locale NOT LIKE '#{base_locale}%'")
             .where("pl.id IS NULL")
 
         posts = posts.joins(:topic)
@@ -46,14 +48,6 @@ def execute(args)
             )
         end
 
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          posts =
-            posts.where(
-              "posts.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
         posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?

app/jobs/regular/localize_topics.rb

@@ -9,22 +9,24 @@ def execute(args)
       limit = args[:limit]
       raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
 
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
       locales.each do |locale|
+        base_locale = locale.split("_").first
         topics =
           Topic
             .joins(
-              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
             )
             .where(deleted_at: nil)
             .where("topics.user_id > 0")
             .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("topics.locale NOT LIKE '#{base_locale}%'")
             .where("tl.id IS NULL")
 
         if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -43,14 +45,6 @@ def execute(args)
             )
         end
 
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          topics =
-            topics.where(
-              "topics.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
         topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -7,17 +7,15 @@ class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_hourly_rate
-      return if limit == 0
+      return if !DiscourseAi::Translation.backfill_enabled?
 
       categories = Category.where(locale: nil)
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
         categories = categories.where(read_restricted: false)
       end
 
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
       categories = categories.limit(limit)
       return if categories.empty?
 

app/jobs/scheduled/category_localization_backfill.rb

@@ -6,11 +6,8 @@ class CategoryLocalizationBackfill < ::Jobs::Scheduled
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.content_localization_supported_locales.blank?
+      return if !DiscourseAi::Translation.backfill_enabled?
       limit = SiteSetting.ai_translation_backfill_hourly_rate
-      return if limit == 0
 
       Jobs.enqueue(:localize_categories, limit:)
     end

app/jobs/scheduled/post_localization_backfill.rb

@@ -6,10 +6,8 @@ class PostLocalizationBackfill < ::Jobs::Scheduled
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
-      return if SiteSetting.content_localization_supported_locales.blank?
       limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
       return if limit == 0
 

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -7,16 +7,16 @@ class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
+
       limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-      return if limit == 0
 
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
+          .where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
           .where.not(raw: [nil, ""])
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -33,14 +33,6 @@ def execute(args)
           )
       end
 
-      if SiteSetting.ai_translation_backfill_max_age_days > 0
-        posts =
-          posts.where(
-            "posts.created_at > ?",
-            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-          )
-      end
-
       posts = posts.order(updated_at: :desc).limit(limit)
       return if posts.empty?
 

app/jobs/scheduled/topic_localization_backfill.rb

@@ -6,13 +6,9 @@ class TopicLocalizationBackfill < ::Jobs::Scheduled
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
-      return if SiteSetting.content_localization_supported_locales.blank?
       limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-      return if limit == 0
-
       Jobs.enqueue(:localize_topics, limit:)
     end
   end