DEV: Indicate backfill rate for translations is hourly (#1451)

* DEV: Indicate backfill rate for translations is hourly

* add ai_translation_max_post_length

* default value update

Author: nattsw

app/jobs/regular/localize_categories.rb

@@ -5,51 +5,51 @@ class LocalizeCategories < ::Jobs::Base
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
+      return if limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      cat_id = args[:from_category_id] || Category.order(:id).first&.id
-      last_id = nil
+      categories = Category.where("locale IS NOT NULL")
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
 
-      categories =
-        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
+      categories = categories.order(:id).limit(limit)
       return if categories.empty?
 
-      categories.each do |category|
-        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
-          last_id = category.id
-          next
-        end
+      remaining_limit = limit
 
-        locales.each do |locale|
-          localization = category.category_localizations.find_by(locale:)
-
-          if locale == category.locale && localization
-            localization.destroy
-          else
-            next if locale == category.locale
-            begin
-              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-            rescue FinalDestination::SSRFDetector::LookupFailedError
-              # do nothing, there are too many sporadic lookup failures
-            rescue => e
-              DiscourseAi::Translation::VerboseLogger.log(
-                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
-              )
-            end
+      categories.each do |category|
+        break if remaining_limit <= 0
+
+        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
+        missing_locales = locales - existing_locales - [category.locale]
+        missing_locales.each do |locale|
+          break if remaining_limit <= 0
+
+          begin
+            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+            )
+          ensure
+            remaining_limit -= 1
           end
         end
-        last_id = category.id
-      end
 
-      if categories.size == BATCH_SIZE
-        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+        if existing_locales.include?(category.locale)
+          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
+        end
       end
     end
   end

app/jobs/regular/localize_posts.rb

@@ -5,17 +5,16 @@ class LocalizePosts < ::Jobs::Base
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         posts =
           Post

app/jobs/regular/localize_topics.rb

@@ -5,17 +5,16 @@ class LocalizeTopics < ::Jobs::Base
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         topics =
           Topic

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -9,15 +9,16 @@ class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
       categories = Category.where(locale: nil)
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
         categories = categories.where(read_restricted: false)
       end
 
-      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      categories = categories.limit(limit)
       return if categories.empty?
 
       categories.each do |category|

app/jobs/scheduled/category_localization_backfill.rb

@@ -2,15 +2,17 @@
 
 module Jobs
   class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 12.hours
+    every 1.hour
     cluster_concurrency 1
 
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
       return if SiteSetting.content_localization_supported_locales.blank?
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
-      Jobs.enqueue(:localize_categories)
+      Jobs.enqueue(:localize_categories, limit:)
     end
   end
 end

app/jobs/scheduled/post_localization_backfill.rb

@@ -10,9 +10,10 @@ def execute(args)
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_posts, limit:)
     end
   end
 end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -9,7 +9,8 @@ class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
       posts =
         Post
@@ -35,7 +36,7 @@ def execute(args)
           )
       end
 
-      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      posts = posts.order(updated_at: :desc).limit(limit)
       return if posts.empty?
 
       posts.each do |post|

app/jobs/scheduled/topic_localization_backfill.rb

@@ -10,9 +10,10 @@ def execute(args)
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_topics, limit:)
     end
   end
 end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -9,8 +9,7 @@ class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_rate
-
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
       return if limit == 0
 
       topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")

config/locales/server.en.yml

@@ -118,6 +118,12 @@ en:
     ai_discord_allowed_guilds: "Discord guilds (servers) where the bot is allowed to search"
     ai_bot_enable_dedicated_ux: "Allow for full screen bot interface, instead of a PM"
 
+    ai_translation_enabled: "Enables the AI translation feature"
+    ai_translation_model: "The model to use for translation. This model must support translation. Personas can override this setting."
+    ai_translation_backfill_limit_to_public_content: "When enabled, only content in public categories will be translated. When disabled, content in group PMs and private categories will also be sent for translation."
+    ai_translation_max_post_length: "The maximum length of a post to be translated. Posts longer than this will not be translated."
+    ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated."
+
   reviewables:
     reasons:
       flagged_by_toxicity: The AI plugin flagged this after classifying it as toxic.