DEV: Change backfill rate handling

app/jobs/scheduled/automatic_translation_backfill.rb

@@ -14,34 +14,41 @@ def execute(args = nil)
 
     def fetch_untranslated_model_ids(model, content_column, limit, target_locale)
       m = model.name.downcase
+
+      # Query selects every model (post/topic) *except* those who are **both**
+      # already locale detected and translated
       DB.query_single(<<~SQL, target_locale: target_locale, limit: limit)
-        SELECT m.id
-        FROM #{model.table_name} m
-        #{limit_to_public_clause(model)}
-        WHERE m.deleted_at IS NULL
-          AND m.#{content_column} != ''
-          AND m.user_id > 0
-          #{max_age_clause}
-          AND (
-            NOT EXISTS (
-              SELECT 1
-              FROM discourse_translator_#{m}_locales
-              WHERE #{m}_id = m.id
+        SELECT * FROM
+        (
+          ( -- every post / topic
+            SELECT m.id
+            FROM #{model.table_name} m
+            #{limit_to_public_clause(model)}
+            WHERE m.deleted_at IS NULL
+              AND m.#{content_column} != ''
+              AND m.user_id > 0
+              #{max_age_clause}
+            ORDER BY m.updated_at DESC
+          )
+          EXCEPT
+          (
+            ( -- locale detected
+              SELECT
+                #{m}_id
+              FROM
+                discourse_translator_#{m}_locales
+              WHERE
+                detected_locale = :target_locale
             )
-            OR EXISTS (
-              SELECT 1
-              FROM discourse_translator_#{m}_locales
-              WHERE #{m}_id = m.id
-              AND detected_locale != :target_locale
+            INTERSECT
+            ( -- translated
+              SELECT #{m}_id
+              FROM discourse_translator_#{m}_translations
+              WHERE
+                locale = :target_locale
             )
           )
-          AND NOT EXISTS (
-            SELECT 1
-            FROM discourse_translator_#{m}_translations
-            WHERE #{m}_id = m.id
-            AND locale = :target_locale
-          )
-        ORDER BY m.updated_at DESC
+        ) AS subquery
         LIMIT :limit
       SQL
     end
@@ -50,14 +57,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale)
 
     def should_backfill?
       return false if SiteSetting.automatic_translation_target_languages.blank?
-      return false if SiteSetting.automatic_translation_backfill_maximum_translations_per_hour == 0
+      return false if SiteSetting.automatic_translation_backfill_rate == 0
       true
     end
 
-    def translations_per_run
-      [(SiteSetting.automatic_translation_backfill_maximum_translations_per_hour / 12), 1].max
-    end
-
     def backfill_locales
       @backfill_locales ||=
         SiteSetting.automatic_translation_target_languages.split("|").map { |l| l.gsub("_", "-") }
@@ -83,23 +86,12 @@ def translate_records(type, record_ids, target_locale)
     end
 
     def process_batch
-      models_translated = [Post, Topic].size
-      avg_translations_per_model_per_language = [
-        translations_per_run / models_translated / backfill_locales.size,
-        1,
-      ].max
-
-      records_to_translate = avg_translations_per_model_per_language
+      records_to_translate = SiteSetting.automatic_translation_backfill_rate
       backfill_locales.each_with_index do |target_locale, i|
         topic_ids =
           fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale)
         post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale)
 
-        # if we end up translating fewer records than records_to_translate,
-        # add to the value so that the next locales can have more quota
-        records_to_translate =
-          avg_translations_per_model_per_language +
-            ((records_to_translate - topic_ids.size - post_ids.size) / backfill_locales.size - i)
         next if topic_ids.empty? && post_ids.empty?
 
         DiscourseTranslator::VerboseLogger.log(

config/locales/server.de.yml

@@ -25,7 +25,7 @@ de:
     restrict_translation_by_poster_group: "Erlaube nur die Übersetzung von Beiträgen von Benutzern aus zulässigen Gruppen. Wenn das Feld leer ist, wird die Übersetzung von Beiträgen aller Benutzer erlaubt."
     experimental_anon_language_switcher: "Aktiviere die experimentelle Sprachumschaltung. Damit können nicht eingeloggte Besucher zwischen übersetzten Versionen von Discourse und von Benutzern beigetragenen Inhalten in Themen wechseln."
     errors:
-      needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_maximum_translations_per_hour' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen."
+      needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_rate' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen."
       experimental_anon_language_switcher_requirements: "Die experimentelle Sprachumschaltung setzt voraus, dass die Website-Einstellung `set locale from cookie` aktiviert ist und `automatic translation target languages` mindestens eine Sprache enthält."
     experimental_inline_translation: "Aktiviere die experimentelle Übersetzungsfunktion mit Überschreiben. Diese ersetzt die bisherige parallele Übersetzung und ermöglicht es Besuchern der Website, die nicht das Standardgebietsschema verwenden, Inhalte direkt in ihrer Sprache zu sehen."
     automatic_translation_target_languages: "Die Sprachen, in die Benutzerinhalte (Beiträge, Themen) automatisch übersetzt werden sollen. Wenn nichts ausgewählt ist, wird in keine Sprachen automatisch übersetzt."

config/locales/server.en.yml

@@ -19,7 +19,7 @@ en:
     restrict_translation_by_poster_group: "Only allow translation of posts made by users in allowed groups. If empty, allow translations of posts from all users."
     experimental_anon_language_switcher: "Enable experimental language switcher. This will allow site visitors who are not logged in to switch between translated versions of Discourse and user-contributed content in topics."
     errors:
-      needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_maximum_translations_per_hour' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit."
+      needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_rate' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit."
       experimental_anon_language_switcher_requirements: "The experimental language switcher requires the `set locale from cookie` site setting to be enabled, and the `automatic translation target languages` to have at least one language."
     experimental_inline_translation: "Enable experimental inline translation feature. This replaces existing parallel translation, allowing site visitors with a non-default locale to view content in their language."
     automatic_translation_target_languages: "The languages to automatically translate user content (posts, topics) to. If empty, no languages will be automatically translated."

config/locales/server.he.yml

@@ -25,7 +25,7 @@ he:
     restrict_translation_by_poster_group: "לאפשר תרגום של פוסטים שנוצרו על ידי משתמשים בקבוצות מורשות בלבד. אם ריק, לאפשר תרגומים של פוסטים מכל המשתמשים."
     experimental_anon_language_switcher: "הפעלת בורר שפות ניסיוני. כך יוכלו מבקרי האתר שלא נכנסו לחשבונות המשתמש שלהם לעבור בין הגרסאות המתורגמות של Discourse ותוכן שנתרם על ידי המשתמשים בנושאים."
     errors:
-      needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_maximum_translations_per_hour’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת."
+      needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_rate’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת."
       experimental_anon_language_switcher_requirements: "בורר השפה הניסיוני דורש את הפעלת הגדרת האתר `set locale from cookie` (הגדרת השפה מהעוגיות), ושתחת `automatic translation target languages` (שפות יעד לתרגום אוטומטי) תהיה לפחות שפה אחת."
     experimental_inline_translation: "הפעלת יכולת ניסיונית לתרגום פנימי. מחליפה תרגום מקבילי קיים, מה שמאפשר למבקרי האתר עם שפה ששונה מברירת המחדל לצפות בתוכן בשפה שלהם."
     automatic_translation_target_languages: "השפה אליה לתרגם את תוכן המשתמשים אוטומטית (פוסטים, נושאים). אם השדה ריק, אף שפה לא תתורגם אוטומטית."

config/locales/server.ro.yml

@@ -24,7 +24,7 @@ ro:
     restrict_translation_by_group: "Doar grupurile permise pot traduce"
     restrict_translation_by_poster_group: "Permite traducerea numai a articolelor făcute de utilizatorii din grupurile permise. Dacă este gol, permite traducerea articolelor de la toți utilizatorii."
     errors:
-      needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_maximum_translations_per_hour” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită."
+      needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_rate” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită."
     experimental_inline_translation: "Activează funcția experimentală de traducere în-linie. Aceasta înlocuiește traducerea paralelă existentă, permițând vizitatorilor site-ului cu o locală care nu este implicită să vadă conținutul în limba lor."
     automatic_translation_target_languages: "Limbile pentru a traduce automat conținutul utilizatorului (postări, subiecte). Dacă este gol, nicio limbă nu va fi tradusă automat."
   translator:

config/settings.yml

@@ -22,7 +22,7 @@ discourse_translator:
     allow_any: false
     choices: "DiscourseTranslator::TranslatableLanguagesSetting.values"
     validator: "DiscourseTranslator::Validators::TranslatableLanguagesValidator"
-  automatic_translation_backfill_maximum_translations_per_hour:
+  automatic_translation_backfill_rate:
     default: 0
     client: false
     hidden: true

lib/discourse_translator/validators/translatable_languages_validator.rb

@@ -10,7 +10,7 @@ def initialize(opts = {})
       def valid_value?(val)
         return true if val.blank?
 
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour > 0
+        SiteSetting.automatic_translation_backfill_rate > 0
       end
 
       def error_message

spec/jobs/automatic_translation_backfill_spec.rb

@@ -59,15 +59,15 @@ def expect_google_translate(text)
     end
 
     it "does not backfill if backfill limit is set to 0" do
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "de"
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 0
+      SiteSetting.automatic_translation_backfill_rate = 0
       expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch)
     end
 
     describe "with two locales ['de', 'es']" do
       before do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+        SiteSetting.automatic_translation_backfill_rate = 100
         SiteSetting.automatic_translation_target_languages = "de|es"
         SiteSetting.automatic_translation_backfill_limit_to_public_content = true
         SiteSetting.automatic_translation_backfill_max_age_days = 5
@@ -159,7 +159,7 @@ def expect_google_translate(text)
 
     describe "with just one locale ['de']" do
       before do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+        SiteSetting.automatic_translation_backfill_rate = 100
         SiteSetting.automatic_translation_target_languages = "de"
         expect_google_check_language
       end

spec/jobs/translate_translatable_spec.rb

@@ -8,7 +8,7 @@
   before do
     SiteSetting.translator_enabled = true
     SiteSetting.translator_provider = "Google"
-    SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+    SiteSetting.automatic_translation_backfill_rate = 100
     SiteSetting.automatic_translation_target_languages = "es|fr"
     allow(DiscourseTranslator::Google).to receive(:translate)
   end

spec/lib/guardian_extension_spec.rb

@@ -180,7 +180,7 @@
         before do
           SiteSetting.experimental_inline_translation = true
 
-          SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+          SiteSetting.automatic_translation_backfill_rate = 1
           SiteSetting.automatic_translation_target_languages = "pt"
         end
 

spec/models/post_spec.rb

@@ -139,7 +139,7 @@
     fab!(:user)
 
     it "enqueues translate_translatable job when post cooked" do
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+      SiteSetting.automatic_translation_backfill_rate = 100
       SiteSetting.automatic_translation_target_languages = "es"
       post = Fabricate(:post, user: user)
       CookedPostProcessor.new(post).post_process
@@ -154,7 +154,7 @@
     end
 
     it "does not enqueue translate_translatable job for bot posts" do
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "es"
       post = Fabricate(:post, user: Discourse.system_user)
       CookedPostProcessor.new(post).post_process

spec/serializers/basic_topic_serializer_spec.rb

@@ -21,7 +21,7 @@
       SiteSetting.experimental_inline_translation = true
       I18n.locale = "ja"
 
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "ja"
     end
 

spec/serializers/post_serializer_spec.rb

@@ -77,7 +77,7 @@
     it "returns true when there is a translation for the user's locale in target languages" do
       SiteSetting.translator_enabled = true
       SiteSetting.experimental_inline_translation = true
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "ja"
       I18n.locale = "ja"
       post.set_detected_locale("en")
@@ -90,7 +90,7 @@
     it "returns false when there is a translation for the user's locale not in target languages" do
       SiteSetting.translator_enabled = true
       SiteSetting.experimental_inline_translation = true
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "es"
       I18n.locale = "ja"
       post.set_detected_locale("en")
@@ -103,7 +103,7 @@
     it "returns false when there is no translation for the current locale in target languages" do
       SiteSetting.translator_enabled = true
       SiteSetting.experimental_inline_translation = true
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "ja"
       I18n.locale = "ja"
       post.set_translation("es", "Hola")
@@ -133,7 +133,7 @@ def serialize_post(guardian_user: user, params: {})
 
     it "does not return translated_cooked when show=original param is present" do
       I18n.locale = "ja"
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "ja"
       post.set_translation("ja", "こんにちは")
 
@@ -150,7 +150,7 @@ def serialize_post(guardian_user: user, params: {})
     end
 
     it "returns translated content based on locale presence in target languages" do
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       post.set_translation("ja", "こんにちは")
       post.set_translation("es", "Hola")
       I18n.locale = "ja"

spec/serializers/topic_view_serializer_spec.rb

@@ -59,7 +59,7 @@
       before do
         SiteSetting.experimental_inline_translation = true
 
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+        SiteSetting.automatic_translation_backfill_rate = 1
         SiteSetting.automatic_translation_target_languages = "es"
 
         SiteSetting.default_locale = "en"
@@ -159,7 +159,7 @@ def serialize_topic(guardian_user: user, params: {})
       end
 
       it "returns translated title in fancy_title when translation exists for current locale" do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+        SiteSetting.automatic_translation_backfill_rate = 1
         SiteSetting.automatic_translation_target_languages = "ja"
         topic.set_translation("ja", jap_title)
         expect(serialize_topic.fancy_title).to eq("<h1>フス・ロ・ダ・ア</h1>")
@@ -170,7 +170,7 @@ def serialize_topic(guardian_user: user, params: {})
       fab!(:user)
 
       before do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+        SiteSetting.automatic_translation_backfill_rate = 1
         SiteSetting.automatic_translation_target_languages = "ja"
       end
 

spec/system/anon_language_switcher_spec.rb

@@ -12,7 +12,7 @@
     SiteSetting.translator_enabled = true
     SiteSetting.allow_user_locale = true
     SiteSetting.set_locale_from_cookie = true
-    SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+    SiteSetting.automatic_translation_backfill_rate = 1
     SiteSetting.automatic_translation_target_languages = "es|ja"
     SiteSetting.experimental_anon_language_switcher = true
     visit("/")

spec/system/full_page_translation_spec.rb

@@ -34,7 +34,7 @@
       SiteSetting.set_locale_from_cookie = true
       SiteSetting.set_locale_from_param = true
       SiteSetting.experimental_inline_translation = true
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "ja"
     end