DEV: Change backfill rate handling (#256)

Author: xfalcox

app/jobs/scheduled/automatic_translation_backfill.rb

@@ -14,34 +14,41 @@ def execute(args = nil)
 
     def fetch_untranslated_model_ids(model, content_column, limit, target_locale)
       m = model.name.downcase
+
+      # Query selects every model (post/topic) *except* those who are **both**
+      # already locale detected and translated
       DB.query_single(<<~SQL, target_locale: target_locale, limit: limit)
-        SELECT m.id
-        FROM #{model.table_name} m
-        #{limit_to_public_clause(model)}
-        WHERE m.deleted_at IS NULL
-          AND m.#{content_column} != ''
-          AND m.user_id > 0
-          #{max_age_clause}
-          AND (
-            NOT EXISTS (
-              SELECT 1
-              FROM discourse_translator_#{m}_locales
-              WHERE #{m}_id = m.id
+        SELECT * FROM
+        (
+          ( -- every post / topic
+            SELECT m.id
+            FROM #{model.table_name} m
+            #{limit_to_public_clause(model)}
+            WHERE m.deleted_at IS NULL
+              AND m.#{content_column} != ''
+              AND m.user_id > 0
+              #{max_age_clause}
+            ORDER BY m.updated_at DESC
+          )
+          EXCEPT
+          (
+            ( -- locale detected
+              SELECT
+                #{m}_id
+              FROM
+                discourse_translator_#{m}_locales
+              WHERE
+                detected_locale = :target_locale
             )
-            OR EXISTS (
-              SELECT 1
-              FROM discourse_translator_#{m}_locales
-              WHERE #{m}_id = m.id
-              AND detected_locale != :target_locale
+            INTERSECT
+            ( -- translated
+              SELECT #{m}_id
+              FROM discourse_translator_#{m}_translations
+              WHERE
+                locale = :target_locale
             )
           )
-          AND NOT EXISTS (
-            SELECT 1
-            FROM discourse_translator_#{m}_translations
-            WHERE #{m}_id = m.id
-            AND locale = :target_locale
-          )
-        ORDER BY m.updated_at DESC
+        ) AS subquery
         LIMIT :limit
       SQL
     end
@@ -50,14 +57,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale)
 
     def should_backfill?
       return false if SiteSetting.automatic_translation_target_languages.blank?
-      return false if SiteSetting.automatic_translation_backfill_maximum_translations_per_hour == 0
+      return false if SiteSetting.automatic_translation_backfill_rate == 0
       true
     end
 
-    def translations_per_run
-      [(SiteSetting.automatic_translation_backfill_maximum_translations_per_hour / 12), 1].max
-    end
-
     def backfill_locales
       @backfill_locales ||=
         SiteSetting.automatic_translation_target_languages.split("|").map { |l| l.gsub("_", "-") }
@@ -83,23 +86,12 @@ def translate_records(type, record_ids, target_locale)
     end
 
     def process_batch
-      models_translated = [Post, Topic].size
-      avg_translations_per_model_per_language = [
-        translations_per_run / models_translated / backfill_locales.size,
-        1,
-      ].max
-
-      records_to_translate = avg_translations_per_model_per_language
+      records_to_translate = SiteSetting.automatic_translation_backfill_rate
       backfill_locales.each_with_index do |target_locale, i|
         topic_ids =
           fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale)
         post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale)
 
-        # if we end up translating fewer records than records_to_translate,
-        # add to the value so that the next locales can have more quota
-        records_to_translate =
-          avg_translations_per_model_per_language +
-            ((records_to_translate - topic_ids.size - post_ids.size) / backfill_locales.size - i)
         next if topic_ids.empty? && post_ids.empty?
 
         DiscourseTranslator::VerboseLogger.log(

config/locales/server.de.yml

@@ -25,7 +25,7 @@ de:
     restrict_translation_by_poster_group: "Erlaube nur die Übersetzung von Beiträgen von Benutzern aus zulässigen Gruppen. Wenn das Feld leer ist, wird die Übersetzung von Beiträgen aller Benutzer erlaubt."
     experimental_anon_language_switcher: "Aktiviere die experimentelle Sprachumschaltung. Damit können nicht eingeloggte Besucher zwischen übersetzten Versionen von Discourse und von Benutzern beigetragenen Inhalten in Themen wechseln."
     errors:
-      needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_maximum_translations_per_hour' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen."
+      needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_rate' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen."
       experimental_anon_language_switcher_requirements: "Die experimentelle Sprachumschaltung setzt voraus, dass die Website-Einstellung `set locale from cookie` aktiviert ist und `automatic translation target languages` mindestens eine Sprache enthält."
     experimental_inline_translation: "Aktiviere die experimentelle Übersetzungsfunktion mit Überschreiben. Diese ersetzt die bisherige parallele Übersetzung und ermöglicht es Besuchern der Website, die nicht das Standardgebietsschema verwenden, Inhalte direkt in ihrer Sprache zu sehen."
     automatic_translation_target_languages: "Die Sprachen, in die Benutzerinhalte (Beiträge, Themen) automatisch übersetzt werden sollen. Wenn nichts ausgewählt ist, wird in keine Sprachen automatisch übersetzt."

config/locales/server.en.yml

@@ -19,7 +19,7 @@ en:
     restrict_translation_by_poster_group: "Only allow translation of posts made by users in allowed groups. If empty, allow translations of posts from all users."
     experimental_anon_language_switcher: "Enable experimental language switcher. This will allow site visitors who are not logged in to switch between translated versions of Discourse and user-contributed content in topics."
     errors:
-      needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_maximum_translations_per_hour' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit."
+      needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_rate' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit."
       experimental_anon_language_switcher_requirements: "The experimental language switcher requires the `set locale from cookie` site setting to be enabled, and the `automatic translation target languages` to have at least one language."
     experimental_inline_translation: "Enable experimental inline translation feature. This replaces existing parallel translation, allowing site visitors with a non-default locale to view content in their language."
     automatic_translation_target_languages: "The languages to automatically translate user content (posts, topics) to. If empty, no languages will be automatically translated."

config/locales/server.he.yml

@@ -25,7 +25,7 @@ he:
     restrict_translation_by_poster_group: "לאפשר תרגום של פוסטים שנוצרו על ידי משתמשים בקבוצות מורשות בלבד. אם ריק, לאפשר תרגומים של פוסטים מכל המשתמשים."
     experimental_anon_language_switcher: "הפעלת בורר שפות ניסיוני. כך יוכלו מבקרי האתר שלא נכנסו לחשבונות המשתמש שלהם לעבור בין הגרסאות המתורגמות של Discourse ותוכן שנתרם על ידי המשתמשים בנושאים."
     errors:
-      needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_maximum_translations_per_hour’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת."
+      needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_rate’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת."
       experimental_anon_language_switcher_requirements: "בורר השפה הניסיוני דורש את הפעלת הגדרת האתר `set locale from cookie` (הגדרת השפה מהעוגיות), ושתחת `automatic translation target languages` (שפות יעד לתרגום אוטומטי) תהיה לפחות שפה אחת."
     experimental_inline_translation: "הפעלת יכולת ניסיונית לתרגום פנימי. מחליפה תרגום מקבילי קיים, מה שמאפשר למבקרי האתר עם שפה ששונה מברירת המחדל לצפות בתוכן בשפה שלהם."
     automatic_translation_target_languages: "השפה אליה לתרגם את תוכן המשתמשים אוטומטית (פוסטים, נושאים). אם השדה ריק, אף שפה לא תתורגם אוטומטית."

config/locales/server.ro.yml

@@ -24,7 +24,7 @@ ro:
     restrict_translation_by_group: "Doar grupurile permise pot traduce"
     restrict_translation_by_poster_group: "Permite traducerea numai a articolelor făcute de utilizatorii din grupurile permise. Dacă este gol, permite traducerea articolelor de la toți utilizatorii."
     errors:
-      needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_maximum_translations_per_hour” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită."
+      needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_rate” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită."
     experimental_inline_translation: "Activează funcția experimentală de traducere în-linie. Aceasta înlocuiește traducerea paralelă existentă, permițând vizitatorilor site-ului cu o locală care nu este implicită să vadă conținutul în limba lor."
     automatic_translation_target_languages: "Limbile pentru a traduce automat conținutul utilizatorului (postări, subiecte). Dacă este gol, nicio limbă nu va fi tradusă automat."
   translator:

config/settings.yml

@@ -22,7 +22,7 @@ discourse_translator:
     allow_any: false
     choices: "DiscourseTranslator::TranslatableLanguagesSetting.values"
     validator: "DiscourseTranslator::Validators::TranslatableLanguagesValidator"
-  automatic_translation_backfill_maximum_translations_per_hour:
+  automatic_translation_backfill_rate:
     default: 0
     client: false
     hidden: true

lib/discourse_translator/validators/translatable_languages_validator.rb

@@ -10,7 +10,7 @@ def initialize(opts = {})
       def valid_value?(val)
         return true if val.blank?
 
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour > 0
+        SiteSetting.automatic_translation_backfill_rate > 0
       end
 
       def error_message

spec/jobs/automatic_translation_backfill_spec.rb

@@ -59,15 +59,15 @@ def expect_google_translate(text)
     end
 
     it "does not backfill if backfill limit is set to 0" do
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+      SiteSetting.automatic_translation_backfill_rate = 1
       SiteSetting.automatic_translation_target_languages = "de"
-      SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 0
+      SiteSetting.automatic_translation_backfill_rate = 0
       expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch)
     end
 
     describe "with two locales ['de', 'es']" do
       before do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+        SiteSetting.automatic_translation_backfill_rate = 100
         SiteSetting.automatic_translation_target_languages = "de|es"
         SiteSetting.automatic_translation_backfill_limit_to_public_content = true
         SiteSetting.automatic_translation_backfill_max_age_days = 5
@@ -159,7 +159,7 @@ def expect_google_translate(text)
 
     describe "with just one locale ['de']" do
       before do
-        SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+        SiteSetting.automatic_translation_backfill_rate = 100
         SiteSetting.automatic_translation_target_languages = "de"
         expect_google_check_language
       end

spec/jobs/translate_translatable_spec.rb

@@ -8,7 +8,7 @@
   before do
     SiteSetting.translator_enabled = true
     SiteSetting.translator_provider = "Google"
-    SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100
+    SiteSetting.automatic_translation_backfill_rate = 100
     SiteSetting.automatic_translation_target_languages = "es|fr"
     allow(DiscourseTranslator::Google).to receive(:translate)
   end

spec/lib/guardian_extension_spec.rb

@@ -180,7 +180,7 @@
         before do
           SiteSetting.experimental_inline_translation = true
 
-          SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1
+          SiteSetting.automatic_translation_backfill_rate = 1
           SiteSetting.automatic_translation_target_languages = "pt"
         end