DEV: Move translation custom fields into their own tables

Author: nattsw

app/jobs/scheduled/detect_posts_language.rb

@@ -3,10 +3,10 @@
 module ::Jobs
   class DetectPostsLanguage < ::Jobs::Scheduled
     sidekiq_options retry: false
-    every 5.minutes
+    every 1.minutes
 
-    BATCH_SIZE = 100
-    MAX_QUEUE_SIZE = 1000
+    BATCH_SIZE = 10
+    MAX_QUEUE_SIZE = 100
 
     def execute(args)
       return unless SiteSetting.translator_enabled

app/models/discourse_translator/post_locale.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostLocale < ActiveRecord::Base
+    self.table_name = "discourse_translator_post_locales"
+
+    belongs_to :post
+
+    validates :post_id, presence: true, uniqueness: true
+    validates :detected_locale, presence: true
+  end
+end

app/models/discourse_translator/post_translation.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostTranslation < ActiveRecord::Base
+    self.table_name = "discourse_translator_post_translations"
+
+    belongs_to :post
+
+    validates :post_id, presence: true
+    validates :locale, presence: true
+    validates :translation, presence: true
+    validates :locale, uniqueness: { scope: :post_id }
+
+    def self.translation_for(post_id, locale)
+      find_by(post_id: post_id, locale: locale)&.translation
+    end
+  end
+end

app/models/discourse_translator/topic_locale.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicLocale < ActiveRecord::Base
+    self.table_name = "discourse_translator_topic_locales"
+
+    belongs_to :topic
+
+    validates :topic_id, presence: true, uniqueness: true
+    validates :detected_locale, presence: true
+  end
+end

app/models/discourse_translator/topic_translation.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TopicTranslation < ActiveRecord::Base
+    self.table_name = "discourse_translator_topic_translations"
+
+    belongs_to :topic
+
+    validates :topic_id, presence: true
+    validates :locale, presence: true
+    validates :translation, presence: true
+    validates :locale, uniqueness: { scope: :topic_id }
+
+    def self.translation_for(topic_id, locale)
+      find_by(topic_id: topic_id, locale: locale)&.translation
+    end
+  end
+end

db/migrate/20250205082400_create_translation_tables.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class CreateTranslationTables < ActiveRecord::Migration[7.2]
+  def change
+    create_table :discourse_translator_topic_locales do |t|
+      t.integer :topic_id, null: false
+      t.string :detected_locale, limit: 20, null: false
+      t.timestamps
+    end
+
+    create_table :discourse_translator_topic_translations do |t|
+      t.integer :topic_id, null: false
+      t.string :locale, null: false
+      t.text :translation, null: false
+      t.timestamps
+    end
+
+    create_table :discourse_translator_post_locales do |t|
+      t.integer :post_id, null: false
+      t.string :detected_locale, limit: 20, null: false
+      t.timestamps
+    end
+
+    create_table :discourse_translator_post_translations do |t|
+      t.integer :post_id, null: false
+      t.string :locale, null: false
+      t.text :translation, null: false
+      t.timestamps
+    end
+  end
+end

db/migrate/20250205082401_move_translations_custom_fields_to_table.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class MoveTranslationsCustomFieldsToTable < ActiveRecord::Migration[7.2] # frozen_string_literal: true
+  BATCH_SIZE = 1000
+
+  def up
+    migrate_custom_fields("topic")
+    migrate_custom_fields("post")
+  end
+
+  def down
+    execute "TRUNCATE discourse_translator_topic_locales"
+    execute "TRUNCATE discourse_translator_topic_translations"
+    execute "TRUNCATE discourse_translator_post_locales"
+    execute "TRUNCATE discourse_translator_post_translations"
+  end
+
+  private
+
+  def migrate_custom_fields(model)
+    start_id = 0
+    loop do
+      last_id = DB.query_single(<<~SQL, model:, start_id:, limit: BATCH_SIZE)
+        WITH to_insert AS (
+          SELECT id, #{model}_id, value
+          FROM #{model}_custom_fields
+          WHERE name = 'post_detected_lang'
+          AND id > :start_id
+          ORDER BY id
+          LIMIT :limit
+        ),
+        do_insert AS (
+          INSERT INTO discourse_translator_#{model}_locales (#{model}_id, detected_locale, created_at, updated_at)
+          SELECT #{model}_id, value, NOW(), NOW()
+          FROM to_insert
+        ),
+        to_translate AS (
+          SELECT id, #{model}_id, value::jsonb, created_at, updated_at
+          FROM #{model}_custom_fields
+          WHERE id > :start_id
+          AND name = 'translated_text'
+          AND value LIKE '{%}'
+          ORDER BY id
+          LIMIT :limit
+        ),
+        do_translate AS (
+          INSERT INTO discourse_translator_#{model}_translations (#{model}_id, locale, translation, created_at, updated_at)
+          SELECT b.#{model}_id, jb.key as locale, jb.value as translation, b.created_at, b.updated_at
+          FROM to_translate b, jsonb_each_text(b.value) jb
+          WHERE LENGTH(jb.key) <= 20
+        ),
+        max_value AS (SELECT COALESCE(GREATEST((SELECT MAX(id) FROM to_insert), (SELECT MAX(id) FROM to_translate) ), -1) as max_id)
+        SELECT max_id FROM max_value
+      SQL
+      start_id = last_id.last
+      break if start_id == -1
+    end
+  end
+end

db/migrate/20250205082402_create_translation_indexes.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateTranslationIndexes < ActiveRecord::Migration[7.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :discourse_translator_topic_translations,
+              %i[topic_id locale],
+              unique: true,
+              algorithm: :concurrently
+
+    add_index :discourse_translator_post_translations,
+              %i[post_id locale],
+              unique: true,
+              algorithm: :concurrently
+  end
+end

spec/models/translation_tables_spec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative "../../db/migrate/20250205082401_move_translations_custom_fields_to_table"
+
+module DiscourseTranslator
+  describe MoveTranslationsCustomFieldsToTable do
+    let!(:batch_size) { 5 }
+
+    before { described_class.const_set(:BATCH_SIZE, batch_size) }
+
+    def create_custom_fields(count)
+      count.times do
+        post = Fabricate(:post)
+        topic = Fabricate(:topic)
+
+        post.custom_fields[DETECTED_LANG_CUSTOM_FIELD] = "pt"
+        post.save_custom_fields
+
+        topic.custom_fields[DETECTED_LANG_CUSTOM_FIELD] = "es"
+        topic.save_custom_fields
+
+        post.custom_fields[TRANSLATED_CUSTOM_FIELD] = {
+          en_GB: "The Romance of the Three Kingdoms",
+          de: "Die Romanze der Drei Königreiche",
+        }
+        post.save_custom_fields
+
+        topic.custom_fields[TRANSLATED_CUSTOM_FIELD] = {
+          en_GB: "The Romance of the Three Kingdoms",
+          de: "Die Romanze der Drei Königreiche",
+        }
+        topic.save_custom_fields
+      end
+    end
+
+    it "correctly migrates custom fields in batches" do
+      # batch size is 5
+      create_custom_fields(12)
+
+      migration = described_class.new
+      migration.up
+
+      # 12 posts * 2 translations each
+      expect(PostLocale.count).to eq(12)
+      expect(PostTranslation.count).to eq(24)
+
+      # 12 topics * 2 translations each
+      expect(TopicLocale.count).to eq(12)
+      expect(TopicTranslation.count).to eq(24)
+
+      expect(PostLocale.first.detected_locale).to eq("pt")
+
+      expect(PostTranslation.where(post_id: Post.first.id).pluck(:locale, :translation)).to include(
+        ["en_GB", "The Romance of the Three Kingdoms"],
+        ["de", "Die Romanze der Drei Königreiche"],
+      )
+
+      migration.down
+      expect(PostLocale.count).to eq(0)
+      expect(PostTranslation.count).to eq(0)
+      expect(TopicLocale.count).to eq(0)
+      expect(TopicTranslation.count).to eq(0)
+    end
+
+    it "ignores invalid JSON in translated_text" do
+      post = Fabricate(:post)
+      post.custom_fields[TRANSLATED_CUSTOM_FIELD] = "invalid json"
+      post.save_custom_fields(true)
+
+      migration = described_class.new
+      expect { migration.up }.not_to raise_error
+      expect(PostTranslation.count).to eq(0)
+    end
+
+    it "ignores translations with locale longer than 20 chars" do
+      post = Fabricate(:post)
+      post.custom_fields[TRANSLATED_CUSTOM_FIELD] = { very_very_long_locale_name: "test" }
+      post.custom_fields[DETECTED_LANG_CUSTOM_FIELD] = "very_very_long_locale_name"
+      post.save_custom_fields(true)
+
+      migration = described_class.new
+      migration.up
+      expect(PostLocale.count).to eq(0)
+      expect(PostTranslation.count).to eq(0)
+    end
+  end
+end