FIX: Split backfill into separate migrations to use independent transactions

Author: romanrizzi

db/migrate/20241230153300_new_embeddings_tables.rb

@@ -65,22 +65,6 @@ def up
         WHERE model_id = #{model_id} AND strategy_id = 1;
       SQL
     end
-
-    # Copy data from old tables to new tables
-    execute <<~SQL
-      INSERT INTO ai_topics_embeddings (topic_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
-      SELECT * FROM ai_topic_embeddings;
-    SQL
-
-    execute <<~SQL
-      INSERT INTO ai_posts_embeddings (post_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
-      SELECT * FROM ai_post_embeddings;
-    SQL
-
-    execute <<~SQL
-      INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
-      SELECT * FROM ai_document_fragment_embeddings;
-    SQL
   end
 
   def down

db/migrate/20241230153301_backfill_topic_embeddings.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+class BackfillTopicEmbeddings < ActiveRecord::Migration[7.2]
+  def up
+    not_backfilled = DB.query_single("SELECT COUNT(*) FROM ai_topics_embeddings").first.to_i == 0
+
+    if not_backfilled
+      # Copy data from old tables to new tables
+      execute <<~SQL
+        INSERT INTO ai_topics_embeddings (topic_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
+        SELECT * FROM ai_topic_embeddings;
+      SQL
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20241230153302_backfill_post_embeddings.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+class BackfillPostEmbeddings < ActiveRecord::Migration[7.2]
+  def up
+    not_backfilled = DB.query_single("SELECT COUNT(*) FROM ai_posts_embeddings").first.to_i == 0
+
+    if not_backfilled
+      # Copy data from old tables to new tables
+      execute <<~SQL
+        INSERT INTO ai_posts_embeddings (post_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
+        SELECT * FROM ai_post_embeddings;
+      SQL
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20241230153303_backfill_rag_embeddings.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+class BackfillRagEmbeddings < ActiveRecord::Migration[7.2]
+  def up
+    not_backfilled =
+      DB.query_single("SELECT COUNT(*) FROM ai_document_fragments_embeddings").first.to_i == 0
+
+    if not_backfilled
+      # Copy data from old tables to new tables
+      execute <<~SQL
+        INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
+        SELECT * FROM ai_document_fragment_embeddings;
+      SQL
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end