Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.

Commit 65456c8

Browse files
romanrizziromanrizzi
authored
DEV: Migration to remove old embeddings tables~ (#1067)
* DEV: Migration to remove old embeddings tables~ * Check for table existence
1 parent c4d2b7d commit 65456c8

File tree

4 files changed

+102
-43
lines changed

4 files changed

+102
-43
lines changed

‎db/migrate/20250114160417_backfill_topic_embeddings.rb‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,26 @@ class BackfillTopicEmbeddings < ActiveRecord::Migration[7.2]
33
disable_ddl_transaction!
44

55
def up
6-
loop do
7-
count = execute(<<~SQL).cmd_tuples
8-
INSERT INTO ai_topics_embeddings (topic_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
9-
SELECT source.*
10-
FROM (
11-
SELECT old_table.*
12-
FROM ai_topic_embeddings old_table
13-
LEFT JOIN ai_topics_embeddings target ON (
14-
target.model_id = old_table.model_id AND
15-
target.strategy_id = old_table.strategy_id AND
16-
target.topic_id = old_table.topic_id
17-
)
18-
WHERE target.topic_id IS NULL
19-
LIMIT 10000
20-
) source
21-
SQL
6+
if table_exists?(:ai_topic_embeddings)
7+
loop do
8+
count = execute(<<~SQL).cmd_tuples
9+
INSERT INTO ai_topics_embeddings (topic_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
10+
SELECT source.*
11+
FROM (
12+
SELECT old_table.*
13+
FROM ai_topic_embeddings old_table
14+
LEFT JOIN ai_topics_embeddings target ON (
15+
target.model_id = old_table.model_id AND
16+
target.strategy_id = old_table.strategy_id AND
17+
target.topic_id = old_table.topic_id
18+
)
19+
WHERE target.topic_id IS NULL
20+
LIMIT 10000
21+
) source
22+
SQL
2223

23-
break if count == 0
24+
break if count == 0
25+
end
2426
end
2527
end
2628

‎db/migrate/20250114160446_backfill_post_embeddings.rb‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,28 @@ class BackfillPostEmbeddings < ActiveRecord::Migration[7.2]
33
disable_ddl_transaction!
44

55
def up
6-
# Copy data from old tables to new tables in batches.
6+
if table_exists?(:ai_post_embeddings)
7+
# Copy data from old tables to new tables in batches.
78

8-
loop do
9-
count = execute(<<~SQL).cmd_tuples
10-
INSERT INTO ai_posts_embeddings (post_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
11-
SELECT source.*
12-
FROM (
13-
SELECT old_table.*
14-
FROM ai_post_embeddings old_table
15-
LEFT JOIN ai_posts_embeddings target ON (
16-
target.model_id = old_table.model_id AND
17-
target.strategy_id = old_table.strategy_id AND
18-
target.post_id = old_table.post_id
19-
)
20-
WHERE target.post_id IS NULL
21-
LIMIT 10000
22-
) source
23-
SQL
9+
loop do
10+
count = execute(<<~SQL).cmd_tuples
11+
INSERT INTO ai_posts_embeddings (post_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
12+
SELECT source.*
13+
FROM (
14+
SELECT old_table.*
15+
FROM ai_post_embeddings old_table
16+
LEFT JOIN ai_posts_embeddings target ON (
17+
target.model_id = old_table.model_id AND
18+
target.strategy_id = old_table.strategy_id AND
19+
target.post_id = old_table.post_id
20+
)
21+
WHERE target.post_id IS NULL
22+
LIMIT 10000
23+
) source
24+
SQL
2425

25-
break if count == 0
26+
break if count == 0
27+
end
2628
end
2729
end
2830

‎db/migrate/20250114160500_backfill_rag_embeddings.rb‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
# frozen_string_literal: true
22
class BackfillRagEmbeddings < ActiveRecord::Migration[7.2]
33
def up
4-
not_backfilled =
5-
DB.query_single("SELECT COUNT(*) FROM ai_document_fragments_embeddings").first.to_i == 0
4+
if table_exists?(:ai_document_fragment_embeddings)
5+
not_backfilled =
6+
DB.query_single("SELECT COUNT(*) FROM ai_document_fragments_embeddings").first.to_i == 0
67

7-
if not_backfilled
8-
# Copy data from old tables to new tables
9-
execute <<~SQL
10-
INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
11-
SELECT * FROM ai_document_fragment_embeddings;
12-
SQL
8+
if not_backfilled
9+
# Copy data from old tables to new tables
10+
execute <<~SQL
11+
INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
12+
SELECT * FROM ai_document_fragment_embeddings;
13+
SQL
14+
end
1315
end
1416
end
1517

‎db/post_migrate/20250114184356_drop_old_embedding_tables2.rb‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# frozen_string_literal: true
2+
class DropOldEmbeddingTables2 < ActiveRecord::Migration[7.2]
3+
def up
4+
# Copy rag embeddings created during deploy.
5+
execute <<~SQL
6+
INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
7+
(
8+
SELECT old_table.*
9+
FROM ai_document_fragment_embeddings old_table
10+
LEFT OUTER JOIN ai_document_fragments_embeddings target ON (
11+
target.model_id = old_table.model_id AND
12+
target.strategy_id = old_table.strategy_id AND
13+
target.rag_document_fragment_id = old_table.rag_document_fragment_id
14+
)
15+
WHERE target.rag_document_fragment_id IS NULL
16+
)
17+
SQL
18+
19+
execute <<~SQL
20+
DROP INDEX IF EXISTS ai_topic_embeddings_1_1_search_bit;
21+
DROP INDEX IF EXISTS ai_topic_embeddings_2_1_search_bit;
22+
DROP INDEX IF EXISTS ai_topic_embeddings_3_1_search_bit;
23+
DROP INDEX IF EXISTS ai_topic_embeddings_4_1_search_bit;
24+
DROP INDEX IF EXISTS ai_topic_embeddings_5_1_search_bit;
25+
DROP INDEX IF EXISTS ai_topic_embeddings_6_1_search_bit;
26+
DROP INDEX IF EXISTS ai_topic_embeddings_7_1_search_bit;
27+
DROP INDEX IF EXISTS ai_topic_embeddings_8_1_search_bit;
28+
DROP INDEX IF EXISTS ai_post_embeddings_1_1_search_bit;
29+
DROP INDEX IF EXISTS ai_post_embeddings_2_1_search_bit;
30+
DROP INDEX IF EXISTS ai_post_embeddings_3_1_search_bit;
31+
DROP INDEX IF EXISTS ai_post_embeddings_4_1_search_bit;
32+
DROP INDEX IF EXISTS ai_post_embeddings_5_1_search_bit;
33+
DROP INDEX IF EXISTS ai_post_embeddings_6_1_search_bit;
34+
DROP INDEX IF EXISTS ai_post_embeddings_7_1_search_bit;
35+
DROP INDEX IF EXISTS ai_post_embeddings_8_1_search_bit;
36+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_1_1_search_bit;
37+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_2_1_search_bit;
38+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_3_1_search_bit;
39+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_4_1_search_bit;
40+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_5_1_search_bit;
41+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_6_1_search_bit;
42+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_7_1_search_bit;
43+
DROP INDEX IF EXISTS ai_document_fragment_embeddings_8_1_search_bit;
44+
SQL
45+
46+
drop_table :ai_topic_embeddings, if_exists: true
47+
drop_table :ai_post_embeddings, if_exists: true
48+
drop_table :ai_document_fragment_embeddings, if_exists: true
49+
end
50+
51+
def down
52+
end
53+
end

0 commit comments

Comments
 (0)