Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions lib/tasks/modules/embeddings/database.rake
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
# frozen_string_literal: true

desc "Backfill embeddings for all topics and posts"
task "ai:embeddings:backfill", %i[model concurrency] => [:environment] do |_, args|
task "ai:embeddings:backfill", %i[embedding_def_id concurrency] => [:environment] do |_, args|
public_categories = Category.where(read_restricted: false).pluck(:id)

if args[:model].present?
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_rep =
DiscourseAi::Embeddings::VectorRepresentations::Base.find_representation(args[:model]).new(
strategy,
)
if args[:embedding_def_id].present?
vdef = EmbeddingDefinition.find(args[:embedding_def_id])
vector_rep = DiscourseAi::Embeddings::Vector.new(vdef)
else
vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
vector_rep = DiscourseAi::Embeddings::Vector.instance
end
table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
topics_table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE

topics =
Topic
.joins("LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id")
.where("#{table_name}.topic_id IS NULL")
.joins("LEFT JOIN #{topics_table_name} ON #{topics_table_name}.topic_id = topics.id")
.where("#{topics_table_name}.topic_id IS NULL")
.where("category_id IN (?)", public_categories)
.where(deleted_at: nil)
.order("topics.id DESC")
Expand All @@ -29,11 +26,11 @@ task "ai:embeddings:backfill", %i[model concurrency] => [:environment] do |_, ar
end
end

table_name = vector_rep.post_table_name
posts_table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
posts =
Post
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
.where("#{table_name}.post_id IS NULL")
.joins("LEFT JOIN #{posts_table_name} ON #{posts_table_name}.post_id = posts.id")
.where("#{posts_table_name}.post_id IS NULL")
.where(deleted_at: nil)
.order("posts.id DESC")

Expand Down
Loading