Merge branch 'main' into ux/use-d-stat-tiles

Author: martin-brennan

app/jobs/regular/digest_rag_upload.rb

@@ -18,9 +18,7 @@ def execute(args)
       target = target_type.constantize.find_by(id: target_id)
       return if !target
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
 
       tokenizer = vector_rep.tokenizer
       chunk_tokens = target.rag_chunk_tokens

app/jobs/regular/fast_track_topic_gist.rb

@@ -14,7 +14,7 @@ def execute(args)
 
       summarizer = DiscourseAi::Summarization.topic_gist(topic)
       gist = summarizer.existing_summary
-      return if gist.present? && !gist.outdated
+      return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
 
       summarizer.force_summarize(Discourse.system_user)
     end

app/jobs/regular/generate_embeddings.rb

@@ -16,9 +16,7 @@ def execute(args)
       return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
       return if post.raw.blank?
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
 
       vector_rep.generate_representation_from(target)
     end

app/jobs/regular/generate_rag_embeddings.rb

@@ -8,9 +8,7 @@ class GenerateRagEmbeddings < ::Jobs::Base
     def execute(args)
       return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
 
       # generate_representation_from checks compares the digest value to make sure
       # the embedding is only generated once per fragment unless something changes.

app/jobs/scheduled/embeddings_backfill.rb

@@ -20,10 +20,8 @@ def execute(args)
 
       rebaked = 0
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-      table_name = vector_rep.topic_table_name
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
+      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
 
       topics =
         Topic
@@ -41,7 +39,7 @@ def execute(args)
       relation = topics.where(<<~SQL).limit(limit - rebaked)
           #{table_name}.model_version < #{vector_rep.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_rep.strategy_version}
         SQL
 
       rebaked += populate_topic_embeddings(vector_rep, relation)
@@ -63,7 +61,7 @@ def execute(args)
       return unless SiteSetting.ai_embeddings_per_post_enabled
 
       # Now for posts
-      table_name = vector_rep.post_table_name
+      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
       posts_batch_size = 1000
 
       posts =
@@ -90,7 +88,7 @@ def execute(args)
         .where(<<~SQL)
           #{table_name}.model_version < #{vector_rep.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_rep.strategy_version}
         SQL
         .limit(limit - rebaked)
         .pluck(:id)
@@ -121,7 +119,8 @@ def execute(args)
     def populate_topic_embeddings(vector_rep, topics, force: false)
       done = 0
 
-      topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
+      topics =
+        topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
 
       ids = topics.pluck("topics.id")
       batch_size = 1000

app/jobs/scheduled/summaries_backfill.rb

@@ -14,6 +14,7 @@ def execute(_args)
 
       if SiteSetting.ai_summary_gists_enabled
         gist_t = AiSummary.summary_types[:gist]
+
         backfill_candidates(gist_t)
           .limit(current_budget(gist_t))
           .each do |topic|
@@ -42,8 +43,13 @@ def backfill_candidates(summary_type)
         SQL
         .where("topics.created_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
         .where(
-          "ais.id IS NULL OR UPPER(ais.content_range) < topics.highest_post_number + 1",
-        ) # (1..1) gets stored ad (1..2).
+          <<~SQL, # (1..1) gets stored ad (1..2).
+          ais.id IS NULL OR (
+            UPPER(ais.content_range) < topics.highest_post_number + 1 
+            AND ais.created_at < (current_timestamp - INTERVAL '5 minutes')
+          )
+        SQL
+        )
         .order("ais.created_at DESC NULLS FIRST, topics.last_posted_at DESC")
     end
 

app/models/rag_document_fragment.rb

@@ -39,11 +39,7 @@ def update_target_uploads(target, upload_ids)
     end
 
     def indexing_status(persona, uploads)
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
-
-      embeddings_table = vector_rep.rag_fragments_table_name
+      embeddings_table = DiscourseAi::Embeddings::Schema.for(self).table
 
       results =
         DB.query(

assets/javascripts/discourse/services/gists.js

@@ -7,15 +7,9 @@ export default class Gists extends Service {
   @tracked preference = localStorage.getItem("topicListLayout");
 
   get shouldShow() {
-    const currentRoute = this.router.currentRoute.name;
-    const isDiscovery = currentRoute.includes("discovery");
-    const isNotCategories = !currentRoute.includes("categories");
-    const gistsAvailable =
-      this.router.currentRoute.attributes?.list?.topics?.some(
-        (topic) => topic.ai_topic_gist
-      );
-
-    return isDiscovery && isNotCategories && gistsAvailable;
+    return this.router.currentRoute.attributes?.list?.topics?.some(
+      (topic) => topic.ai_topic_gist
+    );
   }
 
   setPreference(value) {

config/locales/server.en.yml

@@ -12,7 +12,7 @@ en:
         flagged_post: |
           <div>Response from the model:</div>
           <p>%%LLM_RESPONSE%%</p>
-          <b>Triggered by the <a href="/admin/plugins/discourse-automation/%%AUTOMATION_ID%%">%%AUTOMATION_NAME%%</a> rule.</b>
+          <b>Triggered by the <a href="%{base_path}/admin/plugins/discourse-automation/%%AUTOMATION_ID%%">%%AUTOMATION_NAME%%</a> rule.</b>
       llm_report:
         title: Periodic report using AI
         description: "Periodic report based on a large language model"

db/migrate/20230710171143_migrate_embeddings_from_dedicated_database.rb

@@ -14,7 +14,7 @@ def up
       ].map { |k| k.new(truncation) }
 
     vector_reps.each do |vector_rep|
-      new_table_name = vector_rep.topic_table_name
+      new_table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
       old_table_name = "topic_embeddings_#{vector_rep.name.underscore}"
 
       begin