Working deduplication

Author: xfalcox

db/migrate/20250508183456_create_inferred_concepts_topics.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class CreateInferredConceptsTopics < ActiveRecord::Migration[7.0]
+  def change
+    create_table :inferred_concepts_topics, id: false do |t|
+      t.belongs_to :inferred_concept
+      t.belongs_to :topic
+      t.timestamps
+    end
+  end
+end

db/migrate/20250508183456_create_topics_inferred_concepts.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class CreateInferredConceptsPosts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :inferred_concepts_posts, id: false do |t|
+      t.belongs_to :inferred_concept
+      t.belongs_to :post
+      t.timestamps
+    end
+  end
+end

db/migrate/20250509000001_create_inferred_concepts_posts.rb

@@ -65,44 +65,6 @@ def self.post_content_for_analysis(post)
         content
       end
 
-      # Comprehensive method to analyze a topic and apply concepts
-      def self.analyze_and_apply(topic)
-        return if topic.blank?
-
-        # Get content to analyze
-        content = topic_content_for_analysis(topic)
-
-        # Identify concepts
-        concept_names = Finder.identify_concepts(content)
-
-        # Create or find concepts in the database
-        concepts = Finder.create_or_find_concepts(concept_names)
-
-        # Apply concepts to the topic
-        apply_to_topic(topic, concepts)
-
-        concepts
-      end
-
-      # Comprehensive method to analyze a post and apply concepts
-      def self.analyze_and_apply_post(post)
-        return if post.blank?
-
-        # Get content to analyze
-        content = post_content_for_analysis(post)
-
-        # Identify concepts
-        concept_names = Finder.identify_concepts(content)
-
-        # Create or find concepts in the database
-        concepts = Finder.create_or_find_concepts(concept_names)
-
-        # Apply concepts to the post
-        apply_to_post(post, concepts)
-
-        concepts
-      end
-
       # Match a topic with existing concepts
       def self.match_existing_concepts(topic)
         return [] if topic.blank?
@@ -158,7 +120,7 @@ def self.match_concepts_to_content(content, concept_list)
 
         # Use the ConceptMatcher persona to match concepts
         llm = DiscourseAi::Completions::Llm.default_llm
-        persona = DiscourseAi::Personas::ConceptMatcher.new(concept_list: concept_list)
+        persona = DiscourseAi::Personas::ConceptMatcher.new
         context =
           DiscourseAi::Personas::BotContext.new(
             messages: [{ type: :user, content: user_message }],

db/migrate/20250509000001_create_posts_inferred_concepts.rb

@@ -68,7 +68,7 @@ def self.deduplicate_concepts_by_letter(per_letter_batch: 50, full_pass_batch: 1
           final_result.uniq!
 
           # Apply the deduplicated concepts
-          InferredConcept.destroy_all
+          InferredConcept.where.not(name: final_result).destroy_all
           InferredConcept.insert_all(final_result.map { { name: it } })
         end
       end