FEATURE: add inferred concepts system

This commit adds a new inferred concepts system that:
- Creates a model for storing concept labels that can be applied to topics
- Provides AI personas for finding new concepts and matching existing ones
- Adds jobs for generating concepts from popular topics
- Includes a scheduled job that automatically processes engaging topics

Author: xfalcox

app/jobs/regular/apply_inferred_concepts.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Jobs
+  class ApplyInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: 'low'
+
+    # Process a batch of topics to apply existing concepts to them
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [Array<Integer>] :topic_ids Required - List of topic IDs to process
+    # @option args [Integer] :batch_size (100) Number of topics to process in each batch
+    def execute(args = {})
+      return if args[:topic_ids].blank?
+      
+      # Process topics in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+      
+      # Get the list of topic IDs
+      topic_ids = args[:topic_ids]
+      
+      # Process topics in batches
+      topic_ids.each_slice(batch_size) do |batch_topic_ids|
+        process_batch(batch_topic_ids)
+      end
+    end
+    
+    private
+    
+    def process_batch(topic_ids)
+      topics = Topic.where(id: topic_ids)
+      
+      topics.each do |topic|
+        begin
+          process_topic(topic)
+        rescue => e
+          Rails.logger.error("Error applying concepts to topic #{topic.id}: #{e.message}\n#{e.backtrace.join("\n")}")
+        end
+      end
+    end
+    
+    def process_topic(topic)
+      # Match topic against existing concepts and apply them
+      # Pass the topic object directly
+      DiscourseAi::InferredConcepts::Manager.match_topic_to_concepts(topic)
+    end
+  end
+end

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: 'low'
+
+    # Process a batch of topics to generate new concepts (without applying them to topics)
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [Array<Integer>] :topic_ids Required - List of topic IDs to process
+    # @option args [Integer] :batch_size (100) Number of topics to process in each batch
+    def execute(args = {})
+      return if args[:topic_ids].blank?
+      
+      # Process topics in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+      
+      # Get the list of topic IDs
+      topic_ids = args[:topic_ids]
+      
+      # Process topics in batches
+      topic_ids.each_slice(batch_size) do |batch_topic_ids|
+        process_batch(batch_topic_ids)
+      end
+    end
+    
+    private
+    
+    def process_batch(topic_ids)
+      topics = Topic.where(id: topic_ids)
+      
+      topics.each do |topic|
+        begin
+          process_topic(topic)
+        rescue => e
+          Rails.logger.error("Error generating concepts from topic #{topic.id}: #{e.message}\n#{e.backtrace.join("\n")}")
+        end
+      end
+    end
+    
+    def process_topic(topic)
+      # Use the Manager method that handles both identifying and creating concepts
+      # Pass the topic object directly
+      DiscourseAi::InferredConcepts::Manager.generate_concepts_from_topic(topic)
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularTopics < ::Jobs::Scheduled
+    every 1.day
+    
+    # This job runs daily and generates new concepts from popular topics
+    # It selects topics based on engagement metrics and generates concepts from their content
+    def execute(args = {})
+      # Find candidate topics that are popular and don't have concepts yet
+      candidates = DiscourseAi::InferredConcepts::Manager.find_candidate_topics(
+        limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+        min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+        min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+        min_views: SiteSetting.inferred_concepts_min_views || 100,
+        created_after: SiteSetting.inferred_concepts_lookback_days.days.ago
+      )
+      
+      return if candidates.blank?
+      
+      # Process the candidate topics in batches using the regular job
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        topic_ids: candidates.map(&:id),
+        batch_size: 10
+      )
+      
+      # Schedule a follow-up job to apply the concepts to topics
+      # This runs after a delay to ensure concepts have been generated
+      Jobs.enqueue_in(
+        1.hour,
+        :apply_inferred_concepts,
+        topic_ids: candidates.map(&:id),
+        batch_size: 10
+      )
+    end
+  end
+end

app/models/inferred_concept.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class InferredConcept < ActiveRecord::Base
+  has_and_belongs_to_many :topics
+
+  validates :name, presence: true, uniqueness: true
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts
+#
+#  id         :bigint           not null, primary key
+#  name       :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_on_name  (name) UNIQUE
+#

app/serializers/inferred_concept_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InferredConceptSerializer < ApplicationSerializer
+  attributes :id, :name, :created_at, :updated_at
+end

config/settings.yml

@@ -391,3 +391,28 @@ discourse_ai:
     default: true
     client: true
 
+  inferred_concepts_enabled:
+    default: false
+    client: true
+    description: "Enable the inferred concepts system that automatically generates and applies concepts to topics"
+  inferred_concepts_daily_topics_limit:
+    default: 20
+    client: false
+    description: "Maximum number of topics to process each day for concept generation"
+  inferred_concepts_min_posts:
+    default: 5
+    client: false
+    description: "Minimum number of posts a topic must have to be considered for concept generation"
+  inferred_concepts_min_likes:
+    default: 10
+    client: false
+    description: "Minimum number of likes a topic must have to be considered for concept generation"
+  inferred_concepts_min_views:
+    default: 100
+    client: false
+    description: "Minimum number of views a topic must have to be considered for concept generation"
+  inferred_concepts_lookback_days:
+    default: 30
+    client: false
+    description: "Only consider topics created within this many days for concept generation"
+

db/migrate/20250508182047_create_inferred_concepts_table.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+class CreateInferredConceptsTable < ActiveRecord::Migration[7.2]
+  def change
+    create_table :inferred_concepts do |t|
+      t.string :name, null: false
+      t.timestamps
+    end
+    
+    add_index :inferred_concepts, :name, unique: true
+  end
+end

db/migrate/20250508183456_create_topics_inferred_concepts.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateTopicsInferredConcepts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :topics_inferred_concepts do |t|
+      t.integer :topic_id, null: false
+      t.integer :inferred_concept_id, null: false
+      t.timestamps
+    end
+
+    add_index :topics_inferred_concepts, [:topic_id, :inferred_concept_id], unique: true, name: 'idx_unique_topic_inferred_concept'
+    add_index :topics_inferred_concepts, :topic_id
+    add_index :topics_inferred_concepts, :inferred_concept_id
+  end
+end

lib/inferred_concepts/applier.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module InferredConcepts
+    class Applier
+      # Associates the provided concepts with a topic
+      # topic: a Topic instance
+      # concepts: an array of InferredConcept instances
+      def self.apply_to_topic(topic, concepts)
+        return if topic.blank? || concepts.blank?
+        
+        concepts.each do |concept|
+          # Use the join table to associate the concept with the topic
+          # Avoid duplicates by using find_or_create_by
+          ActiveRecord::Base.connection.execute(<<~SQL)
+            INSERT INTO topics_inferred_concepts (topic_id, inferred_concept_id, created_at, updated_at)
+            VALUES (#{topic.id}, #{concept.id}, NOW(), NOW())
+            ON CONFLICT (topic_id, inferred_concept_id) DO NOTHING
+          SQL
+        end
+      end
+      
+      # Extracts content from a topic for concept analysis
+      # Returns a string with the topic title and first few posts
+      def self.topic_content_for_analysis(topic)
+        return "" if topic.blank?
+        
+        # Combine title and first few posts for analysis
+        posts = Post.where(topic_id: topic.id).order(:post_number).limit(10)
+        
+        content = "Title: #{topic.title}\n\n"
+        content += posts.map do |p| 
+          "#{p.post_number}) #{p.user.username}: #{p.raw}"
+        end.join("\n\n")
+        
+        content
+      end
+      
+      # Comprehensive method to analyze a topic and apply concepts
+      def self.analyze_and_apply(topic)
+        return if topic.blank?
+        
+        # Get content to analyze
+        content = topic_content_for_analysis(topic)
+        
+        # Identify concepts
+        concept_names = Finder.identify_concepts(content)
+        
+        # Create or find concepts in the database
+        concepts = Finder.create_or_find_concepts(concept_names)
+        
+        # Apply concepts to the topic
+        apply_to_topic(topic, concepts)
+        
+        concepts
+      end
+      
+      # Match a topic with existing concepts
+      def self.match_existing_concepts(topic)
+        return [] if topic.blank?
+        
+        # Get content to analyze
+        content = topic_content_for_analysis(topic)
+        
+        # Get all existing concepts
+        existing_concepts = InferredConcept.all.pluck(:name)
+        return [] if existing_concepts.empty?
+        
+        # Use the ConceptMatcher persona to match concepts
+        matched_concept_names = match_concepts_to_content(content, existing_concepts)
+        
+        # Find concepts in the database
+        matched_concepts = InferredConcept.where(name: matched_concept_names)
+        
+        # Apply concepts to the topic
+        apply_to_topic(topic, matched_concepts)
+        
+        matched_concepts
+      end
+      
+      # Use ConceptMatcher persona to match content against provided concepts
+      def self.match_concepts_to_content(content, concept_list)
+        return [] if content.blank? || concept_list.blank?
+        
+        # Prepare user message with content and concept list
+        user_message = <<~MESSAGE
+          Content to analyze:
+          #{content}
+          
+          Available concepts to match:
+          #{concept_list.join(", ")}
+        MESSAGE
+        
+        # Use the ConceptMatcher persona to match concepts
+        llm = DiscourseAi::Completions::Llm.default_llm
+        persona = DiscourseAi::Personas::ConceptMatcher.new
+        context = DiscourseAi::Personas::BotContext.new(
+          messages: [{ type: :user, content: user_message }],
+          user: Discourse.system_user
+        )
+        
+        prompt = persona.craft_prompt(context)
+        response = llm.completion(prompt, extract_json: true)
+        
+        return [] unless response.success?
+        
+        matching_concepts = response.parsed_output["matching_concepts"]
+        matching_concepts || []
+      end
+    end
+  end
+end