FEATURE: Extend inferred concepts to include posts

* Adds support for concepts to be inferred from and applied to posts
* Replaces daily task with one that handles both topics and posts
* Adds database migration for posts_inferred_concepts join table
* Updates PersonaContext to include inferred concepts

Author: xfalcox

.claude/settings.local.json

@@ -0,0 +1,8 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(bundle exec rails g migration:*)"
+    ],
+    "deny": []
+  }
+}

app/jobs/regular/apply_inferred_concepts.rb

@@ -4,44 +4,64 @@ module Jobs
   class GenerateInferredConcepts < ::Jobs::Base
     sidekiq_options queue: 'low'
 
-    # Process a batch of topics to generate new concepts (without applying them to topics)
+    # Process items to generate new concepts
     #
     # @param args [Hash] Contains job arguments
-    # @option args [Array<Integer>] :topic_ids Required - List of topic IDs to process
-    # @option args [Integer] :batch_size (100) Number of topics to process in each batch
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
     def execute(args = {})
-      return if args[:topic_ids].blank?
+      return if args[:item_ids].blank? || args[:item_type].blank?
 
-      # Process topics in smaller batches to avoid memory issues
+      unless ['topics', 'posts'].include?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+      
+      # Process items in smaller batches to avoid memory issues
       batch_size = args[:batch_size] || 100
 
-      # Get the list of topic IDs
-      topic_ids = args[:topic_ids]
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
 
-      # Process topics in batches
-      topic_ids.each_slice(batch_size) do |batch_topic_ids|
-        process_batch(batch_topic_ids)
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
       end
     end
 
     private
 
-    def process_batch(topic_ids)
-      topics = Topic.where(id: topic_ids)
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
 
-      topics.each do |topic|
+      items.each do |item|
         begin
-          process_topic(topic)
+          process_item(item, item_type, match_only)
         rescue => e
-          Rails.logger.error("Error generating concepts from topic #{topic.id}: #{e.message}\n#{e.backtrace.join("\n")}")
+          Rails.logger.error("Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}")
         end
       end
     end
 
-    def process_topic(topic)
+    def process_item(item, item_type, match_only)
       # Use the Manager method that handles both identifying and creating concepts
-      # Pass the topic object directly
-      DiscourseAi::InferredConcepts::Manager.generate_concepts_from_topic(topic)
+      if match_only
+        if item_type == 'topics'
+          DiscourseAi::InferredConcepts::Manager.match_topic_to_concepts(item)
+        else # posts
+          DiscourseAi::InferredConcepts::Manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == 'topics'
+          DiscourseAi::InferredConcepts::Manager.analyze_topic(item)
+        else # posts
+          DiscourseAi::InferredConcepts::Manager.analyze_post(item)
+        end
+      end
     end
   end
 end

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(args = {})
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+    
+    private
+    
+    def process_popular_topics
+
+      # Find candidate topics that are popular and don't have concepts yet
+      candidates = DiscourseAi::InferredConcepts::Manager.find_candidate_topics(
+        limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+        min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+        min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+        min_views: SiteSetting.inferred_concepts_min_views || 100,
+        created_after: SiteSetting.inferred_concepts_lookback_days.days.ago
+      )
+
+      return if candidates.blank?
+      
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: 'topics',
+        item_ids: candidates.map(&:id),
+        batch_size: 10
+      )
+      
+      # Schedule a follow-up job to match existing concepts
+      Jobs.enqueue_in(
+        1.hour,
+        :generate_inferred_concepts,
+        item_type: 'topics',
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+        match_only: true
+      )
+    end
+    
+    def process_popular_posts
+
+      # Find candidate posts that are popular and don't have concepts yet
+      candidates = DiscourseAi::InferredConcepts::Manager.find_candidate_posts(
+        limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+        min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+        exclude_first_posts: true,
+        created_after: SiteSetting.inferred_concepts_lookback_days.days.ago
+      )
+
+      return if candidates.blank?
+      
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: 'posts',
+        item_ids: candidates.map(&:id),
+        batch_size: 10
+      )
+      
+      # Schedule a follow-up job to match against existing concepts
+      Jobs.enqueue_in(
+        1.hour,
+        :generate_inferred_concepts,
+        item_type: 'posts',
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+        match_only: true
+      )
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -2,6 +2,7 @@
 
 class InferredConcept < ActiveRecord::Base
   has_and_belongs_to_many :topics
+  has_and_belongs_to_many :posts
 
   validates :name, presence: true, uniqueness: true
 end

app/jobs/scheduled/generate_concepts_from_popular_topics.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AiInferredConceptPostSerializer < ApplicationSerializer
+  attributes :id,
+             :post_number,
+             :topic_id,
+             :topic_title,
+             :username,
+             :avatar_template,
+             :created_at,
+             :updated_at,
+             :excerpt,
+             :truncated,
+             :inferred_concepts
+             
+  def avatar_template
+    User.avatar_template(object.username, object.uploaded_avatar_id)
+  end
+  
+  def excerpt
+    Post.excerpt(object.cooked)
+  end
+  
+  def truncated
+    object.cooked.length > SiteSetting.post_excerpt_maxlength
+  end
+  
+  def inferred_concepts
+    ActiveModel::ArraySerializer.new(
+      object.inferred_concepts,
+      each_serializer: InferredConceptSerializer
+    )
+  end
+end

app/models/inferred_concept.rb

@@ -323,6 +323,12 @@ en:
         short_summarizer:
           name: "Summarizer (short form)"
           description: "Default persona used to power AI short summaries for topic lists' items"
+        concept_finder:
+          name: "Concept Finder"
+          description: "AI Bot specialized in identifying concepts and themes in content"
+        concept_matcher:
+          name: "Concept Matcher"
+          description: "AI Bot specialized in matching content against existing concepts"
       topic_not_found: "Summary unavailable, topic not found!"
       summarizing: "Summarizing topic"
       searching: "Searching for: '%{query}'"

app/serializers/ai_inferred_concept_post_serializer.rb

@@ -415,4 +415,12 @@ discourse_ai:
     default: 30
     client: false
     description: "Only consider topics created within this many days for concept generation"
+  inferred_concepts_daily_posts_limit:
+    default: 30
+    client: false
+    description: "Maximum number of posts to process each day for concept generation"
+  inferred_concepts_post_min_likes:
+    default: 5
+    client: false
+    description: "Minimum number of likes a post must have to be considered for concept generation"
 

config/locales/server.en.yml

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreatePostsInferredConcepts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :posts_inferred_concepts do |t|
+      t.integer :post_id, null: false
+      t.integer :inferred_concept_id, null: false
+      t.timestamps
+    end
+
+    add_index :posts_inferred_concepts, [:post_id, :inferred_concept_id], unique: true, name: 'idx_unique_post_inferred_concept'
+    add_index :posts_inferred_concepts, :post_id
+    add_index :posts_inferred_concepts, :inferred_concept_id
+  end
+end