FEATURE: Backfill posts sentiment.

It adds a scheduled job to backfill posts' sentiment, similar to our existing rake task, but with two settings to control the batch size and posts' max-age.

Author: romanrizzi

app/jobs/scheduled/sentiment_backfill.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Jobs
+  class SentimentBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(_args)
+      return unless SiteSetting.ai_sentiment_enabled
+
+      base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
+      return if base_budget.zero?
+      # Split budget in 12 intervals, but make sure is at least one.
+      #
+      # This is not exact as we don't have a way of tracking how many
+      # posts we classified in the current hour, but it's a good enough approximation.
+      limit_per_job = [base_budget, 12].max / 12
+
+      classificator = DiscourseAi::Sentiment::PostClassification.new
+
+      posts =
+        DiscourseAi::Sentiment::PostClassification.backfill_query(
+          max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
+        ).limit(limit_per_job)
+
+      classificator.bulk_classify!(posts)
+    end
+  end
+end

config/settings.yml

@@ -66,6 +66,15 @@ discourse_ai:
   ai_sentiment_model_configs:
     default: ""
     json_schema: DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema
+  ai_sentiment_backfill_maximum_posts_per_hour:
+    default: 0
+    min: 0
+    max: 10000
+    hidden: true
+  ai_sentiment_backfill_post_max_age_days:
+    default: 60
+    hidden: true
+  
 
   ai_nsfw_detection_enabled:
     default: false

lib/sentiment/post_classification.rb

@@ -3,6 +3,33 @@
 module DiscourseAi
   module Sentiment
     class PostClassification
+      def self.backfill_query(from_post_id: nil, max_age_days: nil)
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+
+        base_query =
+          Post
+            .joins("INNER JOIN topics ON topics.id = posts.topic_id")
+            .joins(
+              "LEFT JOIN classification_results ON classification_results.target_id = posts.id AND classification_results.target_type = 'Post'",
+            )
+            .where("classification_results.target_id IS NULL")
+            .where("category_id IN (?)", public_categories)
+            .where(post_type: Post.types[:regular])
+            .where(posts: { deleted_at: nil })
+            .where(topics: { deleted_at: nil })
+
+        base_query = base_query.where("posts.id >= ?", from_post_id.to_i) if from_post_id.present?
+
+        if max_age_days.present?
+          base_query =
+            base_query.where(
+              "posts.created_at > current_date - INTERVAL '#{max_age_days.to_i} DAY'",
+            )
+        end
+
+        base_query
+      end
+
       def bulk_classify!(relation)
         http_pool_size = 100
         pool =
@@ -13,6 +40,7 @@ def bulk_classify!(relation)
           )
 
         available_classifiers = classifiers
+        return if available_classifiers.blank?
         base_url = Discourse.base_url
 
         promised_classifications =
@@ -52,6 +80,7 @@ def bulk_classify!(relation)
 
       def classify!(target)
         return if target.blank?
+        return if classifiers.blank?
 
         to_classify = prepare_text(target)
         return if to_classify.blank?
@@ -79,7 +108,7 @@ def prepare_text(target)
       end
 
       def classifiers
-        DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values
+        @classifiers ||= DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values
       end
 
       def request_with(content, config, base_url = Discourse.base_url)

lib/sentiment/sentiment_site_setting_json_schema.rb

@@ -27,6 +27,7 @@ def self.schema
       end
 
       def self.values
+        return {} if SiteSetting.ai_sentiment_model_configs.blank?
         JSON.parse(SiteSetting.ai_sentiment_model_configs, object_class: OpenStruct)
       end
     end

lib/tasks/modules/sentiment/backfill.rake

@@ -2,18 +2,8 @@
 
 desc "Backfill sentiment for all posts"
 task "ai:sentiment:backfill", [:start_post] => [:environment] do |_, args|
-  public_categories = Category.where(read_restricted: false).pluck(:id)
-
-  Post
-    .joins("INNER JOIN topics ON topics.id = posts.topic_id")
-    .joins(
-      "LEFT JOIN classification_results ON classification_results.target_id = posts.id AND classification_results.target_type = 'Post'",
-    )
-    .where("classification_results.target_id IS NULL")
-    .where("posts.id >= ?", args[:start_post].to_i || 0)
-    .where("category_id IN (?)", public_categories)
-    .where(posts: { deleted_at: nil })
-    .where(topics: { deleted_at: nil })
+  DiscourseAi::Sentiment::PostClassification
+    .backfill_query(from_post_id: args[:start_post].to_i)
     .find_in_batches do |batch|
       print "."
       DiscourseAi::Sentiment::PostClassification.new.bulk_classify!(batch)

spec/jobs/scheduled/sentiment_backfill_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "../../support/sentiment_inference_stubs"
+
+RSpec.describe Jobs::SentimentBackfill do
+  describe "#execute" do
+    fab!(:post)
+
+    before do
+      SiteSetting.ai_sentiment_enabled = true
+      SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour = 100
+      SiteSetting.ai_sentiment_model_configs =
+        "[{\"model_name\":\"SamLowe/roberta-base-go_emotions\",\"endpoint\":\"http://samlowe-emotion.com\",\"api_key\":\"123\"},{\"model_name\":\"j-hartmann/emotion-english-distilroberta-base\",\"endpoint\":\"http://jhartmann-emotion.com\",\"api_key\":\"123\"},{\"model_name\":\"cardiffnlp/twitter-roberta-base-sentiment-latest\",\"endpoint\":\"http://cardiffnlp-sentiment.com\",\"api_key\":\"123\"}]"
+    end
+
+    let(:expected_analysis) { DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.length }
+
+    it "backfills when settings are correct" do
+      SentimentInferenceStubs.stub_classification(post)
+      subject.execute({})
+
+      expect(ClassificationResult.where(target: post).count).to eq(expected_analysis)
+    end
+
+    it "does nothing when batch size is zero" do
+      SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour = 0
+
+      subject.execute({})
+
+      expect(ClassificationResult.count).to be_zero
+    end
+
+    it "does nothing when sentiment is disabled" do
+      SiteSetting.ai_sentiment_enabled = false
+
+      subject.execute({})
+
+      expect(ClassificationResult.count).to be_zero
+    end
+
+    it "respects the ai_sentiment_backfill_post_max_age_days setting" do
+      SentimentInferenceStubs.stub_classification(post)
+      SiteSetting.ai_sentiment_backfill_post_max_age_days = 80
+      post_2 = Fabricate(:post, created_at: 81.days.ago)
+
+      subject.execute({})
+
+      expect(ClassificationResult.where(target: post).count).to eq(expected_analysis)
+      expect(ClassificationResult.where(target: post_2).count).to be_zero
+    end
+  end
+end

spec/lib/modules/sentiment/post_classification_spec.rb

@@ -3,15 +3,15 @@
 require_relative "../../../support/sentiment_inference_stubs"
 
 RSpec.describe DiscourseAi::Sentiment::PostClassification do
-  fab!(:post_1) { Fabricate(:post, post_number: 2) }
-
   before do
     SiteSetting.ai_sentiment_enabled = true
     SiteSetting.ai_sentiment_model_configs =
       "[{\"model_name\":\"SamLowe/roberta-base-go_emotions\",\"endpoint\":\"http://samlowe-emotion.com\",\"api_key\":\"123\"},{\"model_name\":\"j-hartmann/emotion-english-distilroberta-base\",\"endpoint\":\"http://jhartmann-emotion.com\",\"api_key\":\"123\"},{\"model_name\":\"cardiffnlp/twitter-roberta-base-sentiment-latest\",\"endpoint\":\"http://cardiffnlp-sentiment.com\",\"api_key\":\"123\"}]"
   end
 
   describe "#classify!" do
+    fab!(:post_1) { Fabricate(:post, post_number: 2) }
+
     it "does nothing if the post content is blank" do
       post_1.update_columns(raw: "")
 
@@ -28,9 +28,18 @@
 
       expect(ClassificationResult.where(target: post_1).count).to eq(expected_analysis)
     end
+
+    it "does nothing if there are no classification model" do
+      SiteSetting.ai_sentiment_model_configs = ""
+
+      subject.classify!(post_1)
+
+      expect(ClassificationResult.where(target: post_1).count).to be_zero
+    end
   end
 
   describe "#classify_bulk!" do
+    fab!(:post_1) { Fabricate(:post, post_number: 2) }
     fab!(:post_2) { Fabricate(:post, post_number: 2) }
 
     it "classifies all given posts" do
@@ -43,5 +52,91 @@
       expect(ClassificationResult.where(target: post_1).count).to eq(expected_analysis)
       expect(ClassificationResult.where(target: post_2).count).to eq(expected_analysis)
     end
+
+    it "does nothing if there are no classification model" do
+      SiteSetting.ai_sentiment_model_configs = ""
+
+      subject.bulk_classify!(Post.where(id: [post_1.id, post_2.id]))
+
+      expect(ClassificationResult.where(target: post_1).count).to be_zero
+      expect(ClassificationResult.where(target: post_2).count).to be_zero
+    end
+  end
+
+  describe ".backfill_query" do
+    it "excludes posts in personal messages" do
+      Fabricate(:private_message_post)
+
+      posts = described_class.backfill_query
+
+      expect(posts).to be_empty
+    end
+
+    it "excludes posts in restricted categories" do
+      sec_cat = Fabricate(:category, read_restricted: true)
+      topic = Fabricate(:topic, category: sec_cat)
+      Fabricate(:post, topic: topic)
+
+      posts = described_class.backfill_query
+
+      expect(posts).to be_empty
+    end
+
+    it "includes regular posts only" do
+      Fabricate(:small_action)
+
+      posts = described_class.backfill_query
+
+      expect(posts).to be_empty
+    end
+
+    it "excludes posts from deleted topics" do
+      topic = Fabricate(:topic, deleted_at: 1.hour.ago)
+      Fabricate(:post, topic: topic)
+
+      posts = described_class.backfill_query
+
+      expect(posts).to be_empty
+    end
+
+    it "excludes deleted posts" do
+      Fabricate(:post, deleted_at: 1.hour.ago)
+
+      posts = described_class.backfill_query
+
+      expect(posts).to be_empty
+    end
+
+    context "with max_age_days" do
+      fab!(:age_post) { Fabricate(:post, created_at: 3.days.ago) }
+
+      it "includes a post when is younger" do
+        posts = described_class.backfill_query(max_age_days: 4)
+
+        expect(posts).to contain_exactly(age_post)
+      end
+
+      it "excludes posts when it's older" do
+        posts = described_class.backfill_query(max_age_days: 2)
+
+        expect(posts).to be_empty
+      end
+    end
+
+    context "with from_post_id" do
+      fab!(:post)
+
+      it "includes post if ID is higher" do
+        posts = described_class.backfill_query(from_post_id: post.id - 1)
+
+        expect(posts).to contain_exactly(post)
+      end
+
+      it "excludes post if ID is lower" do
+        posts = described_class.backfill_query(from_post_id: post.id + 1)
+
+        expect(posts).to be_empty
+      end
+    end
   end
 end