working on specs

Author: SamSaffron

app/models/ai_spam_log.rb

@@ -1,18 +1,20 @@
 # frozen_string_literal: true
 class AiSpamLog < ActiveRecord::Base
+  belongs_to :post
+  belongs_to :llm_model
+  belongs_to :ai_api_audit_log
 end
 
 # == Schema Information
 #
 # Table name: ai_spam_logs
 #
-#  id                       :bigint           not null, primary key
-#  post_id                  :bigint           not null
-#  llm_model_id             :bigint           not null
-#  last_ai_api_audit_log_id :bigint           not null
-#  scan_count               :integer          default(1), not null
-#  is_spam                  :boolean          not null
-#  last_scan_payload        :text             default(""), not null
-#  created_at               :datetime         not null
-#  updated_at               :datetime         not null
+#  id                  :bigint           not null, primary key
+#  post_id             :bigint           not null
+#  llm_model_id        :bigint           not null
+#  ai_api_audit_log_id :bigint           not null
+#  is_spam             :boolean          not null
+#  payload             :text             default(""), not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
 #

config/locales/server.en.yml

@@ -251,6 +251,8 @@ en:
         other_content_in_pm: "Personal messages containing posts from other people cannot be shared publicly"
         failed_to_share: "Failed to share the conversation"
         conversation_deleted: "Conversation share deleted successfully"
+    spam_detection:
+      flag_reason: "Flagged as spam by Discourse AI"
     ai_bot:
       default_pm_prefix: "[Untitled AI bot PM]"
       personas:

db/migrate/20241206051223_add_ai_spam_logs.rb renamed to db/migrate/20241206051224_add_ai_spam_logs.rb

@@ -4,11 +4,12 @@ def change
     create_table :ai_spam_logs do |t|
       t.bigint :post_id, null: false
       t.bigint :llm_model_id, null: false
-      t.bigint :last_ai_api_audit_log_id, null: false
-      t.integer :scan_count, null: false, default: 1
+      t.bigint :ai_api_audit_log_id
       t.boolean :is_spam, null: false
-      t.text :last_scan_payload, null: false, default: "", limit: 20_000
+      t.text :payload, null: false, default: "", limit: 20_000
       t.timestamps
     end
+
+    add_index :ai_spam_logs, :post_id
   end
 end

lib/ai_moderation/spam_scanner.rb

@@ -44,13 +44,20 @@ def self.enabled?
       def self.should_scan_post?(post)
         return false if !post.present?
         return false if post.user.trust_level > TrustLevel[1]
-        return false if post.user.post_count > POSTS_TO_SCAN
         return false if post.topic.private_message?
+        if Post
+             .where(user_id: post.user_id)
+             .joins(:topic)
+             .where(topic: { archetype: Archetype.default })
+             .limit(4)
+             .count > 3
+          return false
+        end
         true
       end
 
       def self.scanned_max_times?(post)
-        AiSpamLog.where(post_id: post.id).sum(:scan_count) >= 3
+        AiSpamLog.where(post_id: post.id).count >= 3
       end
 
       def self.significant_change?(previous_version, current_version)
@@ -92,13 +99,26 @@ def self.perform_scan(post)
               },
             )&.strip
 
-          is_spam = result.present? && result.downcase.include?("spam")
+          is_spam = (result.present? && result.downcase.include?("spam"))
+
+          log = AiApiAuditLog.order(id: :desc).where(feature_name: "spam_detection").first
 
-          create_log_entry(post, settings.llm_model, result, is_spam)
+          AiSpamLog.transaction do
+            AiSpamLog.create!(
+              post: post,
+              llm_model: settings.llm_model,
+              ai_api_audit_log: log,
+              is_spam: is_spam,
+              payload: context,
+            )
+            handle_spam(post, result) if is_spam
+          end
 
-          handle_spam(post, result) if is_spam
         rescue StandardError => e
-          Rails.logger.error("Error in SpamScanner for post #{post.id}: #{e.message}")
+          if Rails.env.test?
+            raise e
+          end
+          Discourse.warn_exception(e, message: "Error in SpamScanner for post #{post.id}")
         end
       end
 
@@ -134,12 +154,6 @@ def self.build_context(post)
 
         context << "\nPost Content:"
         context << post.raw
-
-        if post.linked_urls.present?
-          context << "\nLinks in post:"
-          context << post.linked_urls.join(", ")
-        end
-
         context.join("\n")
       end
 
@@ -179,23 +193,6 @@ def self.build_system_prompt(custom_instructions)
         base_prompt
       end
 
-      def self.create_log_entry(post, llm_model, result, is_spam)
-        log = AiSpamLog.find_or_initialize_by(post_id: post.id)
-
-        if log.new_record?
-          log.llm_model = llm_model
-          log.is_spam = is_spam
-          log.scan_count = 1
-        else
-          log.scan_count += 1
-        end
-
-        last_audit = DiscourseAi::ApiAuditLog.last
-        log.last_ai_api_audit_log_id = last_audit.id if last_audit
-
-        log.save!
-      end
-
       def self.handle_spam(post, result)
         SpamRule::AutoSilence.new(post.user, post).silence_user
 

spec/lib/modules/ai_moderation/spam_scanner_spec.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe DiscourseAi::AiModeration::SpamScanner do
+  fab!(:user) { Fabricate(:user, trust_level: TrustLevel[0]) }
+  fab!(:topic) { Fabricate(:topic) }
+  fab!(:post) { Fabricate(:post, user: user, topic: topic) }
+  fab!(:llm_model) { Fabricate(:llm_model) }
+  fab!(:spam_setting) do
+    AiModerationSetting.create!(
+      setting_type: :spam,
+      llm_model: llm_model,
+      data: { custom_instructions: "test instructions" }
+    )
+  end
+
+  before do
+    SiteSetting.discourse_ai_enabled = true
+    SiteSetting.ai_spam_detection_enabled = true
+  end
+
+  describe ".enabled?" do
+    it "returns true when both settings are enabled" do
+      expect(described_class.enabled?).to eq(true)
+    end
+
+    it "returns false when discourse_ai is disabled" do
+      SiteSetting.discourse_ai_enabled = false
+      expect(described_class.enabled?).to eq(false)
+    end
+
+    it "returns false when spam detection is disabled" do
+      SiteSetting.ai_spam_detection_enabled = false
+      expect(described_class.enabled?).to eq(false)
+    end
+  end
+
+  describe ".should_scan_post?" do
+    it "returns true for new users' posts" do
+      expect(described_class.should_scan_post?(post)).to eq(true)
+    end
+
+    it "returns false for trusted users" do
+      post.user.trust_level = TrustLevel[2]
+      expect(described_class.should_scan_post?(post)).to eq(false)
+    end
+
+    it "returns false for users with many public posts" do
+      Fabricate(:post, user: user, topic: topic)
+      Fabricate(:post, user: user, topic: topic)
+      expect(described_class.should_scan_post?(post)).to eq(true)
+
+      pm = Fabricate(:private_message_topic, user: user)
+      Fabricate(:post, user: user, topic: pm)
+
+      expect(described_class.should_scan_post?(post)).to eq(true)
+
+      topic = Fabricate(:topic, user: user)
+      Fabricate(:post, user: user, topic: topic)
+
+      expect(described_class.should_scan_post?(post)).to eq(false)
+    end
+
+    it "returns false for private messages" do
+      pm_topic = Fabricate(:private_message_topic)
+      pm_post = Fabricate(:post, topic: pm_topic, user: user)
+      expect(described_class.should_scan_post?(pm_post)).to eq(false)
+    end
+
+    it "returns false for nil posts" do
+      expect(described_class.should_scan_post?(nil)).to eq(false)
+    end
+  end
+
+  describe ".scanned_max_times?" do
+    it "returns true when post has been scanned 3 times" do
+      3.times do
+        AiSpamLog.create!(
+          post: post,
+          llm_model: llm_model,
+          ai_api_audit_log_id: 1,
+          is_spam: false
+        )
+      end
+
+      expect(described_class.scanned_max_times?(post)).to eq(true)
+    end
+
+    it "returns false for posts scanned less than 3 times" do
+      expect(described_class.scanned_max_times?(post)).to eq(false)
+    end
+  end
+
+  describe ".significant_change?" do
+    it "returns true for first edits" do
+      expect(described_class.significant_change?(nil, "new content")).to eq(true)
+    end
+
+    it "returns true for significant changes" do
+      old_version = "This is a test post"
+      new_version = "This is a completely different post with new content"
+      expect(described_class.significant_change?(old_version, new_version)).to eq(true)
+    end
+
+    it "returns false for minor changes" do
+      old_version = "This is a test post"
+      new_version = "This is a test Post" # Only capitalization change
+      expect(described_class.significant_change?(old_version, new_version)).to eq(false)
+    end
+  end
+
+  describe ".new_post" do
+    it "enqueues spam scan job for eligible posts" do
+      Jobs.expects(:enqueue).with(:ai_spam_scan, post_id: post.id)
+      described_class.new_post(post)
+    end
+
+    it "doesn't enqueue jobs when disabled" do
+      SiteSetting.ai_spam_detection_enabled = false
+      Jobs.expects(:enqueue).never
+      described_class.new_post(post)
+    end
+  end
+
+  describe ".edited_post" do
+    it "enqueues spam scan job for eligible edited posts" do
+      PostRevision.create!(
+        post: post,
+        modifications: { raw: ["old content", "completely new content"] }
+      )
+
+      Jobs.expects(:enqueue).with(:ai_spam_scan, post_id: post.id)
+      described_class.edited_post(post)
+    end
+
+    it "schedules delayed job when edited too soon after last scan" do
+      AiSpamLog.create!(
+        post: post,
+        llm_model: llm_model,
+        ai_api_audit_log_id: 1,
+        is_spam: false,
+        created_at: 5.minutes.ago
+      )
+
+      Jobs.expects(:enqueue_in)
+      described_class.edited_post(post)
+    end
+  end
+
+  describe "integration test" do
+    fab!(:llm_model) { Fabricate(:llm_model) }
+    let(:api_audit_log) { Fabricate(:api_audit_log) }
+
+    before do
+      Jobs.run_immediately!
+    end
+
+    it "Correctly handles spam scanning" do
+      # we need a proper audit log so
+      prompt = nil
+      DiscourseAi::Completions::Llm.with_prepared_responses(["spam"]) do |_,_,_prompts|
+        described_class.new_post(post)
+        prompt = _prompts.first
+      end
+
+      content = prompt.messages[1][:content]
+      expect(content).to include(post.topic.title)
+      expect(content).to include(post.raw)
+
+      log = AiSpamLog.find_by(post: post)
+
+      expect(log.payload).to eq(content)
+      expect(log.is_spam).to eq(true)
+      expect(post.user.reload.silenced_till).to be_present
+
+      # hmm maybe it should be?
+      #expect(post.topic.visible).to eq(false)
+    end
+  end
+end