scan images as well during spam scan

(if llm supports it)

Author: SamSaffron

lib/ai_moderation/entry_point.rb

@@ -5,8 +5,8 @@ module AiModeration
     class EntryPoint
       def inject_into(plugin)
         plugin.on(:post_created) { |post| SpamScanner.new_post(post) }
-
         plugin.on(:post_edited) { |post| SpamScanner.edited_post(post) }
+        plugin.on(:post_process_cooked) { |_doc, post| SpamScanner.after_cooked_post(post) }
       end
     end
   end

lib/ai_moderation/spam_scanner.rb

@@ -6,12 +6,30 @@ class SpamScanner
       POSTS_TO_SCAN = 3
       MINIMUM_EDIT_DIFFERENCE = 10
       EDIT_DELAY_MINUTES = 10
+      MAX_AGE_TO_SCAN = 1.day
+
+      SHOULD_SCAN_POST_CUSTOM_FIELD = "discourse_ai_should_scan_post"
 
       def self.new_post(post)
         return if !enabled?
         return if !should_scan_post?(post)
 
-        Jobs.enqueue(:ai_spam_scan, post_id: post.id)
+        flag_post_for_scanning(post)
+      end
+
+      def self.after_cooked_post(post)
+        return if !post.custom_fields[SHOULD_SCAN_POST_CUSTOM_FIELD]
+        return if post.updated_at < MAX_AGE_TO_SCAN.ago
+
+        last_scan = AiSpamLog.where(post_id: post.id).order(created_at: :desc).first
+
+        if last_scan && last_scan.created_at > EDIT_DELAY_MINUTES.minutes.ago
+          delay_minutes =
+            ((last_scan.created_at + EDIT_DELAY_MINUTES.minutes) - Time.current).to_i / 60
+          Jobs.enqueue_in(delay_minutes.minutes, :ai_spam_scan, post_id: post.id)
+        else
+          Jobs.enqueue(:ai_spam_scan, post_id: post.id)
+        end
       end
 
       def self.edited_post(post)
@@ -22,19 +40,14 @@ def self.edited_post(post)
         previous_version = post.revisions.last&.modifications&.dig("raw", 0)
         current_version = post.raw
 
-        # Skip if we can't determine the difference or if change is too small
         return if !significant_change?(previous_version, current_version)
 
-        last_scan = AiSpamLog.where(post_id: post.id).order(created_at: :desc).first
+        flag_post_for_scanning(post)
+      end
 
-        if last_scan && last_scan.created_at > EDIT_DELAY_MINUTES.minutes.ago
-          # Schedule delayed job if too soon after last scan
-          delay_minutes =
-            ((last_scan.created_at + EDIT_DELAY_MINUTES.minutes) - Time.current).to_i / 60
-          Jobs.enqueue_in(delay_minutes.minutes, :ai_spam_scan, post_id: post.id)
-        else
-          Jobs.enqueue(:ai_spam_scan, post_id: post.id)
-        end
+      def self.flag_post_for_scanning(post)
+        post.custom_fields[SHOULD_SCAN_POST_CUSTOM_FIELD] = "true"
+        post.save_custom_fields
       end
 
       def self.enabled?
@@ -84,7 +97,14 @@ def self.perform_scan(post)
         prompt = DiscourseAi::Completions::Prompt.new(system_prompt)
 
         context = build_context(post)
-        prompt.push(type: :user, content: context)
+
+        args = {type: :user, content: context}
+        upload_ids = post.upload_ids
+        if upload_ids.present?
+          args[:upload_ids] = upload_ids.take(3)
+        end
+
+        prompt.push(**args)
 
         begin
           result =
@@ -156,6 +176,7 @@ def self.build_context(post)
         context.join("\n")
       end
 
+
       def self.build_system_prompt(custom_instructions)
         base_prompt = +<<~PROMPT
           You are a spam detection system. Analyze the following post content and context.

spec/lib/modules/ai_moderation/spam_scanner_spec.rb

@@ -109,7 +109,10 @@
 
   describe ".new_post" do
     it "enqueues spam scan job for eligible posts" do
-      expect { described_class.new_post(post) }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
+      expect {
+        described_class.new_post(post)
+        described_class.after_cooked_post(post)
+      }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
     end
 
     it "doesn't enqueue jobs when disabled" do
@@ -127,7 +130,10 @@
         },
       )
 
-      expect { described_class.edited_post(post) }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
+      expect {
+        described_class.edited_post(post)
+        described_class.after_cooked_post(post)
+      }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
     end
 
     it "schedules delayed job when edited too soon after last scan" do
@@ -139,28 +145,41 @@
         created_at: 5.minutes.ago,
       )
 
-      expect { described_class.edited_post(post) }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
+      expect {
+        described_class.edited_post(post)
+        described_class.after_cooked_post(post)
+      }.to change(Jobs::AiSpamScan.jobs, :size).by(1)
     end
   end
 
   describe "integration test" do
     fab!(:llm_model)
     let(:api_audit_log) { Fabricate(:api_audit_log) }
+    fab!(:post_with_uploaded_image)
 
     before { Jobs.run_immediately! }
 
     it "Correctly handles spam scanning" do
-      # we need a proper audit log so
+      # flag post for scanning
+      post = post_with_uploaded_image
+
+      described_class.new_post(post)
+
       prompt = nil
       DiscourseAi::Completions::Llm.with_prepared_responses(["spam"]) do |_, _, _prompts|
-        described_class.new_post(post)
+        # force a rebake so we actually scan
+        post.rebake!
         prompt = _prompts.first
       end
 
       content = prompt.messages[1][:content]
       expect(content).to include(post.topic.title)
       expect(content).to include(post.raw)
 
+      upload_ids = prompt.messages[1][:upload_ids]
+      expect(upload_ids).to be_present
+      expect(upload_ids).to eq(post.upload_ids)
+
       log = AiSpamLog.find_by(post: post)
 
       expect(log.payload).to eq(content)