Dedicated user for spam scanning

Author: SamSaffron

config/settings.yml

@@ -326,6 +326,10 @@ discourse_ai:
     default: false
     hidden: true
 
+  ai_spam_detection_user_id:
+    default: ""
+    hidden: true
+
   ai_spam_detection_model_allowed_seeded_models:
     default: ""
     hidden: true

lib/ai_moderation/entry_point.rb

@@ -7,6 +7,12 @@ def inject_into(plugin)
         plugin.on(:post_created) { |post| SpamScanner.new_post(post) }
         plugin.on(:post_edited) { |post| SpamScanner.edited_post(post) }
         plugin.on(:post_process_cooked) { |_doc, post| SpamScanner.after_cooked_post(post) }
+
+        plugin.on(:site_setting_changed) do |name, _old_value, new_value|
+          if name == :ai_spam_detection_enabled && new_value
+            SpamScanner.ensure_flagging_user!
+          end
+        end
       end
     end
   end

lib/ai_moderation/spam_scanner.rb

@@ -7,6 +7,7 @@ class SpamScanner
       MINIMUM_EDIT_DIFFERENCE = 10
       EDIT_DELAY_MINUTES = 10
       MAX_AGE_TO_SCAN = 1.day
+      MAX_RAW_SCAN_LENGTH = 5000
 
       SHOULD_SCAN_POST_CUSTOM_FIELD = "discourse_ai_should_scan_post"
 
@@ -17,6 +18,39 @@ def self.new_post(post)
         flag_post_for_scanning(post)
       end
 
+      def self.ensure_flagging_user!
+        if !SiteSetting.ai_spam_detection_user_id.present?
+          User.transaction do
+            # prefer a "high" id for this bot
+            id = User.where("id > -20").minimum(:id) - 1
+            id = User.minimum(:id) - 1 if id == -100
+
+            user =
+              User.create!(
+                id: id,
+                username: UserNameSuggester.suggest("discourse_ai_spam"),
+                name: "Discourse AI Spam Scanner",
+                email: "#{SecureRandom.hex(10)}@invalid.invalid",
+                active: true,
+                approved: true,
+                trust_level: TrustLevel[4],
+                admin: true,
+              )
+            Group.user_trust_level_change!(user.id, user.trust_level)
+
+            SiteSetting.ai_spam_detection_user_id = user.id
+          end
+        end
+      end
+
+      def self.flagging_user
+        user = nil
+        if SiteSetting.ai_spam_detection_user_id.present?
+          user = User.find_by(id: SiteSetting.ai_spam_detection_user_id)
+        end
+        user || Discourse.system_user
+      end
+
       def self.after_cooked_post(post)
         return if !post.custom_fields[SHOULD_SCAN_POST_CUSTOM_FIELD]
         return if post.updated_at < MAX_AGE_TO_SCAN.ago
@@ -98,11 +132,9 @@ def self.perform_scan(post)
 
         context = build_context(post)
 
-        args = {type: :user, content: context}
+        args = { type: :user, content: context }
         upload_ids = post.upload_ids
-        if upload_ids.present?
-          args[:upload_ids] = upload_ids.take(3)
-        end
+        args[:upload_ids] = upload_ids.take(3) if upload_ids.present?
 
         prompt.push(**args)
 
@@ -171,12 +203,11 @@ def self.build_context(post)
         context << "- Total posts: #{post.user.post_count}"
         context << "- Trust level: #{post.user.trust_level}"
 
-        context << "\nPost Content:"
-        context << post.raw
+        context << "\nPost Content (first #{MAX_RAW_SCAN_LENGTH} chars):\n"
+        context << post.raw[0..MAX_RAW_SCAN_LENGTH]
         context.join("\n")
       end
 
-
       def self.build_system_prompt(custom_instructions)
         base_prompt = +<<~PROMPT
           You are a spam detection system. Analyze the following post content and context.
@@ -228,7 +259,7 @@ def self.handle_spam(post, log)
 
         result =
           PostActionCreator.new(
-            Discourse.system_user,
+            flagging_user,
             post,
             PostActionType.types[:spam],
             reason: reason,
@@ -237,6 +268,30 @@ def self.handle_spam(post, log)
 
         log.update!(reviewable: result.reviewable)
         SpamRule::AutoSilence.new(post.user, post).silence_user
+        # this is required cause tl1 is not auto hidden
+        # we want to also handle tl1
+        hide_posts_and_topics(post.user)
+      end
+
+      def self.hide_posts_and_topics(user)
+        Post
+          .where(user_id: user.id)
+          .where("created_at > ?", 24.hours.ago)
+          .update_all(
+            [
+              "hidden = true, hidden_reason_id = COALESCE(hidden_reason_id, ?)",
+              Post.hidden_reasons[:new_user_spam_threshold_reached],
+            ],
+          )
+        topic_ids = Post
+          .where(user_id: user.id, post_number: 1)
+          .where("created_at > ?", 24.hours.ago)
+          .select(:topic_id)
+
+        Topic
+          .where(id: topic_ids)
+          .update_all(visible: false)
+
       end
     end
   end

spec/lib/modules/ai_moderation/spam_scanner_spec.rb

@@ -160,6 +160,9 @@
     before { Jobs.run_immediately! }
 
     it "Correctly handles spam scanning" do
+
+      expect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)
+
       # flag post for scanning
       post = post_with_uploaded_image
 
@@ -188,6 +191,7 @@
       expect(post.topic.reload.visible).to eq(false)
 
       expect(log.reviewable).to be_present
+      expect(log.reviewable.created_by_id).to eq(described_class.flagging_user.id)
     end
   end
 end