Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions lib/ai_moderation/spam_scanner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,20 @@ def self.flagging_user
user = nil
if SiteSetting.ai_spam_detection_user_id.present?
user = User.find_by(id: SiteSetting.ai_spam_detection_user_id)
ensure_safe_flagging_user!(user)
end
user || Discourse.system_user
end

def self.ensure_safe_flagging_user!(user)
# only do repair on bot users, if somehow it is set to a human skip repairs
return if !user.bot?
user.update!(silenced_till: nil) if user.silenced?
user.update!(trust_level: TrustLevel[4]) if user.trust_level != TrustLevel[4]
user.update!(suspended_till: nil, suspended_at: nil) if user.suspended?
user.update!(active: true) if !user.active?
Copy link
Contributor

@lis2 lis2 Jun 17, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of having this code, what do you think of a migration to unsilence, activate and unsuspend all bots?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I worry that then users can still do this to themselves on the admin page for the bot without understanding the impact 😢 maybe we should ban particular actions on bots from the admin interface?

end

def self.after_cooked_post(post)
return if !enabled?
return if !should_scan_post?(post)
Expand Down Expand Up @@ -94,6 +104,9 @@ def self.should_scan_post?(post)
return false if !post.present?
return false if post.user.trust_level > TrustLevel[1]
return false if post.topic.private_message?
return false if post.user.bot?
return false if post.user.staff?

if Post
.where(user_id: post.user_id)
.joins(:topic)
Expand Down
37 changes: 36 additions & 1 deletion spec/lib/modules/ai_moderation/spam_scanner_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,16 @@
expect(described_class.should_scan_post?(post)).to eq(false)
end

it "returns false for bots" do
post.user.id = -100
expect(described_class.should_scan_post?(post)).to eq(false)
end

it "returns false for staff" do
post.user.moderator = true
expect(described_class.should_scan_post?(post)).to eq(false)
end

it "returns false for users with many public posts" do
Fabricate(:post, user: user, topic: topic)
Fabricate(:post, user: user, topic: topic)
Expand Down Expand Up @@ -207,6 +217,26 @@
end
end

it "unsilences flagging user if erronuously silenced" do
described_class.flagging_user.update!(silenced_till: 1.day.from_now)
expect(described_class.flagging_user.silenced?).to eq(false)
end

it "ensures flagging user is tl4" do
described_class.flagging_user.update!(trust_level: 0)
expect(described_class.flagging_user.trust_level).to eq(4)
end

it "unsuspends user if it was erronuously suspended" do
described_class.flagging_user.update!(suspended_till: 1.day.from_now, suspended_at: 1.day.ago)
expect(described_class.flagging_user.suspended?).to eq(false)
end

it "makes sure account is active" do
described_class.flagging_user.update!(active: false)
expect(described_class.flagging_user.active).to eq(true)
end

describe "integration test" do
fab!(:llm_model)
let(:api_audit_log) { Fabricate(:api_audit_log) }
Expand Down Expand Up @@ -243,8 +273,13 @@
it "correctly handles spam scanning" do
expect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)

# flag post for scanning
post = post_with_uploaded_image
# this is surprising, core fabricator is not linking
# we need it linked so we scan uploads
post.link_post_uploads

expect(described_class.should_scan_post?(post)).to eq(true)
expect(post.upload_ids).to be_present

described_class.new_post(post)

Expand Down
Loading