Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.

Commit 2263550

Browse files
FIX: Check post action creator result when flagging spam
Currently in core re-flagging something that is already flagged as spam is not supported, long term we may want to support this but in the meantime we should not be silencing/hiding if the PostActionCreator fails when flagging things as spam.
1 parent b60926c commit 2263550

File tree

4 files changed

+80
-25
lines changed

4 files changed

+80
-25
lines changed

lib/ai_moderation/spam_scanner.rb

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -394,22 +394,27 @@ def self.handle_spam(post, log)
394394
queue_for_review: true,
395395
).perform
396396

397-
log.update!(reviewable: result.reviewable)
398-
399-
reason = I18n.t("discourse_ai.spam_detection.silence_reason", url: url)
400-
silencer =
401-
UserSilencer.new(
402-
post.user,
403-
flagging_user,
404-
message: :too_many_spam_flags,
405-
post_id: post.id,
406-
reason: reason,
407-
keep_posts: true,
408-
)
409-
silencer.silence
397+
# Currently in core re-flagging something that is already flagged as spam
398+
# is not supported, long term we may want to support this but in the meantime
399+
# we should not be silencing/hiding if the PostActionCreator fails.
400+
if result.success?
401+
log.update!(reviewable: result.reviewable)
402+
403+
reason = I18n.t("discourse_ai.spam_detection.silence_reason", url: url)
404+
silencer =
405+
UserSilencer.new(
406+
post.user,
407+
flagging_user,
408+
message: :too_many_spam_flags,
409+
post_id: post.id,
410+
reason: reason,
411+
keep_posts: true,
412+
)
413+
silencer.silence
410414

411-
# silencer will not hide tl1 posts, so we do this here
412-
hide_post(post)
415+
# silencer will not hide tl1 posts, so we do this here
416+
hide_post(post)
417+
end
413418
end
414419

415420
def self.hide_post(post)

lib/automation/llm_triage.rb

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,15 +88,18 @@ def self.handle(
8888
.sub("%%AUTOMATION_NAME%%", automation&.name.to_s)
8989

9090
if flag_type == :spam || flag_type == :spam_silence
91-
PostActionCreator.new(
92-
Discourse.system_user,
93-
post,
94-
PostActionType.types[:spam],
95-
message: score_reason,
96-
queue_for_review: true,
97-
).perform
91+
result =
92+
PostActionCreator.new(
93+
Discourse.system_user,
94+
post,
95+
PostActionType.types[:spam],
96+
message: score_reason,
97+
queue_for_review: true,
98+
).perform
9899

99-
SpamRule::AutoSilence.new(post.user, post).silence_user if flag_type == :spam_silence
100+
if flag_type == :spam_silence && result.success?
101+
SpamRule::AutoSilence.new(post.user, post).silence_user
102+
end
100103
else
101104
reviewable =
102105
ReviewablePost.needs_review!(target: post, created_by: Discourse.system_user)

spec/lib/modules/ai_moderation/spam_scanner_spec.rb

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@
214214

215215
before { Jobs.run_immediately! }
216216

217-
it "Can correctly run tests" do
217+
it "can correctly run tests" do
218218
prompts = nil
219219
result =
220220
DiscourseAi::Completions::Llm.with_prepared_responses(
@@ -240,7 +240,7 @@
240240
expect(result[:is_spam]).to eq(false)
241241
end
242242

243-
it "Correctly handles spam scanning" do
243+
it "correctly handles spam scanning" do
244244
expect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)
245245

246246
# flag post for scanning
@@ -288,6 +288,31 @@
288288
expect(post.topic.reload.visible).to eq(true)
289289
expect(post.user.reload.silenced?).to eq(false)
290290
end
291+
292+
it "does not silence the user or hide the post when a flag cannot be created" do
293+
post = post_with_uploaded_image
294+
Fabricate(
295+
:post_action,
296+
post: post,
297+
user: described_class.flagging_user,
298+
post_action_type_id: PostActionType.types[:spam],
299+
)
300+
301+
described_class.new_post(post)
302+
303+
prompt = nil
304+
DiscourseAi::Completions::Llm.with_prepared_responses(["spam"]) do |_, _, _prompts|
305+
# force a rebake so we actually scan
306+
post.rebake!
307+
prompt = _prompts.first
308+
end
309+
310+
log = AiSpamLog.find_by(post: post)
311+
312+
expect(log.reviewable).to be_nil
313+
expect(post.user.reload.silenced_till).to be_nil
314+
expect(post.topic.reload.visible).to eq(true)
315+
end
291316
end
292317

293318
it "includes location information and email in context" do

spec/lib/modules/automation/llm_triage_spec.rb

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,28 @@ def triage(**args)
128128
expect(post.user.silenced?).to eq(true)
129129
end
130130

131+
it "does not silence the user if the flag fails" do
132+
Fabricate(
133+
:post_action,
134+
post: post,
135+
user: Discourse.system_user,
136+
post_action_type_id: PostActionType.types[:spam],
137+
)
138+
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
139+
triage(
140+
post: post,
141+
model: "custom:#{llm_model.id}",
142+
system_prompt: "test %%POST%%",
143+
search_for_text: "bad",
144+
flag_post: true,
145+
flag_type: :spam_silence,
146+
automation: nil,
147+
)
148+
end
149+
150+
expect(post.user.silenced?).to eq(false)
151+
end
152+
131153
it "can handle garbled output from LLM" do
132154
DiscourseAi::Completions::Llm.with_prepared_responses(["Bad.\n\nYo"]) do
133155
triage(

0 commit comments

Comments
 (0)