Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.

Commit 32df522

Browse files
committed
oops files in wrong spot
1 parent 0cb14a3 commit 32df522

File tree

2 files changed

+203
-203
lines changed

2 files changed

+203
-203
lines changed

lib/ai_moderation/entry_point.rb

Lines changed: 6 additions & 203 deletions
Original file line numberDiff line numberDiff line change
@@ -2,213 +2,16 @@
22

33
module DiscourseAi
44
module AiModeration
5-
class SpamScanner
6-
POSTS_TO_SCAN = 3
7-
MINIMUM_EDIT_DIFFERENCE = 10
8-
EDIT_DELAY_MINUTES = 10
9-
10-
def self.new_post(post)
11-
return if !enabled?
12-
return if !should_scan_post?(post)
13-
14-
Jobs.enqueue(:ai_spam_scan, post_id: post.id)
15-
end
16-
17-
def self.edited_post(post)
18-
return if !enabled?
19-
return if !should_scan_post?(post)
20-
return if scanned_max_times?(post)
21-
22-
previous_version = post.revisions.last&.modifications&.dig("raw", 0)
23-
current_version = post.raw
24-
25-
# Skip if we can't determine the difference or if change is too small
26-
return if !significant_change?(previous_version, current_version)
27-
28-
last_scan = AiSpamLog.where(post_id: post.id).order(created_at: :desc).first
29-
30-
if last_scan && last_scan.created_at > EDIT_DELAY_MINUTES.minutes.ago
31-
# Schedule delayed job if too soon after last scan
32-
delay_minutes =
33-
((last_scan.created_at + EDIT_DELAY_MINUTES.minutes) - Time.current).to_i / 60
34-
Jobs.enqueue_in(delay_minutes.minutes, :ai_spam_scan, post_id: post.id)
35-
else
36-
Jobs.enqueue(:ai_spam_scan, post_id: post.id)
5+
class EntryPoint
6+
def inject_into(plugin)
7+
plugin.on(:post_created) do |post|
8+
SpamScanner.new_post(post)
379
end
38-
end
39-
40-
def self.enabled?
41-
SiteSetting.ai_spam_detection_enabled && SiteSetting.discourse_ai_enabled
42-
end
43-
44-
def self.should_scan_post?(post)
45-
return false if !post.present?
46-
return false if post.user.trust_level > TrustLevel[1]
47-
return false if post.user.post_count > POSTS_TO_SCAN
48-
return false if post.topic.private_message?
49-
true
50-
end
51-
52-
def self.scanned_max_times?(post)
53-
AiSpamLog.where(post_id: post.id).sum(:scan_count) >= 3
54-
end
5510

56-
def self.significant_change?(previous_version, current_version)
57-
return true if previous_version.nil? # First edit should be scanned
58-
59-
# Use Discourse's built-in levenshtein implementation
60-
distance =
61-
ScreenedEmail.levenshtein(previous_version.to_s[0...1000], current_version.to_s[0...1000])
62-
63-
distance >= MINIMUM_EDIT_DIFFERENCE
64-
end
65-
66-
def self.perform_scan(post)
67-
return if !enabled?
68-
return if !should_scan_post?(post)
69-
70-
settings = AiModerationSetting.spam
71-
return if !settings || !settings.llm_model
72-
73-
llm = settings.llm_model.to_llm
74-
custom_instructions = settings.custom_instructions.presence
75-
76-
system_prompt = build_system_prompt(custom_instructions)
77-
prompt = DiscourseAi::Completions::Prompt.new(system_prompt)
78-
79-
context = build_context(post)
80-
prompt.push(type: :user, content: context)
81-
82-
begin
83-
result =
84-
llm.generate(
85-
prompt,
86-
temperature: 0.1,
87-
max_tokens: 100,
88-
user: Discourse.system_user,
89-
feature_name: "spam_detection",
90-
feature_context: {
91-
post_id: post.id,
92-
},
93-
)&.strip
94-
95-
is_spam = result.present? && result.downcase.include?("spam")
96-
97-
create_log_entry(post, settings.llm_model, result, is_spam)
98-
99-
handle_spam(post, result) if is_spam
100-
rescue StandardError => e
101-
Rails.logger.error("Error in SpamScanner for post #{post.id}: #{e.message}")
11+
plugin.on(:post_edited) do |post|
12+
SpamScanner.edited_post(post)
10213
end
10314
end
104-
105-
private
106-
107-
def self.build_context(post)
108-
context = []
109-
110-
# Clear distinction between reply and new topic
111-
if post.is_first_post?
112-
context << "NEW TOPIC POST ANALYSIS"
113-
context << "- Topic title: #{post.topic.title}"
114-
context << "- Category: #{post.topic.category&.name}"
115-
else
116-
context << "REPLY POST ANALYSIS"
117-
context << "- In topic: #{post.topic.title}"
118-
context << "- Topic started by: #{post.topic.user.username}"
119-
120-
# Include parent post context for replies
121-
if post.reply_to_post.present?
122-
parent = post.reply_to_post
123-
context << "\nReplying to #{parent.user.username}'s post:"
124-
context << "#{parent.raw[0..500]}..." if parent.raw.length > 500
125-
context << parent.raw if parent.raw.length <= 500
126-
end
127-
end
128-
129-
context << "\nPost Author Information:"
130-
context << "- Username: #{post.user.username}"
131-
context << "- Account age: #{(Time.current - post.user.created_at).to_i / 86_400} days"
132-
context << "- Total posts: #{post.user.post_count}"
133-
context << "- Trust level: #{post.user.trust_level}"
134-
135-
context << "\nPost Content:"
136-
context << post.raw
137-
138-
if post.linked_urls.present?
139-
context << "\nLinks in post:"
140-
context << post.linked_urls.join(", ")
141-
end
142-
143-
context.join("\n")
144-
end
145-
146-
def self.build_system_prompt(custom_instructions)
147-
base_prompt = <<~PROMPT
148-
You are a spam detection system. Analyze the following post content and context.
149-
Respond with "SPAM" if the post is spam, or "NOT_SPAM" if it's legitimate.
150-
151-
Consider the post type carefully:
152-
- For REPLY posts: Check if the response is relevant and topical to the thread
153-
- For NEW TOPIC posts: Check if it's a legitimate topic or spam promotion
154-
155-
A post is spam if it matches any of these criteria:
156-
- Contains unsolicited commercial content or promotions
157-
- Has suspicious or unrelated external links
158-
- Shows patterns of automated/bot posting
159-
- Contains irrelevant content or advertisements
160-
- For replies: Completely unrelated to the discussion thread
161-
- Uses excessive keywords or repetitive text patterns
162-
- Shows suspicious formatting or character usage
163-
164-
Be especially strict with:
165-
- Replies that ignore the previous conversation
166-
- Posts containing multiple unrelated external links
167-
- Generic responses that could be posted anywhere
168-
169-
Be fair to:
170-
- New users making legitimate first contributions
171-
- Non-native speakers making genuine efforts to participate
172-
- Topic-relevant product mentions in appropriate contexts
173-
PROMPT
174-
175-
if custom_instructions.present?
176-
base_prompt += "\n\nAdditional site-specific instructions:\n#{custom_instructions}"
177-
end
178-
179-
base_prompt
180-
end
181-
182-
def self.create_log_entry(post, llm_model, result, is_spam)
183-
log = AiSpamLog.find_or_initialize_by(post_id: post.id)
184-
185-
if log.new_record?
186-
log.llm_model = llm_model
187-
log.is_spam = is_spam
188-
log.scan_count = 1
189-
else
190-
log.scan_count += 1
191-
end
192-
193-
last_audit = DiscourseAi::ApiAuditLog.last
194-
log.last_ai_api_audit_log_id = last_audit.id if last_audit
195-
196-
log.save!
197-
end
198-
199-
def self.handle_spam(post, result)
200-
SpamRule::AutoSilence.new(post.user, post).silence_user
201-
202-
reason = I18n.t("discourse_ai.spam_detection.flag_reason", result: result)
203-
204-
PostActionCreator.new(
205-
Discourse.system_user,
206-
post,
207-
PostActionType.types[:spam],
208-
message: reason,
209-
queue_for_review: true,
210-
).perform
211-
end
21215
end
21316
end
21417
end

0 commit comments

Comments
 (0)