Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.

Commit 0f8954b

Browse files
nattswnattsw
committed
FEATURE: Automatic translation and localization of posts, topics, categories
1 parent 39653ae commit 0f8954b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2760
-0
lines changed

‎app/jobs/regular/detect_translate_post.rb‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class DetectTranslatePost < ::Jobs::Base
5+
sidekiq_options retry: false
6+
7+
def execute(args)
8+
return if !SiteSetting.discourse_ai_enabled
9+
return if !SiteSetting.ai_translation_enabled
10+
return if args[:post_id].blank?
11+
12+
post = Post.find_by(id: args[:post_id])
13+
return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
14+
15+
if SiteSetting.ai_translation_backfill_limit_to_public_content
16+
topic = post.topic
17+
return if topic.blank? || topic.category&.read_restricted?
18+
end
19+
20+
begin
21+
detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
22+
rescue FinalDestination::SSRFDetector::LookupFailedError
23+
# this job is non-critical
24+
# the backfill job will handle failures
25+
return
26+
end
27+
28+
locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
29+
return if locales.blank?
30+
31+
locales.each do |locale|
32+
next if locale == detected_locale
33+
34+
begin
35+
DiscourseAi::Translation::PostLocalizer.localize(post, locale)
36+
rescue FinalDestination::SSRFDetector::LookupFailedError
37+
# do nothing, there are too many sporadic lookup failures
38+
rescue => e
39+
DiscourseAi::Translation::VerboseLogger.log(
40+
"Failed to translate post #{post.id} to #{locale}: #{e.message}",
41+
)
42+
end
43+
end
44+
end
45+
end
46+
end

‎app/jobs/regular/detect_translate_topic.rb‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class DetectTranslateTopic < ::Jobs::Base
5+
sidekiq_options retry: false
6+
7+
def execute(args)
8+
return if !SiteSetting.discourse_ai_enabled
9+
return if !SiteSetting.ai_translation_enabled
10+
return if args[:topic_id].blank?
11+
12+
topic = Topic.find_by(id: args[:topic_id])
13+
if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
14+
return
15+
end
16+
17+
if SiteSetting.ai_translation_backfill_limit_to_public_content
18+
return if topic.category&.read_restricted?
19+
end
20+
21+
begin
22+
detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
23+
rescue FinalDestination::SSRFDetector::LookupFailedError
24+
# this job is non-critical
25+
# the backfill job will handle failures
26+
return
27+
end
28+
29+
locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
30+
return if locales.blank?
31+
32+
locales.each do |locale|
33+
next if locale == detected_locale
34+
35+
begin
36+
DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
37+
rescue FinalDestination::SSRFDetector::LookupFailedError
38+
# do nothing, there are too many sporadic lookup failures
39+
rescue => e
40+
DiscourseAi::Translation::VerboseLogger.log(
41+
"Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
42+
)
43+
end
44+
end
45+
end
46+
end
47+
end

‎app/jobs/regular/localize_categories.rb‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class LocalizeCategories < ::Jobs::Base
5+
cluster_concurrency 1
6+
sidekiq_options retry: false
7+
8+
BATCH_SIZE = 50
9+
10+
def execute(args)
11+
return if !SiteSetting.discourse_ai_enabled
12+
return if !SiteSetting.ai_translation_enabled
13+
14+
locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
15+
return if locales.blank?
16+
17+
cat_id = args[:from_category_id] || Category.order(:id).first&.id
18+
last_id = nil
19+
20+
categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
21+
return if categories.empty?
22+
23+
categories.each do |category|
24+
if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
25+
last_id = category.id
26+
next
27+
end
28+
29+
CategoryLocalization.transaction do
30+
locales.each do |locale|
31+
next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
32+
begin
33+
DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
34+
rescue FinalDestination::SSRFDetector::LookupFailedError
35+
# do nothing, there are too many sporadic lookup failures
36+
rescue => e
37+
DiscourseAi::Translation::VerboseLogger.log(
38+
"Failed to translate category #{category.id} to #{locale}: #{e.message}",
39+
)
40+
end
41+
end
42+
end
43+
last_id = category.id
44+
end
45+
46+
if categories.size == BATCH_SIZE
47+
Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
48+
end
49+
end
50+
end
51+
end

‎app/jobs/regular/localize_posts.rb‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class LocalizePosts < ::Jobs::Base
5+
cluster_concurrency 1
6+
sidekiq_options retry: false
7+
8+
BATCH_SIZE = 50
9+
10+
def execute(args)
11+
return if !SiteSetting.discourse_ai_enabled
12+
return if !SiteSetting.ai_translation_enabled
13+
14+
locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
15+
return if locales.blank?
16+
17+
limit = args[:limit] || BATCH_SIZE
18+
19+
locales.each do |locale|
20+
posts =
21+
Post
22+
.joins(
23+
"LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
24+
)
25+
.where(deleted_at: nil)
26+
.where("posts.user_id > 0")
27+
.where.not(raw: [nil, ""])
28+
.where.not(locale: nil)
29+
.where.not(locale: locale)
30+
.where("pl.id IS NULL")
31+
32+
if SiteSetting.ai_translation_backfill_limit_to_public_content
33+
posts =
34+
posts.joins(:topic).where(
35+
topics: {
36+
category_id: Category.where(read_restricted: false).select(:id),
37+
archetype: "regular",
38+
},
39+
)
40+
end
41+
42+
if SiteSetting.ai_translation_backfill_max_age_days > 0
43+
posts =
44+
posts.where(
45+
"posts.created_at > ?",
46+
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
47+
)
48+
end
49+
50+
posts = posts.order(updated_at: :desc).limit(limit)
51+
52+
next if posts.empty?
53+
54+
posts.each do |post|
55+
begin
56+
DiscourseAi::Translation::PostLocalizer.localize(post, locale)
57+
rescue FinalDestination::SSRFDetector::LookupFailedError
58+
# do nothing, there are too many sporadic lookup failures
59+
rescue => e
60+
DiscourseAi::Translation::VerboseLogger.log(
61+
"Failed to translate post #{post.id} to #{locale}: #{e.message}",
62+
)
63+
end
64+
end
65+
66+
DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
67+
end
68+
end
69+
end
70+
end

‎app/jobs/regular/localize_topics.rb‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class LocalizeTopics < ::Jobs::Base
5+
cluster_concurrency 1
6+
sidekiq_options retry: false
7+
8+
BATCH_SIZE = 50
9+
10+
def execute(args)
11+
return if !SiteSetting.discourse_ai_enabled
12+
return if !SiteSetting.ai_translation_enabled
13+
14+
locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
15+
return if locales.blank?
16+
17+
limit = args[:limit] || BATCH_SIZE
18+
19+
locales.each do |locale|
20+
topics =
21+
Topic
22+
.joins(
23+
"LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
24+
)
25+
.where(deleted_at: nil)
26+
.where("topics.user_id > 0")
27+
.where.not(locale: nil)
28+
.where.not(locale: locale)
29+
.where("tl.id IS NULL")
30+
31+
if SiteSetting.ai_translation_backfill_limit_to_public_content
32+
topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
33+
end
34+
35+
if SiteSetting.ai_translation_backfill_max_age_days > 0
36+
topics =
37+
topics.where(
38+
"topics.created_at > ?",
39+
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
40+
)
41+
end
42+
43+
topics = topics.order(updated_at: :desc).limit(limit)
44+
45+
next if topics.empty?
46+
47+
topics.each do |topic|
48+
begin
49+
DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
50+
rescue FinalDestination::SSRFDetector::LookupFailedError
51+
# do nothing, there are too many sporadic lookup failures
52+
rescue => e
53+
DiscourseAi::Translation::VerboseLogger.log(
54+
"Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
55+
)
56+
end
57+
end
58+
59+
DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
60+
end
61+
end
62+
end
63+
end

‎app/jobs/scheduled/category_localization_backfill.rb‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class CategoryLocalizationBackfill < ::Jobs::Scheduled
5+
every 12.hours
6+
cluster_concurrency 1
7+
8+
def execute(args)
9+
return if !SiteSetting.discourse_ai_enabled
10+
return if !SiteSetting.ai_translation_enabled
11+
return if SiteSetting.experimental_content_localization_supported_locales.blank?
12+
13+
Jobs.enqueue(:localize_categories)
14+
end
15+
end
16+
end

‎app/jobs/scheduled/post_localization_backfill.rb‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class PostLocalizationBackfill < ::Jobs::Scheduled
5+
every 5.minutes
6+
cluster_concurrency 1
7+
8+
def execute(args)
9+
return if !SiteSetting.discourse_ai_enabled
10+
return if !SiteSetting.ai_translation_enabled
11+
12+
return if SiteSetting.experimental_content_localization_supported_locales.blank?
13+
return if SiteSetting.ai_translation_backfill_rate == 0
14+
15+
Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
16+
end
17+
end
18+
end

‎app/jobs/scheduled/posts_locale_detection_backfill.rb‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
5+
every 5.minutes
6+
sidekiq_options retry: false
7+
cluster_concurrency 1
8+
9+
def execute(args)
10+
return if !SiteSetting.discourse_ai_enabled
11+
return if !SiteSetting.ai_translation_enabled
12+
return if SiteSetting.ai_translation_backfill_rate == 0
13+
14+
posts =
15+
Post
16+
.where(locale: nil)
17+
.where(deleted_at: nil)
18+
.where("posts.user_id > 0")
19+
.where.not(raw: [nil, ""])
20+
21+
if SiteSetting.ai_translation_backfill_limit_to_public_content
22+
public_categories = Category.where(read_restricted: false).pluck(:id)
23+
posts =
24+
posts
25+
.joins(:topic)
26+
.where(topics: { category_id: public_categories })
27+
.where(topics: { archetype: "regular" })
28+
end
29+
30+
if SiteSetting.ai_translation_backfill_max_age_days > 0
31+
posts =
32+
posts.where(
33+
"posts.created_at > ?",
34+
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
35+
)
36+
end
37+
38+
posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
39+
return if posts.empty?
40+
41+
posts.each do |post|
42+
begin
43+
DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
44+
rescue FinalDestination::SSRFDetector::LookupFailedError
45+
# do nothing, there are too many sporadic lookup failures
46+
rescue => e
47+
DiscourseAi::Translation::VerboseLogger.log(
48+
"Failed to detect post #{post.id}'s locale: #{e.message}",
49+
)
50+
end
51+
end
52+
53+
DiscourseAi::Translation::VerboseLogger.log("Detected #{posts.size} post locales")
54+
end
55+
end
56+
end

‎app/jobs/scheduled/topic_localization_backfill.rb‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# frozen_string_literal: true
2+
3+
module Jobs
4+
class TopicLocalizationBackfill < ::Jobs::Scheduled
5+
every 5.minutes
6+
cluster_concurrency 1
7+
8+
def execute(args)
9+
return if !SiteSetting.discourse_ai_enabled
10+
return if !SiteSetting.ai_translation_enabled
11+
12+
return if SiteSetting.experimental_content_localization_supported_locales.blank?
13+
return if SiteSetting.ai_translation_backfill_rate == 0
14+
15+
Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
16+
end
17+
end
18+
end

0 commit comments

Comments
 (0)