Skip to content
This repository was archived by the owner on Jul 22, 2025. It is now read-only.

Commit cbb8065

Browse files
nattswnattsw
committed
DEV: Normalize locales that are similar (e.g. en and en_GB) so they do not get translated
1 parent 699ea3f commit cbb8065

25 files changed

+209
-134
lines changed

‎app/jobs/regular/detect_translate_post.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ class DetectTranslatePost < ::Jobs::Base
66
sidekiq_options retry: false
77

88
def execute(args)
9-
return if !SiteSetting.discourse_ai_enabled
10-
return if !SiteSetting.ai_translation_enabled
9+
return if !DiscourseAi::Translation.enabled?
1110
return if args[:post_id].blank?
1211

1312
post = Post.find_by(id: args[:post_id])
@@ -36,12 +35,14 @@ def execute(args)
3635
end
3736
end
3837

38+
return if detected_locale.blank?
3939
locales = SiteSetting.content_localization_supported_locales.split("|")
4040
return if locales.blank?
4141

4242
locales.each do |locale|
43-
next if locale == detected_locale
44-
next if post.post_localizations.exists?(locale:)
43+
next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
44+
regionless_locale = locale.split("_").first
45+
next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
4546

4647
begin
4748
DiscourseAi::Translation::PostLocalizer.localize(post, locale)

‎app/jobs/regular/detect_translate_topic.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ class DetectTranslateTopic < ::Jobs::Base
66
sidekiq_options retry: false
77

88
def execute(args)
9-
return if !SiteSetting.discourse_ai_enabled
10-
return if !SiteSetting.ai_translation_enabled
9+
return if !DiscourseAi::Translation.enabled?
1110
return if args[:topic_id].blank?
1211

1312
topic = Topic.find_by(id: args[:topic_id])
@@ -34,12 +33,14 @@ def execute(args)
3433
end
3534
end
3635

36+
return if detected_locale.blank?
3737
locales = SiteSetting.content_localization_supported_locales.split("|")
3838
return if locales.blank?
3939

4040
locales.each do |locale|
41-
next if locale == detected_locale
42-
next if topic.topic_localizations.exists?(locale:)
41+
next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
42+
regionless_locale = locale.split("_").first
43+
next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
4344

4445
begin
4546
DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)

‎app/jobs/regular/localize_categories.rb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,17 @@ class LocalizeCategories < ::Jobs::Base
66
sidekiq_options retry: false
77

88
def execute(args)
9+
return if !DiscourseAi::Translation.enabled?
10+
911
limit = args[:limit]
1012
raise Discourse::InvalidParameters.new(:limit) if limit.nil?
1113
return if limit <= 0
12-
13-
return if !SiteSetting.discourse_ai_enabled
14-
return if !SiteSetting.ai_translation_enabled
1514
locales = SiteSetting.content_localization_supported_locales.split("|")
16-
return if locales.blank?
1715

1816
categories = Category.where("locale IS NOT NULL")
19-
2017
if SiteSetting.ai_translation_backfill_limit_to_public_content
2118
categories = categories.where(read_restricted: false)
2219
end
23-
2420
categories = categories.order(:id).limit(limit)
2521
return if categories.empty?
2622

@@ -33,6 +29,7 @@ def execute(args)
3329
missing_locales = locales - existing_locales - [category.locale]
3430
missing_locales.each do |locale|
3531
break if remaining_limit <= 0
32+
next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
3633

3734
begin
3835
DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)

‎app/jobs/regular/localize_posts.rb‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,25 @@ def execute(args)
99
limit = args[:limit]
1010
raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
1111

12-
return if !SiteSetting.discourse_ai_enabled
13-
return if !SiteSetting.ai_translation_enabled
12+
return if !DiscourseAi::Translation.backfill_enabled?
1413

1514
locales = SiteSetting.content_localization_supported_locales.split("|")
16-
return if locales.blank?
17-
1815
locales.each do |locale|
16+
base_locale = locale.split("_").first
1917
posts =
2018
Post
2119
.joins(
22-
"LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
20+
"LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
21+
)
22+
.where(
23+
"posts.created_at > ?",
24+
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
2325
)
2426
.where(deleted_at: nil)
2527
.where("posts.user_id > 0")
2628
.where.not(raw: [nil, ""])
2729
.where.not(locale: nil)
28-
.where.not(locale: locale)
30+
.where("posts.locale NOT LIKE '#{base_locale}%'")
2931
.where("pl.id IS NULL")
3032

3133
posts = posts.joins(:topic)
@@ -46,14 +48,6 @@ def execute(args)
4648
)
4749
end
4850

49-
if SiteSetting.ai_translation_backfill_max_age_days > 0
50-
posts =
51-
posts.where(
52-
"posts.created_at > ?",
53-
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
54-
)
55-
end
56-
5751
posts = posts.order(updated_at: :desc).limit(limit)
5852

5953
next if posts.empty?

‎app/jobs/regular/localize_topics.rb‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,24 @@ def execute(args)
99
limit = args[:limit]
1010
raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
1111

12-
return if !SiteSetting.discourse_ai_enabled
13-
return if !SiteSetting.ai_translation_enabled
12+
return if !DiscourseAi::Translation.backfill_enabled?
1413

1514
locales = SiteSetting.content_localization_supported_locales.split("|")
16-
return if locales.blank?
17-
1815
locales.each do |locale|
16+
base_locale = locale.split("_").first
1917
topics =
2018
Topic
2119
.joins(
22-
"LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
20+
"LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
21+
)
22+
.where(
23+
"topics.created_at > ?",
24+
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
2325
)
2426
.where(deleted_at: nil)
2527
.where("topics.user_id > 0")
2628
.where.not(locale: nil)
27-
.where.not(locale: locale)
29+
.where("topics.locale NOT LIKE '#{base_locale}%'")
2830
.where("tl.id IS NULL")
2931

3032
if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -43,14 +45,6 @@ def execute(args)
4345
)
4446
end
4547

46-
if SiteSetting.ai_translation_backfill_max_age_days > 0
47-
topics =
48-
topics.where(
49-
"topics.created_at > ?",
50-
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
51-
)
52-
end
53-
5448
topics = topics.order(updated_at: :desc).limit(limit)
5549

5650
next if topics.empty?

‎app/jobs/scheduled/categories_locale_detection_backfill.rb‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,15 @@ class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
77
cluster_concurrency 1
88

99
def execute(args)
10-
return if !SiteSetting.discourse_ai_enabled
11-
return if !SiteSetting.ai_translation_enabled
12-
limit = SiteSetting.ai_translation_backfill_hourly_rate
13-
return if limit == 0
10+
return if !DiscourseAi::Translation.backfill_enabled?
1411

1512
categories = Category.where(locale: nil)
1613

1714
if SiteSetting.ai_translation_backfill_limit_to_public_content
1815
categories = categories.where(read_restricted: false)
1916
end
2017

18+
limit = SiteSetting.ai_translation_backfill_hourly_rate
2119
categories = categories.limit(limit)
2220
return if categories.empty?
2321

‎app/jobs/scheduled/category_localization_backfill.rb‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ class CategoryLocalizationBackfill < ::Jobs::Scheduled
66
cluster_concurrency 1
77

88
def execute(args)
9-
return if !SiteSetting.discourse_ai_enabled
10-
return if !SiteSetting.ai_translation_enabled
11-
return if SiteSetting.content_localization_supported_locales.blank?
9+
return if !DiscourseAi::Translation.backfill_enabled?
1210
limit = SiteSetting.ai_translation_backfill_hourly_rate
13-
return if limit == 0
1411

1512
Jobs.enqueue(:localize_categories, limit:)
1613
end

‎app/jobs/scheduled/post_localization_backfill.rb‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,8 @@ class PostLocalizationBackfill < ::Jobs::Scheduled
66
cluster_concurrency 1
77

88
def execute(args)
9-
return if !SiteSetting.discourse_ai_enabled
10-
return if !SiteSetting.ai_translation_enabled
9+
return if !DiscourseAi::Translation.backfill_enabled?
1110

12-
return if SiteSetting.content_localization_supported_locales.blank?
1311
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
1412
return if limit == 0
1513

‎app/jobs/scheduled/posts_locale_detection_backfill.rb‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,16 @@ class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
77
cluster_concurrency 1
88

99
def execute(args)
10-
return if !SiteSetting.discourse_ai_enabled
11-
return if !SiteSetting.ai_translation_enabled
10+
return if !DiscourseAi::Translation.backfill_enabled?
11+
1212
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
13-
return if limit == 0
1413

1514
posts =
1615
Post
1716
.where(locale: nil)
1817
.where(deleted_at: nil)
1918
.where("posts.user_id > 0")
19+
.where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
2020
.where.not(raw: [nil, ""])
2121

2222
if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -33,14 +33,6 @@ def execute(args)
3333
)
3434
end
3535

36-
if SiteSetting.ai_translation_backfill_max_age_days > 0
37-
posts =
38-
posts.where(
39-
"posts.created_at > ?",
40-
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
41-
)
42-
end
43-
4436
posts = posts.order(updated_at: :desc).limit(limit)
4537
return if posts.empty?
4638

‎app/jobs/scheduled/topic_localization_backfill.rb‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,9 @@ class TopicLocalizationBackfill < ::Jobs::Scheduled
66
cluster_concurrency 1
77

88
def execute(args)
9-
return if !SiteSetting.discourse_ai_enabled
10-
return if !SiteSetting.ai_translation_enabled
9+
return if !DiscourseAi::Translation.backfill_enabled?
1110

12-
return if SiteSetting.content_localization_supported_locales.blank?
1311
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
14-
return if limit == 0
15-
1612
Jobs.enqueue(:localize_topics, limit:)
1713
end
1814
end

0 commit comments

Comments
 (0)