[Tooling] Add localization automation (#765)

* Use latest release-toolkit Gem

* Add Fluent / PO localization lanes

* Update source PO file (en-US.pot) to be synced to GlotPress

* Update generated Fluent files based on latest GlotPress translations

* Update release-toolkit

* Fix Rubocop violations

* Run bundle update

* Create temporary dir for downloading PO files and use open-uri for downloading

* Update code to use `fluent-tools` Gem

* Update generated source PO file

* Update generated Fluent files based on latest GlotPress translations

* Reorder helper lane

* Validate file_path at the beginning of the `generate_fluent_file_from_po` lane

* Update code to exit early in `download_translations` when there are no downloaded files

* Add all locales supported by the iOS and Android apps
h/t to @crazytonyli for highlighting that the supported locales should be a combination of the locales supported in the iOS and Android apps #765 (comment)

* Add generated Fluent files with the updated list of supported locales

* Update fluent-tools to 0.2.0 and re-generate localization files with the latest version

* Update to 0.3.0 release

* Update PO file with latest fluent-tools

Author: iangmaia

Gemfile

@@ -4,3 +4,4 @@ source 'https://rubygems.org'
 
 gem 'fastlane', '~> 2.228'
 gem 'fastlane-plugin-wpmreleasetoolkit', '~> 13.3'
+gem 'fluent-tools', '~> 0.3'

Gemfile.lock

@@ -153,6 +153,8 @@ GEM
       xcodeproj (~> 1.22)
     fastlane-sirp (1.0.0)
       sysrandom (~> 1.0)
+    fluent-tools (0.3.0)
+      thor (~> 1.0)
     gh_inspector (1.1.3)
     git (1.19.1)
       addressable (~> 2.8)
@@ -215,10 +217,22 @@ GEM
     nanaimo (0.4.0)
     naturally (2.3.0)
     nkf (0.2.0)
+    nokogiri (1.18.8-aarch64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.8-aarch64-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.18.8-arm-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.8-arm-linux-musl)
+      racc (~> 1.4)
     nokogiri (1.18.8-arm64-darwin)
       racc (~> 1.4)
+    nokogiri (1.18.8-x86_64-darwin)
+      racc (~> 1.4)
     nokogiri (1.18.8-x86_64-linux-gnu)
       racc (~> 1.4)
+    nokogiri (1.18.8-x86_64-linux-musl)
+      racc (~> 1.4)
     octokit (6.1.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
@@ -262,6 +276,7 @@ GEM
     terminal-notifier (2.0.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
+    thor (1.3.2)
     trailblazer-option (0.1.2)
     tty-cursor (0.7.1)
     tty-screen (0.8.2)
@@ -286,13 +301,19 @@ GEM
       xcpretty (~> 0.2, >= 0.0.7)
 
 PLATFORMS
-  arm64-darwin-22
-  arm64-darwin-23
-  x86_64-linux
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
   fastlane (~> 2.228)
   fastlane-plugin-wpmreleasetoolkit (~> 13.3)
+  fluent-tools (~> 0.3)
 
 BUNDLED WITH
-   2.4.10
+   2.6.3

fastlane/Fastfile

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'open-uri'
+require 'fluent_tools'
+
 PROJECT_ROOT = File.expand_path('..', __dir__)
 
 LANE_VALUE_VERSION = 'WP_VERSION'
@@ -11,6 +14,72 @@ LANE_VALUE_XCFRAMEWORK_CHECKSUM_PATH = 'WP_XCFRAMEWORK_CHECKSUM_PATH'
 GITHUB_REPO = 'automattic/wordpress-rs'
 GIT_REMOTE_NAME = 'origin'
 
+# Localization constants
+LOCALIZATION_FLUENT_FILES_DIR = File.join(PROJECT_ROOT, 'wp_localization', 'localization')
+LOCALIZATION_PO_SOURCE_FILE = File.join(PROJECT_ROOT, 'wp_localization', 'glotpress', 'en-US.pot')
+MAIN_FLUENT_FILE_NAME = 'main.ftl'
+PROJECT_NAME = 'wordpress-rs'
+
+# GlotPress configuration
+GLOTPRESS_PROJECT_BASE_URL = 'https://translate.wordpress.com/projects/mobile/wordpress-rs'
+
+# Supported locales mapping between GlotPress and project locale codes
+# This list combines locales supported in the iOS and Android apps
+SUPPORTED_LOCALES = [
+  { glotpress: 'ar',    project: 'ar'     },
+  { glotpress: 'az',    project: 'az'     },
+  { glotpress: 'bg',    project: 'bg'     },
+  { glotpress: 'cs',    project: 'cs'     },
+  { glotpress: 'cy',    project: 'cy'     },
+  { glotpress: 'da',    project: 'da'     },
+  { glotpress: 'de',    project: 'de'     },
+  { glotpress: 'el',    project: 'el'     },
+  { glotpress: 'en-au', project: 'en-AU'  },
+  { glotpress: 'en-ca', project: 'en-CA'  },
+  { glotpress: 'en-gb', project: 'en-GB'  },
+  { glotpress: 'es',    project: 'es'     },
+  { glotpress: 'es-cl', project: 'es-CL'  },
+  { glotpress: 'es-co', project: 'es-CO'  },
+  { glotpress: 'es-mx', project: 'es-MX'  },
+  { glotpress: 'es-ve', project: 'es-VE'  },
+  { glotpress: 'eu',    project: 'eu'     },
+  { glotpress: 'fr',    project: 'fr'     },
+  { glotpress: 'fr-ca', project: 'fr-CA'  },
+  { glotpress: 'gd',    project: 'gd'     },
+  { glotpress: 'gl',    project: 'gl'     },
+  { glotpress: 'he',    project: 'he'     },
+  { glotpress: 'hi',    project: 'hi'     },
+  { glotpress: 'hr',    project: 'hr'     },
+  { glotpress: 'hu',    project: 'hu'     },
+  { glotpress: 'id',    project: 'id'     },
+  { glotpress: 'is',    project: 'is'     },
+  { glotpress: 'it',    project: 'it'     },
+  { glotpress: 'ja',    project: 'ja'     },
+  { glotpress: 'kmr',   project: 'kmr'    },
+  { glotpress: 'ko',    project: 'ko'     },
+  { glotpress: 'lv',    project: 'lv'     },
+  { glotpress: 'mk',    project: 'mk'     },
+  { glotpress: 'ms',    project: 'ms'     },
+  { glotpress: 'nb',    project: 'nb'     },
+  { glotpress: 'nl',    project: 'nl'     },
+  { glotpress: 'pl',    project: 'pl'     },
+  { glotpress: 'pt',    project: 'pt'     },
+  { glotpress: 'pt-br', project: 'pt-BR'  },
+  { glotpress: 'ro',    project: 'ro'     },
+  { glotpress: 'ru',    project: 'ru'     },
+  { glotpress: 'sk',    project: 'sk'     },
+  { glotpress: 'sq',    project: 'sq'     },
+  { glotpress: 'sr',    project: 'sr'     },
+  { glotpress: 'sv',    project: 'sv'     },
+  { glotpress: 'th',    project: 'th'     },
+  { glotpress: 'tr',    project: 'tr-TR'  },
+  { glotpress: 'uz',    project: 'uz'     },
+  { glotpress: 'vi',    project: 'vi'     },
+  { glotpress: 'zh-cn', project: 'zh-CN'  },
+  { glotpress: 'zh-hk', project: 'zh-HK'  },
+  { glotpress: 'zh-tw', project: 'zh-TW'  }
+].freeze
+
 lane :release do |options|
   version = options[:version] || UI.user_error!('version is required')
   lane_context[LANE_VALUE_VERSION] = version
@@ -105,6 +174,154 @@ lane :publish_to_s3 do
   )
 end
 
+# Converts the English Fluent localization file to PO format for translation
+#
+# The resulting PO file is saved as the source file (.pot) for translations and is synced to GlotPress.
+#
+# @param commit_and_push_changes [Boolean] Whether to commit and push the generated PO file (default: false)
+#
+lane :generate_source_po_file do |commit_and_push_changes: false|
+  UI.header('🔄 Converting English Fluent file to PO format')
+
+  FileUtils.mkdir_p(File.dirname(LOCALIZATION_PO_SOURCE_FILE))
+
+  fluent_file = File.join(LOCALIZATION_FLUENT_FILES_DIR, 'en-US', MAIN_FLUENT_FILE_NAME)
+
+  UI.user_error!("❌ English Fluent file not found: #{fluent_file}") unless File.exist?(fluent_file)
+
+  begin
+    FluentTools.fluent_to_po(
+      fluent_file,
+      LOCALIZATION_PO_SOURCE_FILE,
+      locale: 'en-US'
+    )
+
+    UI.success("✅ #{File.basename(fluent_file)} → #{File.basename(LOCALIZATION_PO_SOURCE_FILE)}")
+  rescue StandardError => e
+    UI.error("❌ Failed to convert English Fluent file: #{e.message}")
+  end
+
+  if commit_and_push_changes
+    commit_changed_files(
+      files: LOCALIZATION_PO_SOURCE_FILE,
+      message: 'Update source PO file (en-US.pot) to be synced to GlotPress'
+    )
+  end
+end
+
+# Downloads the latest translations from GlotPress and updates Fluent files
+#
+# This lane fetches translated PO files from GlotPress for all supported locales,
+# converts them back to Fluent format, and optionally commits and pushes the changes.
+#
+# @param commit_and_push_changes [Boolean] Whether to commit and push the updated Fluent files (default: false)
+#
+lane :download_translations do |commit_and_push_changes: false|
+  UI.header('🌐 Downloading translations from GlotPress')
+
+  Dir.mktmpdir do |temp_download_dir|
+    downloaded_files = download_po_files_from_glotpress(download_dir: temp_download_dir)
+    updated_fluent_files = []
+
+    if downloaded_files.empty?
+      UI.message("No .po files were downloaded from GlotPress")
+      next
+    end
+
+    UI.header('🔄 Converting PO files to Fluent format')
+
+    downloaded_files.each do |file_path|
+      fluent_file_path = generate_fluent_file_from_po(file_path: file_path)
+      updated_fluent_files << fluent_file_path if fluent_file_path
+    rescue StandardError => e
+      UI.error("❌ Failed to convert #{File.basename(file_path)}: #{e.message}")
+    end
+
+    UI.success("✅ Updated Fluent files: #{updated_fluent_files.length} locales")
+
+    if commit_and_push_changes && updated_fluent_files.any?
+      commit_changed_files(
+        files: updated_fluent_files,
+        message: 'Update generated Fluent files based on latest GlotPress translations'
+      )
+    end
+  end
+end
+
+# Downloads PO files from GlotPress for existing project locales
+#
+# This lane fetches translated PO files from GlotPress for all supported locales
+# that exist in the project. It only downloads files for locales that have
+# directories in the source localization folder.
+#
+# @return [Array<String>] List of successfully downloaded files
+#
+lane :download_po_files_from_glotpress do |download_dir:|
+  UI.header('🌐 Downloading PO files from GlotPress')
+
+  UI.user_error!("Download directory does not exist: #{download_dir}") unless Dir.exist?(download_dir)
+
+  downloaded_files = []
+
+  SUPPORTED_LOCALES.each do |locale_mapping|
+    glotpress_locale = locale_mapping[:glotpress]
+    project_locale = locale_mapping[:project]
+
+    po_file_path = File.join(download_dir, "#{project_locale}.po")
+    download_url = "#{GLOTPRESS_PROJECT_BASE_URL}/#{glotpress_locale}/default/export-translations/?format=po"
+
+    UI.message("⬇️ Downloading PO file for #{project_locale}...")
+
+    begin
+      # Download PO file using Ruby's URI.open
+      File.write(po_file_path, URI.open(download_url).read)
+
+      if File.exist?(po_file_path) && !File.empty?(po_file_path)
+        downloaded_files << po_file_path
+      else
+        UI.error("❌ Failed to download #{project_locale}: empty response")
+      end
+    rescue StandardError => e
+      UI.error("❌ Failed to download #{project_locale}: #{e.message}")
+    end
+  end
+
+  UI.success("✅ Downloaded: #{downloaded_files.length} PO files") if downloaded_files.any?
+
+  downloaded_files
+end
+
+# Converts a PO file for a given locale back to Fluent format
+#
+# This lane takes a PO file and converts it to the corresponding Fluent format file.
+# The locale is extracted from the PO filename (e.g., 'fr-FR.po' becomes 'fr-FR').
+# The resulting Fluent file is saved in the appropriate locale directory.
+#
+# @param file_path [String] The PO file path to convert (e.g., 'path/to/fr-FR.po')
+# @return [String] The path to the generated Fluent file
+#
+lane :generate_fluent_file_from_po do |file_path:|
+  UI.user_error!("PO file not found: #{file_path}") unless File.exist?(file_path)
+
+  locale = File.basename(file_path, '.po')
+  fluent_file_path = File.join(LOCALIZATION_FLUENT_FILES_DIR, locale, MAIN_FLUENT_FILE_NAME)
+
+  FileUtils.mkdir_p(File.dirname(fluent_file_path))
+
+  FluentTools.po_to_fluent(
+    file_path,
+    fluent_file_path
+  )
+
+  next if !File.exist?(fluent_file_path) || File.empty?(fluent_file_path)
+
+  UI.message("✅ #{File.basename(file_path)} → #{fluent_file_path}")
+
+  fluent_file_path
+end
+
+# Utils
+
 def xcframework_checksum
   File.read(xcframework_checksum_file_path).strip!
 end
@@ -126,3 +343,18 @@ end
 def remove_lane_context_values(names)
   names.each { |name| lane_context.delete(name) }
 end
+
+def commit_changed_files(files:, message:, push: true)
+  git_add(path: files)
+  result = git_commit(
+    path: files,
+    message: message,
+    allow_nothing_to_commit: true
+  )
+
+  if result.nil?
+    UI.important('⚠️ No changed files')
+  elsif push
+    push_to_git_remote(set_upstream: true, tags: false)
+  end
+end