Skip to content

Commit df2fca5

Browse files
sylvestresylvestre
committed
l10n: when built indiv, only embed the relevant locale file
1 parent afa0f00 commit df2fca5

File tree

6 files changed

+350
-170
lines changed

6 files changed

+350
-170
lines changed

‎.github/workflows/l10n.yml‎

Lines changed: 150 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,10 @@ jobs:
5858
brew install coreutils
5959
;;
6060
esac
61-
- name: Build with l10n
61+
- name: Build with platform features
6262
shell: bash
6363
run: |
64-
## Build with l10n
64+
## Build with platform-specific features to enable l10n functionality
6565
cargo build --features ${{ matrix.job.features }}
6666
- name: Test l10n functionality
6767
shell: bash
@@ -96,14 +96,14 @@ jobs:
9696
# Check if any .ftl files exist
9797
fluent_files=$(find . -name "*.ftl" -type f 2>/dev/null || true)
9898
99-
if [ -z "$fluent_files" ]; then
99+
if [ -n "$fluent_files" ]; then
100+
echo "Found Fluent files:"
101+
echo "$fluent_files"
102+
else
100103
echo "::notice::No Fluent (.ftl) files found in the repository"
101104
exit 0
102105
fi
103106
104-
echo "Found Fluent files:"
105-
echo "$fluent_files"
106-
107107
# Use Mozilla Fluent Linter for comprehensive validation
108108
echo "Running Mozilla Fluent Linter..."
109109
@@ -952,3 +952,147 @@ jobs:
952952
echo "✓ All locale-specific functionality tests passed"
953953
env:
954954
RUST_BACKTRACE: "1"
955+
956+
l10n_locale_embedding_regression_test:
957+
name: L10n/Locale Embedding Regression Test
958+
runs-on: ubuntu-latest
959+
env:
960+
SCCACHE_GHA_ENABLED: "true"
961+
RUSTC_WRAPPER: "sccache"
962+
steps:
963+
- uses: actions/checkout@v4
964+
with:
965+
persist-credentials: false
966+
- uses: dtolnay/rust-toolchain@stable
967+
- uses: Swatinem/rust-cache@v2
968+
- name: Run sccache-cache
969+
uses: mozilla-actions/[email protected]
970+
- name: Install/setup prerequisites
971+
shell: bash
972+
run: |
973+
## Install/setup prerequisites
974+
sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev build-essential
975+
- name: Build binaries for locale embedding test
976+
shell: bash
977+
run: |
978+
## Build individual utilities and multicall binary for locale embedding test
979+
echo "Building binaries with different locale embedding configurations..."
980+
mkdir -p target
981+
982+
# Build cat utility with targeted locale embedding
983+
echo "Building cat utility with targeted locale embedding..."
984+
echo "cat" > target/uucore_target_util.txt
985+
cargo build -p uu_cat --release
986+
987+
# Build ls utility with targeted locale embedding
988+
echo "Building ls utility with targeted locale embedding..."
989+
echo "ls" > target/uucore_target_util.txt
990+
cargo build -p uu_ls --release
991+
992+
# Build multicall binary (should have all locales)
993+
echo "Building multicall binary (should have all locales)..."
994+
echo "multicall" > target/uucore_target_util.txt
995+
cargo build --release
996+
997+
echo "✓ All binaries built successfully"
998+
env:
999+
RUST_BACKTRACE: "1"
1000+
1001+
- name: Analyze embedded locale files
1002+
shell: bash
1003+
run: |
1004+
## Extract and analyze .ftl files embedded in each binary
1005+
echo "=== Embedded Locale File Analysis ==="
1006+
1007+
# Analyze cat binary
1008+
echo "--- cat binary embedded .ftl files ---"
1009+
cat_ftl_files=$(strings target/release/cat | grep -o "[a-z_][a-z_]*/en-US\.ftl" | sort | uniq)
1010+
cat_locales=$(echo "$cat_ftl_files" | wc -l)
1011+
if [ -n "$cat_ftl_files" ]; then
1012+
echo "$cat_ftl_files"
1013+
else
1014+
echo "(no locale keys found)"
1015+
fi
1016+
echo "Total: $cat_locales files"
1017+
echo
1018+
1019+
# Analyze ls binary
1020+
echo "--- ls binary embedded .ftl files ---"
1021+
ls_ftl_files=$(strings target/release/ls | grep -o "[a-z_][a-z_]*/en-US\.ftl" | sort | uniq)
1022+
ls_locales=$(echo "$ls_ftl_files" | wc -l)
1023+
if [ -n "$ls_ftl_files" ]; then
1024+
echo "$ls_ftl_files"
1025+
else
1026+
echo "(no locale keys found)"
1027+
fi
1028+
echo "Total: $ls_locales files"
1029+
echo
1030+
1031+
# Analyze multicall binary
1032+
echo "--- multicall binary embedded .ftl files (first 10) ---"
1033+
multi_ftl_files=$(strings target/release/coreutils | grep -o "[a-z_][a-z_]*/en-US\.ftl" | sort | uniq)
1034+
multi_locales=$(echo "$multi_ftl_files" | wc -l)
1035+
if [ -n "$multi_ftl_files" ]; then
1036+
echo "$multi_ftl_files" | head -10
1037+
echo "... (showing first 10 of $multi_locales total files)"
1038+
else
1039+
echo "(no locale keys found)"
1040+
fi
1041+
echo
1042+
1043+
# Store counts for validation step
1044+
echo "cat_locales=$cat_locales" >> $GITHUB_ENV
1045+
echo "ls_locales=$ls_locales" >> $GITHUB_ENV
1046+
echo "multi_locales=$multi_locales" >> $GITHUB_ENV
1047+
1048+
- name: Validate cat binary locale embedding
1049+
shell: bash
1050+
run: |
1051+
## Validate that cat binary only embeds its own locale files
1052+
echo "Validating cat binary locale embedding..."
1053+
if [ "$cat_locales" -le 5 ]; then
1054+
echo "✓ SUCCESS: cat binary uses targeted locale embedding ($cat_locales files)"
1055+
else
1056+
echo "✗ FAILURE: cat binary has too many embedded locale files ($cat_locales). Expected ≤ 5."
1057+
echo "This indicates LOCALE EMBEDDING REGRESSION - all locales are being embedded instead of just the target utility's locale."
1058+
echo "The optimization is not working correctly!"
1059+
exit 1
1060+
fi
1061+
1062+
- name: Validate ls binary locale embedding
1063+
shell: bash
1064+
run: |
1065+
## Validate that ls binary only embeds its own locale files
1066+
echo "Validating ls binary locale embedding..."
1067+
if [ "$ls_locales" -le 5 ]; then
1068+
echo "✓ SUCCESS: ls binary uses targeted locale embedding ($ls_locales files)"
1069+
else
1070+
echo "✗ FAILURE: ls binary has too many embedded locale files ($ls_locales). Expected ≤ 5."
1071+
echo "This indicates LOCALE EMBEDDING REGRESSION - all locales are being embedded instead of just the target utility's locale."
1072+
echo "The optimization is not working correctly!"
1073+
exit 1
1074+
fi
1075+
1076+
- name: Validate multicall binary locale embedding
1077+
shell: bash
1078+
run: |
1079+
## Validate that multicall binary embeds all utility locale files
1080+
echo "Validating multicall binary locale embedding..."
1081+
if [ "$multi_locales" -ge 80 ]; then
1082+
echo "✓ SUCCESS: multicall binary has all locales ($multi_locales files)"
1083+
else
1084+
echo "✗ FAILURE: multicall binary has too few embedded locale files ($multi_locales). Expected ≥ 80."
1085+
echo "This indicates the multicall binary is not getting all required locales."
1086+
exit 1
1087+
fi
1088+
1089+
- name: Finalize locale embedding tests
1090+
shell: bash
1091+
run: |
1092+
## Clean up and report overall test results
1093+
rm -f test.txt target/uucore_target_util.txt
1094+
echo "✓ All locale embedding regression tests passed"
1095+
echo "Summary:"
1096+
echo " - cat binary: $cat_locales locale files (targeted embedding)"
1097+
echo " - ls binary: $ls_locales locale files (targeted embedding)"
1098+
echo " - multicall binary: $multi_locales locale files (full embedding)"

‎build.rs‎

Lines changed: 0 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -105,83 +105,4 @@ pub fn main() {
105105
mf.write_all(b"\n}\n").unwrap();
106106

107107
mf.flush().unwrap();
108-
109-
// Always generate embedded English locale files
110-
// This ensures English works even when locale files aren't available (e.g., cargo install)
111-
generate_embedded_english_locales(&out_dir, &crates).unwrap();
112-
}
113-
114-
/// Generate embedded English locale files
115-
///
116-
/// # Errors
117-
///
118-
/// Returns an error if file operations fail or if there are I/O issues
119-
fn generate_embedded_english_locales(
120-
out_dir: &str,
121-
crates: &[String],
122-
) -> Result<(), Box<dyn std::error::Error>> {
123-
use std::fs;
124-
125-
let mut embedded_file = File::create(Path::new(&out_dir).join("embedded_locales.rs"))?;
126-
127-
writeln!(embedded_file, "// Generated at compile time - do not edit")?;
128-
writeln!(
129-
embedded_file,
130-
"// This file contains embedded English locale files for all utilities"
131-
)?;
132-
writeln!(embedded_file)?;
133-
writeln!(embedded_file, "use std::collections::HashMap;")?;
134-
writeln!(embedded_file)?;
135-
136-
// Start the function that returns embedded locales
137-
writeln!(
138-
embedded_file,
139-
"pub fn get_embedded_locales() -> HashMap<&'static str, &'static str> {{"
140-
)?;
141-
writeln!(embedded_file, " let mut locales = HashMap::new();")?;
142-
writeln!(embedded_file)?;
143-
144-
// Embed locale files for each utility
145-
for krate in crates {
146-
let util_name = if let Some(stripped) = krate.strip_prefix("uu_") {
147-
stripped
148-
} else {
149-
krate
150-
};
151-
152-
let locale_path = Path::new("src/uu")
153-
.join(util_name)
154-
.join("locales/en-US.ftl");
155-
if locale_path.exists() {
156-
let content = fs::read_to_string(&locale_path)?;
157-
writeln!(embedded_file, " // Locale for {util_name}")?;
158-
writeln!(
159-
embedded_file,
160-
" locales.insert(\"{util_name}/en-US.ftl\", r###\"{content}\"###);"
161-
)?;
162-
writeln!(embedded_file)?;
163-
164-
// Tell Cargo to rerun if this file changes
165-
println!("cargo:rerun-if-changed={}", locale_path.display());
166-
}
167-
}
168-
169-
// Also embed uucore locale file if it exists
170-
let uucore_locale_path = Path::new("src/uucore/locales/en-US.ftl");
171-
if uucore_locale_path.exists() {
172-
let content = fs::read_to_string(uucore_locale_path)?;
173-
writeln!(embedded_file, " // Common uucore locale")?;
174-
writeln!(
175-
embedded_file,
176-
" locales.insert(\"uucore/en-US.ftl\", r###\"{content}\"###);"
177-
)?;
178-
println!("cargo:rerun-if-changed={}", uucore_locale_path.display());
179-
}
180-
181-
writeln!(embedded_file)?;
182-
writeln!(embedded_file, " locales")?;
183-
writeln!(embedded_file, "}}")?;
184-
185-
embedded_file.flush()?;
186-
Ok(())
187108
}

‎docs/src/l10n.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ This guide explains how localization (L10n) is implemented in the **Rust-based c
44

55
## 🏗️ Architecture Overview
66

7-
**English locale files are embedded directly in the binary**, ensuring that English always works regardless of how the software is installed. Other language locale files are loaded from the filesystem at runtime.
7+
**English (US) locale files (`en-US.ftl`) are embedded directly in the binary**, ensuring that English always works regardless of how the software is installed. Other language locale files are loaded from the filesystem at runtime.
88

99
### Source Repository Structure
1010

0 commit comments

Comments
 (0)