Skip to content

Commit 4e311ed

Browse files
authored
Merge pull request #8423 from sylvestre/en-embedded
l10n: embedded english strings + test the whole thing in github
2 parents 3b4b769 + df2fca5 commit 4e311ed

File tree

10 files changed

+1498
-80
lines changed

10 files changed

+1498
-80
lines changed

.github/workflows/l10n.yml

Lines changed: 1098 additions & 0 deletions
Large diffs are not rendered by default.

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,7 @@ digest = "0.10.7"
378378

379379
# Fluent dependencies
380380
fluent = "0.17.0"
381+
fluent-bundle = "0.16.0"
381382
unic-langid = "0.9.6"
382383
fluent-syntax = "0.12.0"
383384

GNUmakefile

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -418,25 +418,29 @@ endif
418418

419419
ifeq ($(LOCALES),y)
420420
locales:
421-
$(foreach prog, $(INSTALLEES), \
422-
if [ -d "$(BASEDIR)/src/uu/$(prog)/locales" ]; then \
423-
mkdir -p "$(BUILDDIR)/locales/$(prog)"; \
424-
for locale_file in "$(BASEDIR)"/src/uu/$(prog)/locales/*.ftl; do \
425-
$(INSTALL) -v "$$locale_file" "$(BUILDDIR)/locales/$(prog)/"; \
421+
@for prog in $(INSTALLEES); do \
422+
if [ -d "$(BASEDIR)/src/uu/$$prog/locales" ]; then \
423+
mkdir -p "$(BUILDDIR)/locales/$$prog"; \
424+
for locale_file in "$(BASEDIR)"/src/uu/$$prog/locales/*.ftl; do \
425+
if [ "$$(basename "$$locale_file")" != "en-US.ftl" ]; then \
426+
$(INSTALL) -v "$$locale_file" "$(BUILDDIR)/locales/$$prog/"; \
427+
fi; \
426428
done; \
427-
fi $(newline) \
428-
)
429+
fi; \
430+
done
429431

430432

431433
install-locales:
432-
$(foreach prog, $(INSTALLEES), \
433-
if [ -d "$(BASEDIR)/src/uu/$(prog)/locales" ]; then \
434-
mkdir -p "$(DESTDIR)$(DATAROOTDIR)/locales/$(prog)"; \
435-
for locale_file in "$(BASEDIR)"/src/uu/$(prog)/locales/*.ftl; do \
436-
$(INSTALL) -v "$$locale_file" "$(DESTDIR)$(DATAROOTDIR)/locales/$(prog)/"; \
434+
@for prog in $(INSTALLEES); do \
435+
if [ -d "$(BASEDIR)/src/uu/$$prog/locales" ]; then \
436+
mkdir -p "$(DESTDIR)$(DATAROOTDIR)/locales/$$prog"; \
437+
for locale_file in "$(BASEDIR)"/src/uu/$$prog/locales/*.ftl; do \
438+
if [ "$$(basename "$$locale_file")" != "en-US.ftl" ]; then \
439+
$(INSTALL) -v "$$locale_file" "$(DESTDIR)$(DATAROOTDIR)/locales/$$prog/"; \
440+
fi; \
437441
done; \
438-
fi $(newline) \
439-
)
442+
fi; \
443+
done
440444
else
441445
install-locales:
442446
endif

docs/src/l10n.md

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
This guide explains how localization (L10n) is implemented in the **Rust-based coreutils project**, detailing the use of [Fluent](https://projectfluent.org/) files, runtime behavior, and developer integration.
44

5+
## 🏗️ Architecture Overview
6+
7+
**English (US) locale files (`en-US.ftl`) are embedded directly in the binary**, ensuring that English always works regardless of how the software is installed. Other language locale files are loaded from the filesystem at runtime.
8+
9+
### Source Repository Structure
10+
11+
- **Main repository**: Contains English (`en-US.ftl`) locale files embedded in binaries
12+
- **Translation repository**: [uutils/coreutils-l10n](https://github.com/uutils/coreutils-l10n) contains all other language translations
13+
514
---
615

716
## 📁 Fluent File Layout
@@ -15,8 +24,8 @@ Each utility has its own set of translation files under:
1524
Examples:
1625

1726
```
18-
src/uu/ls/locales/en-US.ftl
19-
src/uu/ls/locales/fr-FR.ftl
27+
src/uu/ls/locales/en-US.ftl # Embedded in binary
28+
src/uu/ls/locales/fr-FR.ftl # Loaded from filesystem
2029
```
2130

2231
These files follow Fluent syntax and contain localized message patterns.
@@ -31,12 +40,11 @@ Localization must be explicitly initialized at runtime using:
3140
setup_localization(path)
3241
```
3342

34-
3543
This is typically done:
3644
- In `src/bin/coreutils.rs` for **multi-call binaries**
3745
- In `src/uucore/src/lib.rs` for **single-call utilities**
3846

39-
The string parameter determines the lookup path for Fluent files.
47+
The string parameter determines the lookup path for Fluent files. **English always works** because it's embedded, but other languages need their `.ftl` files to be available at runtime.
4048

4149
---
4250

@@ -155,9 +163,13 @@ In release mode, **paths are resolved relative to the executable**:
155163

156164
```
157165
<executable_dir>/locales/<utility>/
166+
<prefix>/share/locales/<utility>/
167+
~/.local/share/coreutils/locales/<utility>/
168+
~/.cargo/share/coreutils/locales/<utility>/
169+
/usr/share/coreutils/locales/<utility>/
158170
```
159171

160-
If both fallback paths fail, an error is returned during `setup_localization()`.
172+
If external locale files aren't found, the system falls back to embedded English locales.
161173

162174
---
163175

@@ -184,3 +196,15 @@ Fluent default (disabled here):
184196
```
185197
"\u{2068}Alice\u{2069}"
186198
```
199+
200+
---
201+
202+
## 🔧 Embedded English Locales
203+
204+
English locale files are always embedded directly in the binary during the build process. This ensures that:
205+
206+
- **English always works** regardless of installation method (e.g., `cargo install`)
207+
- **No runtime dependency** on external `.ftl` files for English
208+
- **Fallback behavior** when other language files are missing
209+
210+
The embedded English locales are generated at build time and included in the binary, providing a reliable fallback while still supporting full localization for other languages when their `.ftl` files are available.

fuzz/Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/uucore/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,11 @@ icu_decimal = { workspace = true, optional = true, features = [
7171
icu_locale = { workspace = true, optional = true, features = ["compiled_data"] }
7272
icu_provider = { workspace = true, optional = true }
7373

74-
# Fluent dependencies
74+
# Fluent dependencies (always available for localization)
7575
fluent = { workspace = true }
7676
fluent-syntax = { workspace = true }
7777
unic-langid = { workspace = true }
78+
fluent-bundle = { workspace = true }
7879
thiserror = { workspace = true }
7980
[target.'cfg(unix)'.dependencies]
8081
walkdir = { workspace = true, optional = true }

src/uucore/build.rs

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// This file is part of the uutils coreutils package.
2+
//
3+
// For the full copyright and license information, please view the LICENSE
4+
// file that was distributed with this source code.
5+
6+
use std::env;
7+
use std::fs::File;
8+
use std::io::Write;
9+
use std::path::Path;
10+
11+
pub fn main() -> Result<(), Box<dyn std::error::Error>> {
12+
let out_dir = env::var("OUT_DIR")?;
13+
14+
let mut embedded_file = File::create(Path::new(&out_dir).join("embedded_locales.rs"))?;
15+
16+
writeln!(embedded_file, "// Generated at compile time - do not edit")?;
17+
writeln!(
18+
embedded_file,
19+
"// This file contains embedded English locale files"
20+
)?;
21+
writeln!(embedded_file)?;
22+
writeln!(embedded_file, "use std::collections::HashMap;")?;
23+
writeln!(embedded_file)?;
24+
25+
// Start the function that returns embedded locales
26+
writeln!(
27+
embedded_file,
28+
"pub fn get_embedded_locales() -> HashMap<&'static str, &'static str> {{"
29+
)?;
30+
writeln!(embedded_file, " let mut locales = HashMap::new();")?;
31+
writeln!(embedded_file)?;
32+
33+
// Try to detect if we're building for a specific utility by checking build configuration
34+
// This attempts to identify individual utility builds vs multicall binary builds
35+
let target_utility = detect_target_utility();
36+
37+
match target_utility {
38+
Some(util_name) => {
39+
// Embed only the specific utility's locale (cat.ftl for cat for example)
40+
embed_single_utility_locale(&mut embedded_file, &project_root()?, &util_name)?;
41+
}
42+
None => {
43+
// Embed all utilities locales (multicall binary or fallback)
44+
embed_all_utilities_locales(&mut embedded_file, &project_root()?)?;
45+
}
46+
}
47+
48+
writeln!(embedded_file)?;
49+
writeln!(embedded_file, " locales")?;
50+
writeln!(embedded_file, "}}")?;
51+
52+
embedded_file.flush()?;
53+
Ok(())
54+
}
55+
56+
/// Get the project root directory
57+
fn project_root() -> Result<std::path::PathBuf, Box<dyn std::error::Error>> {
58+
let manifest_dir = env::var("CARGO_MANIFEST_DIR")?;
59+
let uucore_path = std::path::Path::new(&manifest_dir);
60+
61+
// Navigate from src/uucore to project root
62+
let project_root = uucore_path
63+
.parent() // src/
64+
.and_then(|p| p.parent()) // project root
65+
.ok_or("Could not determine project root")?;
66+
67+
Ok(project_root.to_path_buf())
68+
}
69+
70+
/// Attempt to detect which specific utility is being built
71+
fn detect_target_utility() -> Option<String> {
72+
use std::fs;
73+
74+
// First check if an explicit environment variable was set
75+
if let Ok(target_util) = env::var("UUCORE_TARGET_UTIL") {
76+
if !target_util.is_empty() {
77+
return Some(target_util);
78+
}
79+
}
80+
81+
// Check for a build configuration file in the target directory
82+
if let Ok(target_dir) = env::var("CARGO_TARGET_DIR") {
83+
let config_path = std::path::Path::new(&target_dir).join("uucore_target_util.txt");
84+
if let Ok(content) = fs::read_to_string(&config_path) {
85+
let util_name = content.trim();
86+
if !util_name.is_empty() && util_name != "multicall" {
87+
return Some(util_name.to_string());
88+
}
89+
}
90+
}
91+
92+
// Fallback: Check the default target directory
93+
if let Ok(project_root) = project_root() {
94+
let config_path = project_root.join("target/uucore_target_util.txt");
95+
if let Ok(content) = fs::read_to_string(&config_path) {
96+
let util_name = content.trim();
97+
if !util_name.is_empty() && util_name != "multicall" {
98+
return Some(util_name.to_string());
99+
}
100+
}
101+
}
102+
103+
// If no configuration found, assume multicall build
104+
None
105+
}
106+
107+
/// Embed locale for a single specific utility
108+
fn embed_single_utility_locale(
109+
embedded_file: &mut std::fs::File,
110+
project_root: &Path,
111+
util_name: &str,
112+
) -> Result<(), Box<dyn std::error::Error>> {
113+
use std::fs;
114+
115+
// Embed the specific utility's locale
116+
let locale_path = project_root
117+
.join("src/uu")
118+
.join(util_name)
119+
.join("locales/en-US.ftl");
120+
121+
if locale_path.exists() {
122+
let content = fs::read_to_string(&locale_path)?;
123+
writeln!(embedded_file, " // Locale for {util_name}")?;
124+
writeln!(
125+
embedded_file,
126+
" locales.insert(\"{util_name}/en-US.ftl\", r###\"{content}\"###);"
127+
)?;
128+
writeln!(embedded_file)?;
129+
130+
// Tell Cargo to rerun if this file changes
131+
println!("cargo:rerun-if-changed={}", locale_path.display());
132+
}
133+
134+
// Always embed uucore locale file if it exists
135+
let uucore_locale_path = project_root.join("src/uucore/locales/en-US.ftl");
136+
if uucore_locale_path.exists() {
137+
let content = fs::read_to_string(&uucore_locale_path)?;
138+
writeln!(embedded_file, " // Common uucore locale")?;
139+
writeln!(
140+
embedded_file,
141+
" locales.insert(\"uucore/en-US.ftl\", r###\"{content}\"###);"
142+
)?;
143+
println!("cargo:rerun-if-changed={}", uucore_locale_path.display());
144+
}
145+
146+
Ok(())
147+
}
148+
149+
/// Embed locale files for all utilities (multicall binary)
150+
fn embed_all_utilities_locales(
151+
embedded_file: &mut std::fs::File,
152+
project_root: &Path,
153+
) -> Result<(), Box<dyn std::error::Error>> {
154+
use std::fs;
155+
156+
// Discover all uu_* directories
157+
let src_uu_dir = project_root.join("src/uu");
158+
if !src_uu_dir.exists() {
159+
return Ok(());
160+
}
161+
162+
let mut util_dirs = Vec::new();
163+
for entry in fs::read_dir(&src_uu_dir)? {
164+
let entry = entry?;
165+
if entry.file_type()?.is_dir() {
166+
if let Some(dir_name) = entry.file_name().to_str() {
167+
util_dirs.push(dir_name.to_string());
168+
}
169+
}
170+
}
171+
util_dirs.sort();
172+
173+
// Embed locale files for each utility
174+
for util_name in &util_dirs {
175+
let locale_path = src_uu_dir.join(util_name).join("locales/en-US.ftl");
176+
if locale_path.exists() {
177+
let content = fs::read_to_string(&locale_path)?;
178+
writeln!(embedded_file, " // Locale for {util_name}")?;
179+
writeln!(
180+
embedded_file,
181+
" locales.insert(\"{util_name}/en-US.ftl\", r###\"{content}\"###);"
182+
)?;
183+
writeln!(embedded_file)?;
184+
185+
// Tell Cargo to rerun if this file changes
186+
println!("cargo:rerun-if-changed={}", locale_path.display());
187+
}
188+
}
189+
190+
// Also embed uucore locale file if it exists
191+
let uucore_locale_path = project_root.join("src/uucore/locales/en-US.ftl");
192+
if uucore_locale_path.exists() {
193+
let content = fs::read_to_string(&uucore_locale_path)?;
194+
writeln!(embedded_file, " // Common uucore locale")?;
195+
writeln!(
196+
embedded_file,
197+
" locales.insert(\"uucore/en-US.ftl\", r###\"{content}\"###);"
198+
)?;
199+
println!("cargo:rerun-if-changed={}", uucore_locale_path.display());
200+
}
201+
202+
embedded_file.flush()?;
203+
Ok(())
204+
}

src/uucore/src/lib/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ macro_rules! bin {
185185
uucore::locale::LocalizationError::ParseResource {
186186
error: err_msg,
187187
snippet,
188-
} => eprintln!("Localization parse error at {snippet}: {err_msg}"),
188+
} => eprintln!("Localization parse error at {snippet}: {err_msg:?}"),
189189
other => eprintln!("Could not init the localization system: {other}"),
190190
}
191191
std::process::exit(99)

0 commit comments

Comments
 (0)