Bundle translations (#6661)

This currently doesn't have public API to enable it yet.

TODO:
 - Error handling in the compiler
 - Public API in the compiler configuration
 - Documentation

Author: ogoffart

api/cpp/include/slint.h

@@ -1265,6 +1265,39 @@ inline SharedString translate(const SharedString &original, const SharedString &
     return result;
 }
 
+inline SharedString translate_from_bundle(std::span<const char8_t *const> strs,
+                                          cbindgen_private::Slice<SharedString> arguments)
+{
+    SharedString result;
+    cbindgen_private::slint_translate_from_bundle(
+            cbindgen_private::Slice<const char *>(
+                    const_cast<char const **>(reinterpret_cast<char const *const *>(strs.data())),
+                    strs.size()),
+            arguments, &result);
+    return result;
+}
+inline SharedString
+translate_from_bundle_with_plural(std::span<const char8_t *const> strs,
+                                  std::span<const uint32_t> indices,
+                                  std::span<uintptr_t (*const)(int32_t)> plural_rules,
+                                  cbindgen_private::Slice<SharedString> arguments, int n)
+{
+    SharedString result;
+    cbindgen_private::Slice<const char *> strs_slice(
+            const_cast<char const **>(reinterpret_cast<char const *const *>(strs.data())),
+            strs.size());
+    cbindgen_private::Slice<uint32_t> indices_slice(
+            const_cast<uint32_t *>(reinterpret_cast<const uint32_t *>(indices.data())),
+            indices.size());
+    cbindgen_private::Slice<uintptr_t (*)(int32_t)> plural_rules_slice(
+            const_cast<uintptr_t (**)(int32_t)>(
+                    reinterpret_cast<uintptr_t (*const *)(int32_t)>(plural_rules.data())),
+            plural_rules.size());
+    cbindgen_private::slint_translate_from_bundle_with_plural(
+            strs_slice, indices_slice, plural_rules_slice, arguments, n, &result);
+    return result;
+}
+
 } // namespace private_api
 
 #ifdef SLINT_FEATURE_GETTEXT

api/rs/slint/private_unstable_api.rs

@@ -213,6 +213,9 @@ pub mod re_exports {
     };
     pub use i_slint_core::slice::Slice;
     pub use i_slint_core::timers::{Timer, TimerMode};
+    pub use i_slint_core::translations::{
+        set_language_internal, translate_from_bundle, translate_from_bundle_with_plural,
+    };
     pub use i_slint_core::window::{
         InputMethodRequest, WindowAdapter, WindowAdapterRc, WindowInner,
     };

internal/compiler/Cargo.toml

@@ -29,9 +29,11 @@ display-diagnostics = ["codemap", "codemap-diagnostic"]
 
 # Enabled the support to render images and font in the binary
 software-renderer = ["image", "dep:resvg", "fontdue", "i-slint-common/shared-fontdb", "dep:rayon"]
-
 embed-glyphs-as-sdf = ["dep:fdsm", "dep:ttf-parser-fdsm", "dep:nalgebra", "dep:image-fdsm", "dep:rayon"]
 
+# Translation bundler
+bundle-translations = ["dep:polib"]
+
 default = []
 
 [dependencies]
@@ -65,6 +67,8 @@ ttf-parser-fdsm = { package = "ttf-parser", version = "0.24.1", optional = true
 image-fdsm = { package = "image", version = "0.25", optional = true, default-features = false }
 nalgebra = { version = "0.33.0", optional = true }
 rayon = { workspace = true, optional = true }
+# translations
+polib = { version = "0.2", optional = true }
 
 [dev-dependencies]
 i-slint-parser-test-macro = { path = "./parser-test-macro" }

internal/compiler/generator/cpp.rs

@@ -710,6 +710,11 @@ pub fn generate(
 
     let llr = llr::lower_to_item_tree::lower_to_item_tree(&doc, compiler_config);
 
+    #[cfg(feature = "bundle-translations")]
+    if let Some(translations) = &llr.translations {
+        generate_translation(translations, &llr, &mut file.resources);
+    }
+
     // Forward-declare the root so that sub-components can access singletons, the window, etc.
     file.declarations.extend(
         llr.public_components
@@ -3369,6 +3374,16 @@ fn compile_expression(expr: &llr::Expression, ctx: &EvaluationContext) -> String
             )
         }
         Expression::EmptyComponentFactory => panic!("component-factory not yet supported in C++"),
+        Expression::TranslationReference { format_args, string_index, plural } => {
+            let args = compile_expression(format_args, ctx);
+            match plural {
+                Some(plural) => {
+                    let plural = compile_expression(plural, ctx);
+                    format!("slint::private_api::translate_from_bundle_with_plural(slint_translation_bundle_plural_{string_index}_str, slint_translation_bundle_plural_{string_index}_idx,  slint_translated_plural_rules, {args}, {plural})")
+                }
+                None => format!("slint::private_api::translate_from_bundle(slint_translation_bundle_{string_index}, {args})"),
+            }
+        },
     }
 }
 
@@ -3828,3 +3843,106 @@ fn generate_type_aliases(file: &mut File, doc: &Document) {
 
     file.declarations.extend(type_aliases);
 }
+
+#[cfg(feature = "bundle-translations")]
+fn generate_translation(
+    translations: &llr::translations::Translations,
+    compilation_unit: &llr::CompilationUnit,
+    declarations: &mut Vec<Declaration>,
+) {
+    for (idx, m) in translations.strings.iter().enumerate() {
+        declarations.push(Declaration::Var(Var {
+            ty: "const char8_t* const".into(),
+            name: format_smolstr!("slint_translation_bundle_{idx}"),
+            array_size: Some(m.len()),
+            init: Some(format!(
+                "{{ {} }}",
+                m.iter()
+                    .map(|s| match s {
+                        Some(s) => format_smolstr!("u8\"{}\"", escape_string(s.as_str())),
+                        None => "nullptr".into(),
+                    })
+                    .join(", ")
+            )),
+            ..Default::default()
+        }));
+    }
+    for (idx, ms) in translations.plurals.iter().enumerate() {
+        let all_strs = ms.iter().flatten().flatten();
+        let all_strs_len = all_strs.clone().count();
+        declarations.push(Declaration::Var(Var {
+            ty: "const char8_t* const".into(),
+            name: format_smolstr!("slint_translation_bundle_plural_{}_str", idx),
+            array_size: Some(all_strs_len),
+            init: Some(format!(
+                "{{ {} }}",
+                all_strs.map(|s| format_smolstr!("u8\"{}\"", escape_string(s.as_str()))).join(", ")
+            )),
+            ..Default::default()
+        }));
+
+        let mut count = 0;
+        declarations.push(Declaration::Var(Var {
+            ty: "const uint32_t".into(),
+            name: format_smolstr!("slint_translation_bundle_plural_{}_idx", idx),
+            array_size: Some(ms.len()),
+            init: Some(format!(
+                "{{ {} }}",
+                ms.iter()
+                    .map(|x| {
+                        count += x.as_ref().map_or(0, |x| x.len());
+                        count
+                    })
+                    .join(", ")
+            )),
+            ..Default::default()
+        }));
+    }
+    let lang_len = translations.languages.len();
+    declarations.push(Declaration::Function(Function {
+        name: "slint_set_language".into(),
+        signature: "(std::string_view lang)".into(),
+        is_inline: true,
+        statements: Some(vec![
+            format!("std::array<slint::cbindgen_private::Slice<uint8_t>, {lang_len}> languages {{ {} }};", translations.languages.iter().map(|l| format!("slint::private_api::string_to_slice({l:?})")).join(", ")),
+            format!("return slint::cbindgen_private::slint_translate_set_language(slint::private_api::string_to_slice(lang), {{ languages.data(), {lang_len} }});"
+        )]),
+        ..Default::default()
+    }));
+
+    let ctx = EvaluationContext {
+        compilation_unit,
+        current_sub_component: None,
+        current_global: None,
+        generator_state: CppGeneratorContext {
+            global_access: "\n#error \"language rule can't access state\";".into(),
+            conditional_includes: &Default::default(),
+        },
+        parent: None,
+        argument_types: &[Type::Int32],
+    };
+    declarations.push(Declaration::Var(Var {
+        ty: format_smolstr!(
+            "const std::array<uintptr_t (*const)(int32_t), {}>",
+            translations.plural_rules.len()
+        ),
+        name: "slint_translated_plural_rules".into(),
+        init: Some(format!(
+            "{{ {} }}",
+            translations
+                .plural_rules
+                .iter()
+                .map(|s| match s {
+                    Some(s) => {
+                        format!(
+                            "[]([[maybe_unused]] int32_t arg_0) -> uintptr_t {{ return {}; }}",
+                            compile_expression(s, &ctx)
+                        )
+                    }
+                    None => "nullptr".into(),
+                })
+                .join(", ")
+        )),
+        ..Default::default()
+    }));
+}

internal/compiler/generator/rust.rs

@@ -210,6 +210,15 @@ pub fn generate(doc: &Document, compiler_config: &CompilerConfiguration) -> Toke
         .map(|c| format_ident!("slint_generated{}", ident(&c.id)))
         .unwrap_or_else(|| format_ident!("slint_generated"));
 
+    #[cfg(not(feature = "bundle-translations"))]
+    let (translations, exported_tr) = (quote!(), quote!());
+    #[cfg(feature = "bundle-translations")]
+    let (translations, exported_tr) = llr
+        .translations
+        .as_ref()
+        .map(|t| (generate_translations(t, &llr), quote!(slint_set_language,)))
+        .unzip();
+
     quote! {
         #[allow(non_snake_case, non_camel_case_types)]
         #[allow(unused_braces, unused_parens)]
@@ -224,10 +233,11 @@ pub fn generate(doc: &Document, compiler_config: &CompilerConfiguration) -> Toke
             #(#public_components)*
             #shared_globals
             #(#resource_symbols)*
+            #translations
             const _THE_SAME_VERSION_MUST_BE_USED_FOR_THE_COMPILER_AND_THE_RUNTIME : slint::#version_check = slint::#version_check;
         }
         #[allow(unused_imports)]
-        pub use #generated_mod::{#(#compo_ids,)* #(#structs_and_enums_ids,)* #(#globals_ids,)* #(#named_exports,)*};
+        pub use #generated_mod::{#(#compo_ids,)* #(#structs_and_enums_ids,)* #(#globals_ids,)* #(#named_exports,)* #exported_tr};
         #[allow(unused_imports)]
         pub use slint::{ComponentHandle as _, Global as _, ModelExt as _};
     }
@@ -2574,6 +2584,22 @@ fn compile_expression(expr: &Expression, ctx: &EvaluationContext) -> TokenStream
             }
         }
         Expression::EmptyComponentFactory => quote!(slint::ComponentFactory::default()),
+        Expression::TranslationReference { format_args, string_index, plural } => {
+            let args = compile_expression(format_args, ctx);
+            match plural {
+                Some(plural) => {
+                    let plural = compile_expression(plural, ctx);
+                    quote!(sp::translate_from_bundle_with_plural(
+                        &self::_SLINT_TRANSLATED_STRINGS_PLURALS[#string_index],
+                        &self::_SLINT_TRANSLATED_PLURAL_RULES,
+                        sp::Slice::<sp::SharedString>::from(#args).as_slice(),
+                        #plural as _
+                    ))
+                }
+                None => quote!(sp::translate_from_bundle(&self::_SLINT_TRANSLATED_STRINGS[#string_index], sp::Slice::<sp::SharedString>::from(#args).as_slice())),
+
+            }
+        },
     }
 }
 
@@ -3138,3 +3164,59 @@ fn generate_named_exports(doc: &Document) -> Vec<TokenStream> {
         })
         .collect::<Vec<_>>()
 }
+
+#[cfg(feature = "bundle-translations")]
+fn generate_translations(
+    translations: &llr::translations::Translations,
+    compilation_unit: &llr::CompilationUnit,
+) -> TokenStream {
+    let strings = translations.strings.iter().map(|strings| {
+        let array = strings.iter().map(|s| match s.as_ref().map(SmolStr::as_str) {
+            Some(s) => quote!(Some(#s)),
+            None => quote!(None),
+        });
+        quote!(&[#(#array),*])
+    });
+    let plurals = translations.plurals.iter().map(|plurals| {
+        let array = plurals.iter().map(|p| match p {
+            Some(p) => {
+                let p = p.iter().map(SmolStr::as_str);
+                quote!(Some(&[#(#p),*]))
+            }
+            None => quote!(None),
+        });
+        quote!(&[#(#array),*])
+    });
+
+    let ctx = EvaluationContext {
+        compilation_unit,
+        current_sub_component: None,
+        current_global: None,
+        generator_state: RustGeneratorContext {
+            global_access: quote!(compile_error!("language rule can't access state")),
+        },
+        parent: None,
+        argument_types: &[Type::Int32],
+    };
+    let rules = translations.plural_rules.iter().map(|rule| {
+        let rule = match rule {
+            Some(rule) => {
+                let rule = compile_expression(rule, &ctx);
+                quote!(Some(|arg: i32| { let args = (arg,); (#rule) as usize } ))
+            }
+            None => quote!(None),
+        };
+        quote!(#rule)
+    });
+    let lang = translations.languages.iter().map(SmolStr::as_str).map(|lang| quote!(#lang));
+
+    quote!(
+        const _SLINT_TRANSLATED_STRINGS: &[&[sp::Option<&str>]] = &[#(#strings),*];
+        const _SLINT_TRANSLATED_STRINGS_PLURALS: &[&[sp::Option<&[&str]>]] = &[#(#plurals),*];
+        #[allow(unused)]
+        const _SLINT_TRANSLATED_PLURAL_RULES: &[sp::Option<fn(i32) -> usize>] = &[#(#rules),*];
+        pub fn slint_set_language(lang: &str) -> bool {
+            sp::set_language_internal(lang, &[#(#lang),*])
+        }
+    )
+}

internal/compiler/lib.rs

@@ -132,6 +132,9 @@ pub struct CompilerConfiguration {
 
     /// The domain used as one of the parameter to the translate function
     pub translation_domain: Option<String>,
+    /// When Some, this is the path where the translations are looked at to bundle the translations
+    #[cfg(feature = "bundle-translations")]
+    pub translation_path_bundle: Option<std::path::PathBuf>,
 
     /// C++ namespace
     pub cpp_namespace: Option<String>,
@@ -220,6 +223,10 @@ impl CompilerConfiguration {
             components_to_generate: ComponentSelection::ExportedWindows,
             #[cfg(feature = "software-renderer")]
             font_cache: Default::default(),
+            #[cfg(feature = "bundle-translations")]
+            translation_path_bundle: std::env::var("SLINT_BUNDLE_TRANSLATIONS")
+                .ok()
+                .map(|x| x.into()),
         }
     }
 

internal/compiler/llr.rs

@@ -10,6 +10,8 @@ pub use item_tree::*;
 pub mod lower_expression;
 pub mod lower_to_item_tree;
 pub mod pretty_print;
+#[cfg(feature = "bundle-translations")]
+pub mod translations;
 
 /// The optimization passes over the LLR
 pub mod optim_passes {

internal/compiler/llr/expression.rs

@@ -190,6 +190,15 @@ pub enum Expression {
     },
 
     EmptyComponentFactory,
+
+    /// A reference to bundled translated string
+    TranslationReference {
+        /// An expression of type array of strings
+        format_args: Box<Expression>,
+        string_index: usize,
+        /// The `n` value to use for the plural form if it is a plural form
+        plural: Option<Box<Expression>>,
+    },
 }
 
 impl Expression {
@@ -306,6 +315,7 @@ impl Expression {
             }
             Self::MinMax { ty, .. } => ty.clone(),
             Self::EmptyComponentFactory => Type::ComponentFactory,
+            Self::TranslationReference { .. } => Type::String,
         }
     }
 }
@@ -388,6 +398,12 @@ macro_rules! visit_impl {
                 $visitor(rhs);
             }
             Expression::EmptyComponentFactory => {}
+            Expression::TranslationReference { format_args, plural, string_index: _ } => {
+                $visitor(format_args);
+                if let Some(plural) = plural {
+                    $visitor(plural);
+                }
+            }
         }
     };
 }

internal/compiler/llr/item_tree.rs

@@ -368,6 +368,8 @@ pub struct CompilationUnit {
     pub sub_components: Vec<Rc<SubComponent>>,
     pub globals: Vec<GlobalComponent>,
     pub has_debug_info: bool,
+    #[cfg(feature = "bundle-translations")]
+    pub translations: Option<super::translations::Translations>,
 }
 
 impl CompilationUnit {