Use ICU4X built-in data (#482)

The data that is currently being generated in `parley_data` is the same
data that ICU4X ships with. However, using `try_new_unstable`
constructors with custom data providers can be less efficient than
enabling the `compiled_data` feature, as these constructors do runtime
lookups and branching, whereas most `compiled_data` constructors are
`const`.

Benchmarks look neutral:

```
Default Style - arabic 20 characters               [   9.9 us ...   9.7 us ]      -1.46%*
Default Style - latin 20 characters                [   4.5 us ...   4.3 us ]      -4.27%*
Default Style - japanese 20 characters             [   9.1 us ...   8.9 us ]      -2.30%*
Default Style - arabic 1 paragraph                 [  55.5 us ...  55.6 us ]      +0.13%
Default Style - latin 1 paragraph                  [  18.2 us ...  17.9 us ]      -1.49%*
Default Style - japanese 1 paragraph               [  76.8 us ...  76.9 us ]      +0.16%
Default Style - arabic 4 paragraph                 [ 234.0 us ... 235.1 us ]      +0.48%
Default Style - latin 4 paragraph                  [  69.0 us ...  68.2 us ]      -1.05%*
Default Style - japanese 4 paragraph               [ 131.9 us ... 136.0 us ]      +3.11%
Styled - arabic 20 characters                      [  11.3 us ...  11.3 us ]      -0.43%
Styled - latin 20 characters                       [   6.3 us ...   6.3 us ]      -0.99%
Styled - japanese 20 characters                    [   9.9 us ...   9.7 us ]      -1.80%*
Styled - arabic 1 paragraph                        [  59.4 us ...  58.5 us ]      -1.40%
Styled - latin 1 paragraph                         [  23.7 us ...  23.3 us ]      -1.82%*
Styled - japanese 1 paragraph                      [  86.6 us ...  87.5 us ]      +1.05%*
Styled - arabic 4 paragraph                        [ 251.7 us ... 252.5 us ]      +0.32%
Styled - latin 4 paragraph                         [  90.4 us ...  89.1 us ]      -1.45%*
Styled - japanese 4 paragraph                      [ 123.7 us ... 124.0 us ]      +0.24%
```

---------

Co-authored-by: Taj Pereira <taj@canva.com>

Author: robertbastian

Cargo.lock

@@ -39,15 +39,10 @@ hashbrown = { version = "0.16.1", default-features = false, features = [
 ] }
 icu_codepointtrie_builder = { version = "0.5.1", default-features = false, features = ["wasm"] }
 icu_collections = { version = "2.1.1", default-features = false }
-icu_locale = { version = "2.1.1", default-features = false }
 icu_locale_core = { version = "2.1.1", default-features = false }
-icu_normalizer = { version = "~2.1.1", default-features = false }
-icu_properties = { version = "~2.1.2", default-features = false }
-icu_provider = { version = "2.1.1", default-features = false }
-icu_provider_adapters = { version = "2.1.1", default-features = false }
-icu_provider_export = { version = "2.1.1", default-features = false }
-icu_provider_source = { version = "2.1.1", default-features = false }
-icu_segmenter = { version = "~2.1.1", default-features = false }
+icu_normalizer = { version = "2.1.1", default-features = false }
+icu_properties = { version = "2.1.2", default-features = false }
+icu_segmenter = { version = "2.1.2", default-features = false }
 linebender_resource_handle = { version = "0.1.1", default-features = false }
 parley = { version = "0.7.0", default-features = false, path = "parley" }
 parley_data = { path = "parley_data", default-features = false }

Cargo.toml

@@ -34,13 +34,9 @@ parley_data = { workspace = true, features = ["baked"] }
 accesskit = { workspace = true, optional = true }
 hashbrown = { workspace = true }
 harfrust = { workspace = true }
-icu_collections = { workspace = true }
-icu_normalizer = { workspace = true }
-icu_properties = { workspace = true }
-icu_provider = { workspace = true }
-icu_segmenter = { workspace = true, features = ["auto"] }
-# Used in ICU4X baked data sources
-zerovec = { workspace = true }
+icu_normalizer = { workspace = true, features = ["compiled_data"] }
+icu_properties = { workspace = true, features = ["compiled_data"] }
+icu_segmenter = { workspace = true, features = ["compiled_data"] }
 
 [dev-dependencies]
 parley_dev = { workspace = true }

parley/Cargo.toml

@@ -117,7 +117,7 @@ impl CharCluster {
 
     #[inline(always)]
     fn contributes_to_shaping(ch: char, analysis_data_sources: &AnalysisDataSources) -> bool {
-        let props = analysis_data_sources.composite.properties(ch as u32);
+        let props = analysis_data_sources.properties(ch);
         crate::analysis::contributes_to_shaping(props.general_category(), props.script())
     }
 

parley/src/analysis/cluster.rs

@@ -2,12 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 pub(crate) mod cluster;
-mod provider;
 
 use alloc::vec::Vec;
 use core::marker::PhantomData;
 
-use crate::analysis::provider::PROVIDER;
 use crate::resolve::{RangedStyle, ResolvedStyle};
 use crate::{Brush, LayoutContext, WordBreak};
 
@@ -19,107 +17,80 @@ use icu_properties::props::{BidiMirroringGlyph, GeneralCategory, GraphemeCluster
 use icu_properties::{
     CodePointMapData, CodePointMapDataBorrowed, PropertyNamesShort, PropertyNamesShortBorrowed,
 };
-use icu_segmenter::options::{LineBreakOptions, LineBreakWordOption, WordBreakOptions};
+use icu_segmenter::options::{LineBreakOptions, LineBreakWordOption, WordBreakInvariantOptions};
 use icu_segmenter::{
     GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed, LineSegmenter,
     LineSegmenterBorrowed, WordSegmenter, WordSegmenterBorrowed,
 };
-use parley_data::CompositeProps;
-
-pub(crate) struct AnalysisDataSources {
-    grapheme_segmenter: GraphemeClusterSegmenter,
-    word_segmenter: WordSegmenter,
-    line_segmenters: LineSegmenters,
-    composing_normalizer: CanonicalComposition,
-    decomposing_normalizer: CanonicalDecomposition,
-    script_short_name: PropertyNamesShort<Script>,
-    brackets: CodePointMapData<BidiMirroringGlyph>,
-
-    composite: CompositeProps,
-}
-
-#[derive(Default)]
-struct LineSegmenters {
-    normal: Option<LineSegmenter>,
-    keep_all: Option<LineSegmenter>,
-    break_all: Option<LineSegmenter>,
-}
+use parley_data::Properties;
 
-impl LineSegmenters {
-    fn get(&mut self, word_break_strength: WordBreak) -> LineSegmenterBorrowed<'_> {
-        let segmenter = match word_break_strength {
-            WordBreak::Normal => &mut self.normal,
-            WordBreak::KeepAll => &mut self.keep_all,
-            WordBreak::BreakAll => &mut self.break_all,
-        };
-
-        segmenter
-            .get_or_insert_with(|| {
-                let mut line_break_opts = LineBreakOptions::default();
-                let word_break_strength_icu = match word_break_strength {
-                    WordBreak::Normal => LineBreakWordOption::Normal,
-                    WordBreak::BreakAll => LineBreakWordOption::BreakAll,
-                    WordBreak::KeepAll => LineBreakWordOption::KeepAll,
-                };
-                line_break_opts.word_option = Some(word_break_strength_icu);
-                LineSegmenter::try_new_auto_unstable(&PROVIDER, line_break_opts)
-                    .expect("Failed to create LineSegmenter")
-            })
-            .as_borrowed()
-    }
-}
+pub(crate) struct AnalysisDataSources;
 
 impl AnalysisDataSources {
     pub(crate) fn new() -> Self {
-        Self {
-            grapheme_segmenter: GraphemeClusterSegmenter::try_new_unstable(&PROVIDER).unwrap(),
-            word_segmenter: WordSegmenter::try_new_lstm_unstable(
-                &PROVIDER,
-                WordBreakOptions::default(),
-            )
-            .unwrap(),
-            line_segmenters: LineSegmenters::default(),
-            composing_normalizer: CanonicalComposition::try_new_unstable(&PROVIDER).unwrap(),
-            decomposing_normalizer: CanonicalDecomposition::try_new_unstable(&PROVIDER).unwrap(),
-            script_short_name: PropertyNamesShort::<Script>::try_new_unstable(&PROVIDER).unwrap(),
-            brackets: CodePointMapData::<BidiMirroringGlyph>::try_new_unstable(&PROVIDER).unwrap(),
-            composite: CompositeProps,
-        }
+        Self
     }
 
     #[inline(always)]
-    pub(crate) fn composite(&self) -> &CompositeProps {
-        &self.composite
+    pub(crate) fn properties(&self, c: char) -> Properties {
+        Properties::get(c)
     }
 
     #[inline(always)]
     pub(crate) fn grapheme_segmenter(&self) -> GraphemeClusterSegmenterBorrowed<'_> {
-        self.grapheme_segmenter.as_borrowed()
+        const { GraphemeClusterSegmenter::new() }
     }
 
     #[inline(always)]
-    fn word_segmenter(&self) -> WordSegmenterBorrowed<'_> {
-        self.word_segmenter.as_borrowed()
+    fn word_segmenter(&self) -> WordSegmenterBorrowed<'static> {
+        const { WordSegmenter::new_for_non_complex_scripts(WordBreakInvariantOptions::default()) }
+    }
+
+    #[inline(always)]
+    fn line_segmenter(&self, word_break_strength: WordBreak) -> LineSegmenterBorrowed<'static> {
+        match word_break_strength {
+            WordBreak::Normal => {
+                const {
+                    let mut opt = LineBreakOptions::default();
+                    opt.word_option = Some(LineBreakWordOption::Normal);
+                    LineSegmenter::new_for_non_complex_scripts(opt)
+                }
+            }
+            WordBreak::BreakAll => {
+                const {
+                    let mut opt = LineBreakOptions::default();
+                    opt.word_option = Some(LineBreakWordOption::BreakAll);
+                    LineSegmenter::new_for_non_complex_scripts(opt)
+                }
+            }
+            WordBreak::KeepAll => {
+                const {
+                    let mut opt = LineBreakOptions::default();
+                    opt.word_option = Some(LineBreakWordOption::KeepAll);
+                    LineSegmenter::new_for_non_complex_scripts(opt)
+                }
+            }
+        }
     }
 
     #[inline(always)]
     fn composing_normalizer(&self) -> CanonicalCompositionBorrowed<'_> {
-        self.composing_normalizer.as_borrowed()
+        const { CanonicalComposition::new() }
     }
 
     #[inline(always)]
     fn decomposing_normalizer(&self) -> CanonicalDecompositionBorrowed<'_> {
-        self.decomposing_normalizer.as_borrowed()
+        const { CanonicalDecomposition::new() }
     }
 
     #[inline(always)]
-    pub(crate) fn script_short_name(&self) -> PropertyNamesShortBorrowed<'_, Script> {
-        self.script_short_name.as_borrowed()
+    pub(crate) fn script_short_name(&self) -> PropertyNamesShortBorrowed<'static, Script> {
+        PropertyNamesShort::new()
     }
 
     #[inline(always)]
-    pub(crate) fn brackets(&self) -> CodePointMapDataBorrowed<'_, BidiMirroringGlyph> {
-        self.brackets.as_borrowed()
+    fn brackets(&self) -> CodePointMapDataBorrowed<'_, BidiMirroringGlyph> {
+        const { CodePointMapData::new() }
     }
 }
 
@@ -354,8 +325,7 @@ pub(crate) fn analyze_text<B: Brush>(lcx: &mut LayoutContext<B>, mut text: &str)
         if substring_index == 0 && last {
             let mut lb_iter = lcx
                 .analysis_data_sources
-                .line_segmenters
-                .get(word_break_strength)
+                .line_segmenter(word_break_strength)
                 .segment_str(substring);
 
             let _first = lb_iter.next();
@@ -378,8 +348,7 @@ pub(crate) fn analyze_text<B: Brush>(lcx: &mut LayoutContext<B>, mut text: &str)
 
         let line_boundaries_iter = lcx
             .analysis_data_sources
-            .line_segmenters
-            .get(word_break_strength)
+            .line_segmenter(word_break_strength)
             .segment_str(substring);
 
         let mut substring_chars = substring.chars();
@@ -455,7 +424,7 @@ pub(crate) fn analyze_text<B: Brush>(lcx: &mut LayoutContext<B>, mut text: &str)
         (boundary, ch)
     });
 
-    let composite = lcx.analysis_data_sources.composite();
+    let properties = |c| lcx.analysis_data_sources.properties(c);
 
     let mut needs_bidi_resolution = false;
 
@@ -466,7 +435,7 @@ pub(crate) fn analyze_text<B: Brush>(lcx: &mut LayoutContext<B>, mut text: &str)
         // character's index, but we need our iterators to align, and the rest are simply
         // character-indexed.
         .fold(false, |is_mandatory_linebreak, (boundary, ch)| {
-            let properties = composite.properties(ch as u32);
+            let properties = properties(ch);
             let script = properties.script();
             let grapheme_cluster_break = properties.grapheme_cluster_break();
             let bidi_class = properties.bidi_class();
@@ -496,7 +465,7 @@ pub(crate) fn analyze_text<B: Brush>(lcx: &mut LayoutContext<B>, mut text: &str)
             };
 
             needs_bidi_resolution |= crate::bidi::needs_bidi_resolution(bidi_class);
-            // TODO: maybe extend CompositeProps to u64 to fit BidiMirroringGlyph
+            // TODO: maybe extend Properties to u64 to fit BidiMirroringGlyph
             let bracket = lcx.analysis_data_sources.brackets().get(ch);
 
             lcx.info.push((

parley/src/analysis/mod.rs

@@ -10,12 +10,11 @@ publish = false
 
 [features]
 default = ["baked"]
-baked = ["dep:zerovec"]
-datagen = []
+baked = ["dep:icu_collections", "dep:zerovec"]
 
 [dependencies]
 icu_properties = { workspace = true }
-icu_collections = { workspace = true }
+icu_collections = { workspace = true, optional = true }
 zerovec = { workspace = true, optional = true }
 
 [lints]

parley/src/analysis/provider.rs

@@ -1,17 +1,14 @@
 # Unicode Data
 
-`parley_data` packages the Unicode data that Parley's text analysis and shaping pipeline needs at runtime. It exposes a locale-invariant `CompositePropsV1` provider backed by a compact `CodePointTrie`, allowing the engine to obtain all required character properties with a single lookup.
+`parley_data` packages the Unicode data that Parley's text analysis and shaping pipeline needs at runtime. It exposes a locale-invariant `CompositeProps` data backed by a compact `CodePointTrie`, allowing the engine to obtain all required character properties with a single lookup.
 
 ## What is included
 
-- `CompositePropsV1Data`, a trie that holds script, general category, grapheme cluster break, bidi class, and several emoji-related flags per scalar value.
-- Re-exported ICU4X data providers for grapheme, word, and line breaking, plus Unicode normalization tables used by Parley.
-- An implementation of `unicode_bidi::BidiDataSource`, making the composite provider plug directly into Unicode bidi processing.
+- `CompositeProps`, a trie that holds script, general category, grapheme cluster break, bidi class, and several emoji-related flags per scalar value.
 
 ## Cargo features
 
 - `baked` *(default)* embeds pre-generated ICU4X and composite data from `src/generated`, enabling use in `no_std` targets without a filesystem.
-- `datagen` enables serialization, `databake`, and ICU provider export traits so the crate can participate in regeneration workflows. This feature is intended for developer use when refreshing the baked data and principally used by `../parley_data_gen`.
 
 ## Regenerating the baked data
 
@@ -26,7 +23,7 @@ The generator downloads the latest ICU4X upstream data and recomputes the compos
 
 ## Why have this crate?
 
-You may wonder why we can't simply run `parley_data_gen` within a `build.rs` file of `Parley`. Although being possible, that option increases build time by over a minute and requires a `std` compatible environment.
+You may wonder why we can't simply run `parley_data_gen` within a `build.rs` file of `Parley`. Although being possible, that option increases build time and requires a `std` compatible environment.
 
 ## License