Internal datagen API: Return the distance from skeleton matching (#6485)

Author: sffc

components/datetime/src/provider/pattern/hour_cycle.rs

@@ -104,8 +104,8 @@ impl CoarseHourCycle {
             // requested fields.
             true,
         ) {
-            skeleton::BestSkeleton::AllFieldsMatch(patterns)
-            | skeleton::BestSkeleton::MissingOrExtraFields(patterns) => {
+            skeleton::BestSkeleton::AllFieldsMatch(patterns, _)
+            | skeleton::BestSkeleton::MissingOrExtraFields(patterns, _) => {
                 Some(reference::Pattern::from(&patterns.expect_pattern(
                     "Only week-of patterns have plural variants",
                 )))

components/datetime/src/provider/skeleton/helpers.rs

@@ -60,38 +60,62 @@ const NO_DISTANCE: u32 = 0;
 //   MM ≅ M      (09 ≅ 9)
 const WIDTH_MISMATCH_DISTANCE: u32 = 1;
 
+// If a glue pattern is required, give a small penalty.
+const GLUE_DISTANCE: u32 = 10;
+
 // C. Numeric and text fields are given a larger distance from each other.
 // - MMM ≈ MM    (Sep ≈ 09)
 //   MMM
-const TEXT_VS_NUMERIC_DISTANCE: u32 = 10;
+const TEXT_VS_NUMERIC_DISTANCE: u32 = 100;
 
 // D. Symbols representing substantial differences (week of year vs week of month) are given much
 // larger a distances from each other.
 // - d ≋ D;     (12 ≋ 345) Day of month vs Day of year
-const SUBSTANTIAL_DIFFERENCES_DISTANCE: u32 = 100;
+const SUBSTANTIAL_DIFFERENCES_DISTANCE: u32 = 1000;
 
 // A skeleton had more symbols than what was requested.
-const SKELETON_EXTRA_SYMBOL: u32 = 1000;
+const SKELETON_EXTRA_SYMBOL: u32 = 10000;
 
 // A requested symbol is missing in the skeleton. Note that this final value can be more than
 // MAX_SKELETON_FIELDS, as it's counting the missing requested fields, which can be longer than
 // the stored skeletons. There cannot be any cases higher than this one.
-const REQUESTED_SYMBOL_MISSING: u32 = 10000;
+const REQUESTED_SYMBOL_MISSING: u32 = 100000;
 
 /// The best skeleton found, alongside information on how well it matches.
 ///
 /// According to the [UTS 35 skeleton matching algorithm](https://unicode.org/reports/tr35/tr35-dates.html#Matching_Skeletons)
 /// there will be a guaranteed match for a skeleton. However, with this initial implementation,
 /// there is no attempt to add on missing fields. This enum encodes the variants for the current
 /// search for a best skeleton.
+///
+/// The patterns are paired with a measure of their quality.
 #[derive(Debug, PartialEq, Clone)]
 #[allow(missing_docs)]
 pub enum BestSkeleton<T> {
-    AllFieldsMatch(T),
-    MissingOrExtraFields(T),
+    AllFieldsMatch(T, SkeletonQuality),
+    MissingOrExtraFields(T, SkeletonQuality),
     NoMatch,
 }
 
+/// A measure of the quality of a skeleton.
+///
+/// Internally, this is a u32, a "distance" value. This value is highly
+/// unstable and should not be compared across versions. It should be used
+/// only for comparing against other distances in the same version of ICU4X.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SkeletonQuality(u32);
+
+impl SkeletonQuality {
+    /// Returns the worst possible quality measure.
+    pub fn worst() -> SkeletonQuality {
+        SkeletonQuality(u32::MAX)
+    }
+    /// Returns the best possible quality measure.
+    pub fn best() -> SkeletonQuality {
+        SkeletonQuality(0)
+    }
+}
+
 /// This function swaps out the time zone name field for the appropriate one. Skeleton matching
 /// only needs to find a single "v" field, and then the time zone name can expand from there.
 fn naively_apply_time_zone_name(
@@ -140,51 +164,57 @@ pub fn create_best_pattern_for_fields<'data>(
         get_best_available_format_pattern(skeletons, fields, prefer_matched_pattern);
 
     // Try to match a skeleton to all of the fields.
-    if let BestSkeleton::AllFieldsMatch(mut pattern_plurals) = first_pattern_match {
+    if let BestSkeleton::AllFieldsMatch(mut pattern_plurals, d) = first_pattern_match {
         pattern_plurals.for_each_mut(|pattern| {
             naively_apply_preferences(pattern, components.hour_cycle);
             naively_apply_time_zone_name(pattern, components.time_zone_name);
             apply_subseconds(pattern, components.subsecond);
         });
-        return BestSkeleton::AllFieldsMatch(pattern_plurals);
+        return BestSkeleton::AllFieldsMatch(pattern_plurals, d);
     }
 
     let FieldsByType { date, time } = group_fields_by_type(fields);
 
     if date.is_empty() || time.is_empty() {
         return match first_pattern_match {
-            BestSkeleton::AllFieldsMatch(_) => {
+            BestSkeleton::AllFieldsMatch(_, _) => {
                 unreachable!("Logic error in implementation. AllFieldsMatch handled above.")
             }
-            BestSkeleton::MissingOrExtraFields(mut pattern_plurals) => {
+            BestSkeleton::MissingOrExtraFields(mut pattern_plurals, d) => {
                 if date.is_empty() {
                     pattern_plurals.for_each_mut(|pattern| {
                         naively_apply_preferences(pattern, components.hour_cycle);
                         naively_apply_time_zone_name(pattern, components.time_zone_name);
                         apply_subseconds(pattern, components.subsecond);
                     });
                 }
-                BestSkeleton::MissingOrExtraFields(pattern_plurals)
+                BestSkeleton::MissingOrExtraFields(pattern_plurals, d)
             }
             BestSkeleton::NoMatch => BestSkeleton::NoMatch,
         };
     }
 
     // Match the date and time, and then simplify the combinatorial logic of the results into
     // an optional values of the results, and a boolean value.
-    let (date_patterns, date_missing_or_extra): (Option<PatternPlurals<'data>>, bool) =
-        match get_best_available_format_pattern(skeletons, &date, prefer_matched_pattern) {
-            BestSkeleton::MissingOrExtraFields(fields) => (Some(fields), true),
-            BestSkeleton::AllFieldsMatch(fields) => (Some(fields), false),
-            BestSkeleton::NoMatch => (None, true),
-        };
+    let (date_patterns, date_missing_or_extra, date_distance): (
+        Option<PatternPlurals<'data>>,
+        bool,
+        SkeletonQuality,
+    ) = match get_best_available_format_pattern(skeletons, &date, prefer_matched_pattern) {
+        BestSkeleton::MissingOrExtraFields(fields, d) => (Some(fields), true, d),
+        BestSkeleton::AllFieldsMatch(fields, d) => (Some(fields), false, d),
+        BestSkeleton::NoMatch => (None, true, SkeletonQuality(REQUESTED_SYMBOL_MISSING)),
+    };
 
-    let (time_patterns, time_missing_or_extra): (Option<PatternPlurals<'data>>, bool) =
-        match get_best_available_format_pattern(skeletons, &time, prefer_matched_pattern) {
-            BestSkeleton::MissingOrExtraFields(fields) => (Some(fields), true),
-            BestSkeleton::AllFieldsMatch(fields) => (Some(fields), false),
-            BestSkeleton::NoMatch => (None, true),
-        };
+    let (time_patterns, time_missing_or_extra, time_distance): (
+        Option<PatternPlurals<'data>>,
+        bool,
+        SkeletonQuality,
+    ) = match get_best_available_format_pattern(skeletons, &time, prefer_matched_pattern) {
+        BestSkeleton::MissingOrExtraFields(fields, d) => (Some(fields), true, d),
+        BestSkeleton::AllFieldsMatch(fields, d) => (Some(fields), false, d),
+        BestSkeleton::NoMatch => (None, true, SkeletonQuality(REQUESTED_SYMBOL_MISSING)),
+    };
     let time_pattern: Option<runtime::Pattern<'data>> = time_patterns.map(|pattern_plurals| {
         let mut pattern =
             pattern_plurals.expect_pattern("Only date patterns can contain plural variants");
@@ -254,12 +284,18 @@ pub fn create_best_pattern_for_fields<'data>(
         (None, None) => None,
     };
 
+    let distance = SkeletonQuality(
+        date_distance
+            .0
+            .saturating_add(time_distance.0)
+            .saturating_add(GLUE_DISTANCE),
+    );
     match patterns {
         Some(patterns) => {
             if date_missing_or_extra || time_missing_or_extra {
-                BestSkeleton::MissingOrExtraFields(patterns)
+                BestSkeleton::MissingOrExtraFields(patterns, distance)
             } else {
-                BestSkeleton::AllFieldsMatch(patterns)
+                BestSkeleton::AllFieldsMatch(patterns, distance)
             }
         }
         None => BestSkeleton::NoMatch,
@@ -481,6 +517,7 @@ pub fn get_best_available_format_pattern<'data>(
             // (e.g. text vs numeric). We return the field instead of the matched pattern.
             return BestSkeleton::AllFieldsMatch(
                 runtime::Pattern::from(vec![PatternItem::Field(*field)]).into(),
+                SkeletonQuality(closest_distance),
             );
         }
     }
@@ -496,7 +533,10 @@ pub fn get_best_available_format_pattern<'data>(
     }
 
     if closest_distance == NO_DISTANCE {
-        return BestSkeleton::AllFieldsMatch(closest_format_pattern);
+        return BestSkeleton::AllFieldsMatch(
+            closest_format_pattern,
+            SkeletonQuality(closest_distance),
+        );
     }
 
     // Modify the resulting pattern to have fields of the same length.
@@ -511,8 +551,11 @@ pub fn get_best_available_format_pattern<'data>(
     }
 
     if closest_distance >= SKELETON_EXTRA_SYMBOL {
-        return BestSkeleton::MissingOrExtraFields(closest_format_pattern);
+        return BestSkeleton::MissingOrExtraFields(
+            closest_format_pattern,
+            SkeletonQuality(closest_distance),
+        );
     }
 
-    BestSkeleton::AllFieldsMatch(closest_format_pattern)
+    BestSkeleton::AllFieldsMatch(closest_format_pattern, SkeletonQuality(closest_distance))
 }

provider/source/src/datetime/neo_skeleton.rs

@@ -11,40 +11,31 @@ use icu::datetime::options::Length;
 use icu::datetime::provider::calendar::{DateSkeletonPatterns, TimeLengths};
 use icu::datetime::provider::fields::components;
 use icu::datetime::provider::pattern::{reference, runtime, CoarseHourCycle};
-use icu::datetime::provider::skeleton::PatternPlurals;
+use icu::datetime::provider::skeleton::{PatternPlurals, SkeletonQuality};
 use icu::datetime::provider::*;
 use icu::plurals::PluralElements;
 use icu_locale_core::preferences::extensions::unicode::keywords::HourCycle;
 use icu_provider::prelude::*;
 
 use super::DatagenCalendar;
 
-enum ExactOrSynthetic<T> {
-    Exact(T),
-    Synthetic(T),
+struct PatternsWithDistance<T> {
+    inner: T,
+    distance: SkeletonQuality,
 }
 
-impl<T> ExactOrSynthetic<T> {
-    pub fn map<V>(self, mut f: impl FnMut(T) -> V) -> ExactOrSynthetic<V> {
-        use ExactOrSynthetic::*;
-        match self {
-            Exact(t) => Exact(f(t)),
-            Synthetic(t) => Synthetic(f(t)),
+impl<T> PatternsWithDistance<T> {
+    pub fn map<V>(self, mut f: impl FnMut(T) -> V) -> PatternsWithDistance<V> {
+        PatternsWithDistance {
+            inner: f(self.inner),
+            distance: self.distance,
         }
     }
     pub fn inner(&self) -> &T {
-        use ExactOrSynthetic::*;
-        match self {
-            Exact(t) => t,
-            Synthetic(t) => t,
-        }
+        &self.inner
     }
     pub fn into_inner(self) -> T {
-        use ExactOrSynthetic::*;
-        match self {
-            Exact(t) => t,
-            Synthetic(t) => t,
-        }
+        self.inner
     }
 }
 
@@ -53,7 +44,7 @@ fn select_pattern<'data>(
     skeletons: &DateSkeletonPatterns<'data>,
     preferred_hour_cycle: CoarseHourCycle,
     length_patterns: &GenericLengthPatterns<'data>,
-) -> ExactOrSynthetic<PatternPlurals<'data>> {
+) -> PatternsWithDistance<PatternPlurals<'data>> {
     use icu::datetime::provider::pattern::{runtime, PatternItem};
     use icu::datetime::provider::skeleton::{create_best_pattern_for_fields, BestSkeleton};
     use icu_locale_core::preferences::extensions::unicode::keywords::HourCycle;
@@ -64,8 +55,10 @@ fn select_pattern<'data>(
     };
     let fields = bag.to_vec_fields(default_hour_cycle);
     match create_best_pattern_for_fields(skeletons, length_patterns, &fields, &bag, false) {
-        BestSkeleton::AllFieldsMatch(p) => ExactOrSynthetic::Exact(p),
-        BestSkeleton::MissingOrExtraFields(p) => ExactOrSynthetic::Synthetic(p),
+        BestSkeleton::AllFieldsMatch(p, distance) => PatternsWithDistance { inner: p, distance },
+        BestSkeleton::MissingOrExtraFields(p, distance) => {
+            PatternsWithDistance { inner: p, distance }
+        }
         BestSkeleton::NoMatch => {
             // Build a last-resort pattern that contains all of the requested fields.
             // This is NOT in the CLDR standard! Better would be:
@@ -79,7 +72,10 @@ fn select_pattern<'data>(
                 .skip(1)
                 .collect::<Vec<_>>();
             let pattern = runtime::Pattern::from(pattern_items);
-            ExactOrSynthetic::Synthetic(PatternPlurals::SinglePattern(pattern))
+            PatternsWithDistance {
+                inner: PatternPlurals::SinglePattern(pattern),
+                distance: SkeletonQuality::worst(),
+            }
         }
     }
 }
@@ -133,8 +129,8 @@ impl SourceDataProvider {
             DateSkeletonPatterns::from(&data.datetime_formats.available_formats);
 
         fn expand_pp_to_pe(
-            value: ExactOrSynthetic<PatternPlurals>,
-        ) -> ExactOrSynthetic<PluralElements<runtime::Pattern>> {
+            value: PatternsWithDistance<PatternPlurals>,
+        ) -> PatternsWithDistance<PluralElements<runtime::Pattern>> {
             value.map(|pp| match pp {
                 PatternPlurals::MultipleVariants(variants) => PluralElements::new(variants.other)
                     .with_zero_value(variants.zero.clone())

provider/source/src/datetime/skeletons.rs

@@ -122,8 +122,8 @@ mod test {
         let (_, skeletons) = get_data_payload();
 
         match get_best_available_format_pattern(&skeletons, &requested_fields, false) {
-            BestSkeleton::AllFieldsMatch(available_format_pattern)
-            | BestSkeleton::MissingOrExtraFields(available_format_pattern) => {
+            BestSkeleton::AllFieldsMatch(available_format_pattern, _)
+            | BestSkeleton::MissingOrExtraFields(available_format_pattern, _) => {
                 assert_eq!(
                     available_format_pattern
                         .expect_pattern("pattern should not have plural variants")
@@ -147,7 +147,7 @@ mod test {
         let (_, skeletons) = get_data_payload();
 
         match get_best_available_format_pattern(&skeletons, &requested_fields, false) {
-            BestSkeleton::MissingOrExtraFields(available_format_pattern) => {
+            BestSkeleton::MissingOrExtraFields(available_format_pattern, _) => {
                 assert_eq!(
                     available_format_pattern
                         .expect_pattern("pattern should not have plural variants")
@@ -182,7 +182,7 @@ mod test {
             &Default::default(),
             false,
         ) {
-            BestSkeleton::AllFieldsMatch(available_format_pattern) => {
+            BestSkeleton::AllFieldsMatch(available_format_pattern, _) => {
                 // TODO - Append items are needed here.
                 assert_eq!(
                     available_format_pattern
@@ -376,7 +376,7 @@ mod test {
         let (_, skeletons) = get_data_payload();
 
         match get_best_available_format_pattern(&skeletons, &requested_fields, false) {
-            BestSkeleton::AllFieldsMatch(available_format_pattern) => {
+            BestSkeleton::AllFieldsMatch(available_format_pattern, _) => {
                 assert_eq!(
                     available_format_pattern
                         .expect_pattern("pattern should not have plural variants")
@@ -399,7 +399,7 @@ mod test {
         let (_, skeletons) = get_data_payload();
 
         match get_best_available_format_pattern(&skeletons, &requested_fields, false) {
-            BestSkeleton::AllFieldsMatch(available_format_pattern) => {
+            BestSkeleton::AllFieldsMatch(available_format_pattern, _) => {
                 assert_eq!(
                     available_format_pattern
                         .expect_pattern("pattern should not have plural variants")