ICU4X Segmenter now working for 2.0 beta2 (#465)

* Starting Segmenter tests in ICU4X

* ICU4X segmenter in beta2.02 and later

* Remove option to break lines within words

Author: Craig Cornelius

executors/rust/common/run_all_tests.rs

@@ -12,6 +12,7 @@
 // 9. DONE Modularize into separate files for each type of test
 // 10. Fix test_type and switch statement
 // 11. DONE Add language names --> locale names
+// 12. How to add new component in certain versions?
 
 // References for ICU4X:
 // https://unicode-org.github.io/icu4x-docs/doc/icu_collator/index.html
@@ -34,6 +35,7 @@ pub struct ExecutorFns {
     pub run_numberformat_test: ExecutorFn,
     pub run_plural_rules_test: ExecutorFn,
     pub run_relativedatetimeformat_test: ExecutorFn,
+    pub run_segmenter_test: ExecutorFn,
 }
 
 pub fn main() -> io::Result<()> {
@@ -47,6 +49,10 @@ pub fn main() -> io::Result<()> {
         run_plural_rules_test: executors::pluralrules::run_plural_rules_test,
         run_relativedatetimeformat_test:
             executors::relativedatetime_fmt::run_relativedatetimeformat_test,
+        #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
+        run_segmenter_test: executors::segmenter::run_segmenter_test,
+        #[cfg(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1"))]
+        run_segmenter_test: |_| Err("segmenter not supported".to_string()),
     };
     run_all_tests(executor_fns)
 }
@@ -126,6 +132,8 @@ pub fn run_all_tests(fns: ExecutorFns) -> io::Result<()> {
                 (fns.run_plural_rules_test)(&json_info)
             } else if test_type == "rdt_fmt" {
                 (fns.run_relativedatetimeformat_test)(&json_info)
+            } else if test_type == "segmenter" {
+                (fns.run_segmenter_test)(&json_info)
             } else {
                 Err(test_type.to_string())
             };

executors/rust/src/mod.rs

@@ -10,6 +10,8 @@ pub mod localenames;
 pub mod numberfmt;
 pub mod pluralrules;
 pub mod relativedatetime_fmt;
+#[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
+pub mod segmenter;
 
 #[cfg(any(ver = "1.3", ver = "1.4", ver = "1.5"))]
 #[path = "datetime_1.rs"]

executors/rust/src/segmenter.rs

@@ -0,0 +1,89 @@
+//! Executor provides tests for segmenter in locale-sensitive manner.
+
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+#[cfg(not(any(ver = "2.0-beta1")))]
+use icu::segmenter::{options::*, *};
+
+use super::compat::Locale;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "snake_case")]
+struct SegmenterOptions {
+    granularity: Option<String>,
+}
+
+// Function runs segmenter tests
+pub fn run_segmenter_test(json_obj: &Value) -> Result<Value, String> {
+    // To use the locale
+    let label = &json_obj["label"].as_str().unwrap();
+
+    let locale_str: &str = json_obj["locale"].as_str().unwrap();
+    let locale = locale_str.parse::<Locale>().unwrap();
+    let lang_identifier = locale.id;
+
+    let options = &json_obj["options"]; // This will be an array.
+    let option_struct: SegmenterOptions = serde_json::from_str(&options.to_string()).unwrap();
+
+    let granularity: &str = option_struct.granularity.as_ref().unwrap();
+
+    let input_string: &str = json_obj["input"].as_str().unwrap();
+
+    // Get desired segmenter, then comput the break points from the input.
+    let breakpoints: Vec<usize> = match granularity {
+        "grapheme_cluster" => {
+            let segmenter = GraphemeClusterSegmenter::new();
+            segmenter.segment_str(input_string).collect()
+        }
+        "word" => {
+            // Get options
+            let mut options = WordBreakOptions::default();
+            options.content_locale = Some(&lang_identifier);
+
+            let segmenter = WordSegmenter::try_new_auto(options).unwrap();
+            segmenter.segment_str(input_string).collect()
+        }
+        "sentence" => {
+            // Get options
+            let mut options = SentenceBreakOptions::default();
+            options.content_locale = Some(&lang_identifier);
+            let segmenter = SentenceSegmenter::try_new(options).unwrap();
+            segmenter.segment_str(input_string).collect()
+        }
+        "line" => {
+            // Get options
+            let mut options = LineBreakOptions::default();
+            options.strictness = Some(LineBreakStrictness::Strict);
+            options.content_locale = Some(&lang_identifier);
+            let segmenter = LineSegmenter::new_auto(options);
+            segmenter.segment_str(input_string).collect()
+        }
+        _ => {
+            // This is an error
+            return Ok(json!({
+                "label": label,
+                "locale_label": locale_str,
+                "error": "Unknown segmenter option",
+                "error_type": "unsupported",
+                "unsupported": granularity.to_string(),
+                "error_detail": {"unsupported_locale": locale_str}
+            }));
+        }
+    };
+
+    // Create output as list of strings broken ad the breakpoints.
+    // For breakpoints, extract each part and push to result;
+    let mut segments: Vec<String> = vec![];
+    let mut start: usize = 0;
+    for index in breakpoints.iter() {
+        if *index > 0 {
+            segments.push(input_string[start..*index].to_string());
+        }
+        start = *index;
+    }
+    Ok(json!({
+        "label": label,
+        "result": segments
+    }))
+}

run_config.json

@@ -522,7 +522,8 @@
         "likely_subtags",
         "list_fmt",
         "plural_rules",
-        "rdt_fmt"
+        "rdt_fmt",
+        "segmenter"
       ],
       "per_execution": 10000
     }