Bug 1973862: Support locale tailoring in Intl.Segmenter. r=spidermonkey-reviewers,dminor

Call the new segmenter constructors from ICU4X 2.0 to get locale-dependent
word and sentence segmentation.

Differential Revision: https://phabricator.services.mozilla.com/D255329

Author: anba

config/check_spidermonkey_style.py

@@ -68,6 +68,7 @@
         "icu4x/Date.hpp",  # ICU4X
         "icu4x/GraphemeClusterSegmenter.hpp",  # ICU4X
         "icu4x/IsoDate.hpp",  # ICU4X
+        "icu4x/Locale.hpp",  # ICU4X
         "icu4x/SentenceSegmenter.hpp",  # ICU4X
         "icu4x/WordSegmenter.hpp",  # ICU4X
         "jit/ABIFunctionTypeGenerated.h",  # generated in $OBJDIR"

js/src/builtin/intl/Segmenter.cpp

@@ -18,9 +18,11 @@
 
 #include "builtin/Array.h"
 #include "builtin/intl/CommonFunctions.h"
+#include "builtin/intl/StringAsciiChars.h"
 #include "gc/AllocKind.h"
 #include "gc/GCContext.h"
 #include "icu4x/GraphemeClusterSegmenter.hpp"
+#include "icu4x/Locale.hpp"
 #include "icu4x/SentenceSegmenter.hpp"
 #include "icu4x/WordSegmenter.hpp"
 #include "js/CallArgs.h"
@@ -400,7 +402,7 @@ struct WordSegmenter {
       SegmenterBreakIteratorType<WordSegmenterBreakIteratorTwoByte>;
 
   static constexpr auto& create =
-      icu4x::capi::icu4x_WordSegmenter_create_auto_mv1;
+      icu4x::capi::icu4x_WordSegmenter_create_auto_with_content_locale_mv1;
   static constexpr auto& destroy = icu4x::capi::icu4x_WordSegmenter_destroy_mv1;
 };
 
@@ -444,19 +446,74 @@ struct SentenceSegmenter {
       SegmenterBreakIteratorType<SentenceSegmenterBreakIteratorTwoByte>;
 
   static constexpr auto& create =
-      icu4x::capi::icu4x_SentenceSegmenter_create_mv1;
+      icu4x::capi::icu4x_SentenceSegmenter_create_with_content_locale_mv1;
   static constexpr auto& destroy =
       icu4x::capi::icu4x_SentenceSegmenter_destroy_mv1;
 };
 
+class ICU4XLocaleDeleter {
+ public:
+  void operator()(icu4x::capi::Locale* ptr) {
+    icu4x::capi::icu4x_Locale_destroy_mv1(ptr);
+  }
+};
+
+using UniqueICU4XLocale =
+    mozilla::UniquePtr<icu4x::capi::Locale, ICU4XLocaleDeleter>;
+
+static UniqueICU4XLocale CreateICU4XLocale(JSContext* cx,
+                                           Handle<JSString*> str) {
+  auto* linear = str->ensureLinear(cx);
+  if (!linear) {
+    return nullptr;
+  }
+
+  icu4x::capi::icu4x_Locale_from_string_mv1_result result{};
+  {
+    intl::StringAsciiChars chars(linear);
+    if (!chars.init(cx)) {
+      return nullptr;
+    }
+
+    auto span = static_cast<mozilla::Span<const char>>(chars);
+    result =
+        icu4x::capi::icu4x_Locale_from_string_mv1({span.data(), span.size()});
+  }
+
+  if (!result.is_ok) {
+    intl::ReportInternalError(cx);
+    return nullptr;
+  }
+  return UniqueICU4XLocale{result.ok};
+}
+
 /**
- * Create a new ICU4X segmenter instance.
+ * Create a new, locale-invariant ICU4X segmenter instance.
  */
 template <typename Interface>
 static typename Interface::Segmenter* CreateSegmenter() {
   return Interface::create();
 }
 
+/**
+ * Create a new ICU4X segmenter instance, tailored for |locale|.
+ */
+template <typename Interface>
+static typename Interface::Segmenter* CreateSegmenter(
+    JSContext* cx, Handle<JSString*> locale) {
+  auto loc = CreateICU4XLocale(cx, locale);
+  if (!loc) {
+    return nullptr;
+  }
+
+  auto result = Interface::create(loc.get());
+  if (!result.is_ok) {
+    intl::ReportInternalError(cx);
+    return nullptr;
+  }
+  return result.ok;
+}
+
 static bool EnsureInternalsResolved(JSContext* cx,
                                     Handle<SegmenterObject*> segmenter) {
   if (segmenter->getLocale()) {
@@ -506,15 +563,15 @@ static bool EnsureInternalsResolved(JSContext* cx,
       break;
     }
     case SegmenterGranularity::Word: {
-      auto* seg = CreateSegmenter<WordSegmenter>();
+      auto* seg = CreateSegmenter<WordSegmenter>(cx, locale);
       if (!seg) {
         return false;
       }
       segmenter->setSegmenter(seg);
       break;
     }
     case SegmenterGranularity::Sentence: {
-      auto* seg = CreateSegmenter<SentenceSegmenter>();
+      auto* seg = CreateSegmenter<SentenceSegmenter>(cx, locale);
       if (!seg) {
         return false;
       }

js/src/tests/non262/Intl/Segmenter/sentence-latin.js

@@ -85,11 +85,8 @@ for (let [string, words] of Object.entries(strings)) {
   // A single sentence in English.
   assertEq([...english.segment(string)].length, 1);
 
-  // ICU4C: Two sentences in Greek.
-  // assertEq([...greek.segment(string)].length, 2);
-
-  // ICU4X: A single sentence in Greek.
-  assertEq([...greek.segment(string)].length, 1);
+  // Two sentences in Greek.
+  assertEq([...greek.segment(string)].length, 2);
 }
 
 if (typeof reportCompare === "function")

js/src/tests/non262/Intl/Segmenter/word-latin1.js

@@ -206,5 +206,20 @@ assertSegments("_'_", ["_", "'", "_"]);
 assertEq(new Intl.Segmenter("en-posix").resolvedOptions().locale, "en");
 assertEq(new Intl.Segmenter("en-u-va-posix").resolvedOptions().locale, "en");
 
+// Locale-dependent word segmentation.
+{
+  // https://en.wikipedia.org/wiki/Colon_(punctuation)#Abbreviation_mark
+  let string = "Word:with:colon";
+
+  let english = new Intl.Segmenter("en", {granularity: "word"});
+  let svenska = new Intl.Segmenter("sv", {granularity: "word"});
+
+  // Three words with two separators in English.
+  assertEq([...english.segment(string)].length, 5);
+
+  // A single word in Swedish.
+  assertEq([...svenska.segment(string)].length, 1);
+}
+
 if (typeof reportCompare === "function")
   reportCompare(0, 0);