Merge pull request #9440 from keymanapp/feat/core/9121-regex-epic-ldml

Author: srl295

core/src/kmx/kmx_plus.cpp

@@ -1072,16 +1072,16 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
   return is_valid;
 }
 
-USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount)  {
+SimpleUSet::SimpleUSet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount)  {
   for (size_t i = 0; i < newCount; i++) {
     ranges.emplace_back(newRange[i].start, newRange[i].end);
   }
 }
 
-USet::USet() {
+SimpleUSet::SimpleUSet() {
 }
 
-bool USet::contains(km_kbp_usv ch) const {
+bool SimpleUSet::contains(km_kbp_usv ch) const {
   for (const auto &range : ranges) {
     if (range.start <= ch && range.end >= ch) {
       return true;
@@ -1091,7 +1091,7 @@ bool USet::contains(km_kbp_usv ch) const {
 }
 
 bool
-USet::valid() const {
+SimpleUSet::valid() const {
   // double check
   for (const auto &range : ranges) {
     if (!Uni_IsValid(range.start, range.end)) {
@@ -1103,7 +1103,7 @@ USet::valid() const {
 }
 
 void
-USet::dump() const {
+SimpleUSet::dump() const {
   DebugLog(" - USet size=%d", ranges.size());
   for (const auto &range : ranges) {
     if (range.start == range.end) {
@@ -1114,14 +1114,14 @@ USet::dump() const {
   }
 }
 
-USet
+SimpleUSet
 COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const {
   if (!valid() || i >= uset->usetCount) {
     assert(false);
-    return USet(nullptr, 0); // empty set
+    return SimpleUSet(nullptr, 0); // empty set
   }
   auto &set = usets[i];
-  return USet(getRange(set.range), set.count);
+  return SimpleUSet(getRange(set.range), set.count);
 }
 
 const COMP_KMXPLUS_USET_RANGE *

core/src/kmx/kmx_plus.h

@@ -699,14 +699,16 @@ struct COMP_KMXPLUS_USET_RANGE {
 };
 
 /**
- * represents one of the uset elements
+ * represents one of the uset elements.
+ * TODO-LDML: replace this with a real icu::UnicodeSet? or at least
+ * a function producing the same?
  */
-class USet {
+class SimpleUSet {
   public:
     /** construct a set over the specified range. Data is copied. */
-    USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
+    SimpleUSet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
     /** empty set */
-    USet();
+    SimpleUSet();
     /** true if the uset contains this char */
     bool contains(km_kbp_usv ch) const;
     /** debugging */
@@ -726,7 +728,7 @@ class COMP_KMXPLUS_USET_Helper {
   bool setUset(const COMP_KMXPLUS_USET *newUset);
   inline bool valid() const { return is_valid; }
 
-  USet getUset(KMXPLUS_USET list) const;
+  SimpleUSet getUset(KMXPLUS_USET list) const;
   const COMP_KMXPLUS_USET_RANGE *getRange(KMX_DWORD index) const;
 
 private:

core/src/kmx/kmx_xstring.h

@@ -172,6 +172,19 @@ u16string_to_u32string(const std::u16string &source) {
   return out;
 }
 
+inline std::u16string
+u32string_to_u16string(const std::u32string &source) {
+  std::u16string out;
+  char16_single ch;
+  for (auto c : source) {
+    const auto len = Utf32CharToUtf16(c, ch);
+    for (auto i = 0; i < len; i++) {
+      out.push_back(ch.ch[i]);
+    }
+  }
+  return out;
+}
+
 inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) {
   return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE
 }

core/src/ldml/ldml_transforms.cpp

@@ -11,7 +11,6 @@
 #include <string>
 #include "kmx/kmx_xstring.h"
 
-
 #ifndef assert
 #define assert(x)  // TODO-LDML
 #endif
@@ -36,7 +35,7 @@ namespace ldml {
 #define DebugTran(msg, ...)
 #endif
 
-element::element(const USet &new_u, KMX_DWORD new_flags)
+element::element(const SimpleUSet &new_u, KMX_DWORD new_flags)
     : chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) {
 }
 
@@ -191,7 +190,7 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
       km_kbp_usv ch = e.element;
       emplace_back(ch, flags); // char
     } else if (type == LDML_ELEM_FLAGS_TYPE_USET) {
-      // need to load a USet
+      // need to load a SimpleUSet
       auto u = kplus.usetHelper.getUset(e.element);
       if (!u.valid()) {
         DebugLog("Error, invalid UnicodeSet at element %d", (int)i);
@@ -393,35 +392,96 @@ reorder_group::apply(std::u32string &str) const {
   return applied;
 }
 
+transform_entry::transform_entry(const transform_entry &other) :
+  fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr) {
+  if (other.fFromPattern) {
+    // clone pattern
+    fFromPattern.reset(other.fFromPattern->clone());
+  }
+}
+
 transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) {
+  assert(!fFrom.empty()); // TODO-LDML: should not happen?
+
+  if (!fFrom.empty()) {
+    const std::u16string patstr = km::kbp::kmx::u32string_to_u16string(fFrom);
+    UErrorCode status           = U_ZERO_ERROR;
+    /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length());
+    // add '$' to match to end
+    patustr.append(u'$');
+    fFromPattern.reset(icu::RegexPattern::compile(patustr, 0, status));
+    assert(U_SUCCESS(status)); // TODO-LDML: may be best to propagate status up ^^
+  }
 }
 
 size_t
-transform_entry::match(const std::u32string &input) const {
-  if (input.length() < fFrom.length()) {
-    // TODO-LDML: regex
-    // Too small, can't match.
-    return 0;
+transform_entry::apply(const std::u32string &input, std::u32string &output) const {
+  assert(fFromPattern);
+  // TODO-LDML: Really? can't go from u32 to UnicodeString?
+  // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher.
+  UErrorCode status = U_ZERO_ERROR;
+  const std::u16string matchstr = km::kbp::kmx::u32string_to_u16string(input);
+  icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
+  // TODO-LDML: create a new Matcher every time. These could be cached and reset.
+  std::unique_ptr<icu::RegexMatcher> matcher(fFromPattern->matcher(matchustr, status));
+  assert(U_SUCCESS(status));
+
+  if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str
+    return 0; // no match
   }
-  // string at end
-  auto substr = input.substr(input.length() - fFrom.length(), fFrom.length());
-  if (substr != fFrom) {
-    // end of string doesn't match
+
+  // TODO-LDML: this is UTF-16 len, not UTF-32 len!!
+  // TODO-LDML: if we had an underlying UText this would be simpler.
+  int32_t matchStart = matcher->start(status);
+  int32_t matchEnd   = matcher->end(status);
+  assert(U_SUCCESS(status));
+  // extract..
+  const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd);
+  // preflight to UTF-32 to get length
+  UErrorCode substrStatus = U_ZERO_ERROR;
+  auto matchLen = substr.toUTF32(nullptr, 0, substrStatus);
+  assert(matchLen > 0);
+  if (matchLen == 0) {
     return 0;
   }
-  // match length == fFrom.length
-  return substr.length();
-}
-
-std::u32string
-transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const {
-  // TODO-LDML: regex
-  // For now, we just return the 'to' string literally.
-  return fTo;
+  // Now, we have a matchLen.
+
+  // now, do the replace.
+  // Convert the fTo into u16 TODO-LDML (we could cache this?)
+  const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fTo);
+  icu::UnicodeString rustr  = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
+  // This replace will apply $1, $2 etc.  TODO-LDML it will NOT handle mapFrom or mapTo.
+  icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status);
+  assert(U_SUCCESS(status));
+  // entireOutput includes all of 'input', but modified. Need to substring it.
+  icu::UnicodeString outu = entireOutput.tempSubString(matchStart);
+
+  // Special case if there's no output
+  if (outu.length() == 0) {
+    output.clear();
+  } else {
+    // TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps.
+    UErrorCode preflightStatus = U_ZERO_ERROR;
+    // calculate how big the buffer is
+    auto out32len              = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes
+    // allocate
+    char32_t *s                = new char32_t[out32len + 1];
+    assert(s != nullptr);
+    // convert
+    outu.toUTF32((UChar32 *)s, out32len + 1, status);
+    assert(U_SUCCESS(status));
+    output.assign(s, out32len);
+    // now, build a u32string
+    std::u32string out32(s, out32len);
+    // clean up buffer
+    delete [] s;
+  }
+  return matchLen;
 }
 
 any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() {
 }
+
 any_group::any_group(const reorder_group &g) : type(any_group_type::reorder), transform(), reorder(g) {
 }
 
@@ -444,17 +504,18 @@ transform_group::transform_group() {
 /**
  * return the first transform match in this group
  */
-const transform_entry *
-transform_group::match(const std::u32string &input, size_t &subMatched) const {
+size_t
+transform_group::apply(const std::u32string &input, std::u32string &output) const {
+  size_t subMatched = 0;
   for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) {
     // TODO-LDML: non regex implementation
     // is the match area too short?
-    subMatched = transform->match(input);
+    subMatched = transform->apply(input, output);
     if (subMatched != 0) {
-      return &(*transform);  // return alias to transform
+      return subMatched; // matched. break out.
     }
   }
-  return nullptr;
+  return 0; // no match
 }
 
 /**
@@ -507,20 +568,14 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
     // TODO-LDML: reorders
     // Assume it's a non reorder group
     /** Length of match within this group*/
-    size_t subMatched = 0;
 
     // find the first match in this group (if present)
     // TODO-LDML: check if reorder
     if (group->type == any_group_type::transform) {
-      auto entry = group->transform.match(updatedInput, subMatched);
-
-      if (entry != nullptr) {
-        // now apply the found transform
-
-        // update subOutput (string) and subMatched
-        // the returned string must replace the last "subMatched" chars of the string.
-        std::u32string subOutput = entry->apply(updatedInput, subMatched);
+      std::u32string subOutput;
+      size_t subMatched = group->transform.apply(updatedInput, subOutput);
 
+      if (subMatched != 0) {
         // remove the matched part of the updatedInput
         updatedInput.resize(updatedInput.length() - subMatched);  // chop of the subMatched part at end
         updatedInput.append(subOutput);                           // subOutput could be empty such as in backspace transform