Merge pull request #9504 from keymanapp/feat/core/9121-regex-map-epic-ldml

feat(core): implement mapped set mapping 🙀

Author: srl295

core/src/kmx/kmx_plus.cpp

@@ -13,6 +13,7 @@
 #include "ldml/keyboardprocessor_ldml.h"
 
 #include <assert.h>
+#include "kmx_plus.h"
 
 namespace km {
 namespace kbp {
@@ -386,11 +387,35 @@ COMP_KMXPLUS_ELEM::getElementList(KMX_DWORD elementNumber, KMX_DWORD &length) co
 
 std::u16string
 COMP_KMXPLUS_ELEM_ELEMENT::get_element_string() const {
-  assert((flags & LDML_ELEM_FLAGS_TYPE) == LDML_ELEM_FLAGS_TYPE_CHAR); // should only be called on char
+  assert(type() == LDML_ELEM_FLAGS_TYPE_CHAR); // should only be called on char
   return COMP_KMXPLUS_STRS::str_from_char(element);
 }
 
-// Note: shared with subclass COMP_KMXPLUS_BKSP
+std::deque<std::u32string>
+COMP_KMXPLUS_ELEM_ELEMENT::loadAsStringList(KMX_DWORD length, const COMP_KMXPLUS_STRS &strs) const {
+  std::deque<std::u32string> list;
+  for (KMX_DWORD i = 0; i<length; i++) {
+    const auto &o = this[i];
+    std::u32string str;
+    if (o.type() == LDML_ELEM_FLAGS_TYPE_STR) {
+      // fetch the string
+      const auto str16 = strs.get(o.element);
+      str = km::kbp::kmx::u16string_to_u32string(str16);
+    } else {
+      // single char
+      str = std::u32string(1, (km_kbp_usv)o.element);
+    }
+    list.emplace_back(str);
+  }
+  return list;
+}
+
+KMX_DWORD
+COMP_KMXPLUS_ELEM_ELEMENT::type() const {
+  return (flags & LDML_ELEM_FLAGS_TYPE);
+}
+
+  // Note: shared with subclass COMP_KMXPLUS_BKSP
 bool
 COMP_KMXPLUS_TRAN::valid(KMX_DWORD _kmn_unused(length)) const {
   if (header.size < sizeof(*this) + (sizeof(COMP_KMXPLUS_TRAN_GROUP) * groupCount) +
@@ -503,6 +528,18 @@ COMP_KMXPLUS_TRAN_Helper::setTran(const COMP_KMXPLUS_TRAN *newTran) {
           is_valid = false;
           assert(is_valid);
         }
+        for(KMX_DWORD t = 0; is_valid && t < group.count; t++) {
+          const auto &transform = transforms[group.index + t];
+          if (transform.from == 0) {
+            DebugLog("COMP_KMXPLUS_TRAN_Helper: transform [%d].[%d] has empty 'from' string", i, t);
+            is_valid = false;
+            assert(is_valid);
+          } else if ((transform.mapFrom == 0) != (transform.mapTo == 0)) {
+            DebugLog("COMP_KMXPLUS_TRAN_Helper: transform [%d].[%d] should have neither or both mapFrom=%d/mapTo=%d", i, t, transform.mapFrom, transform.mapTo);
+            is_valid = false;
+            assert(is_valid);
+          }
+        }
       } else if (group.type == LDML_TRAN_GROUP_TYPE_REORDER) {
         DebugLog(" .. type=reorder");
         if ((group.index >= tran->reorderCount) || (group.index + group.count > tran->reorderCount)) {
@@ -511,6 +548,15 @@ COMP_KMXPLUS_TRAN_Helper::setTran(const COMP_KMXPLUS_TRAN *newTran) {
           is_valid = false;
           assert(is_valid);
         }
+        for(KMX_DWORD t = 0; is_valid && t < group.count; t++) {
+          const auto &reorder = reorders[group.index + t];
+          if (reorder.elements == 0) {
+            DebugLog("COMP_KMXPLUS_TRAN_Helper: reorder [%d].[%d] has elements=0", i, t);
+            // TODO-LDML: is this an error?
+            // is_valid = false;
+            // assert(is_valid);
+          }
+        }
       } else {
         DebugLog(" .. type=illegal 0x%X", group.type);
         is_valid = false;
@@ -1248,6 +1294,19 @@ COMP_KMXPLUS_VARS::valid(KMX_DWORD _kmn_unused(length)) const {
   return true;
 }
 
+const COMP_KMXPLUS_VARS_ITEM *COMP_KMXPLUS_VARS::findByStringId(KMX_DWORD strId) const {
+  if (strId == 0) {
+    return nullptr;
+  }
+  for (KMX_DWORD index = 0; index < varCount; index++) {
+    if (varEntries[index].id == strId) {
+      return &(varEntries[index]);
+    }
+  }
+  return nullptr;
+}
+
+
 
 }  // namespace kmx
 }  // namespace kbp

core/src/kmx/kmx_plus.h

@@ -12,6 +12,7 @@
 #include <kmx_file.h>
 #include <ldml/keyboardprocessor_ldml.h>
 #include <list>
+#include <deque>
 
 namespace km {
 namespace kbp {
@@ -47,6 +48,7 @@ struct COMP_KMXPLUS_TRAN;
 struct COMP_KMXPLUS_TRAN_GROUP;
 struct COMP_KMXPLUS_TRAN_TRANSFORM;
 struct COMP_KMXPLUS_TRAN_REORDER;
+struct COMP_KMXPLUS_STRS;
 
 struct COMP_KMXPLUS_HEADER {
   KMXPLUS_IDENT ident;       // 0000 Section name
@@ -103,6 +105,16 @@ struct COMP_KMXPLUS_ELEM_ELEMENT {
    * @return std::u16string
    */
   std::u16string get_element_string() const;
+
+  /**
+   * @brief load this[0]…this[length] as a string list
+   * @param length number of elements, including this one
+   * @return the string elements as a string array
+  */
+  std::deque<std::u32string> loadAsStringList(KMX_DWORD length, const km::kbp::kmx::COMP_KMXPLUS_STRS &strs) const;
+
+  /** @return element type */
+  KMX_DWORD type() const;
 };
 
 struct COMP_KMXPLUS_ELEM_ENTRY {
@@ -260,8 +272,8 @@ struct COMP_KMXPLUS_TRAN_GROUP {
 struct COMP_KMXPLUS_TRAN_TRANSFORM {
     KMXPLUS_STR from;
     KMXPLUS_STR to;
-    KMXPLUS_ELEM mapFrom;
-    KMXPLUS_ELEM mapTo;
+    KMXPLUS_STR mapFrom; // variable name
+    KMXPLUS_STR mapTo;   // variable name
 };
 
 struct COMP_KMXPLUS_TRAN_REORDER {
@@ -341,21 +353,23 @@ struct COMP_KMXPLUS_BKSP : public COMP_KMXPLUS_TRAN {
 
 struct COMP_KMXPLUS_VARS_ITEM {
     KMX_DWORD_unaligned type;
-    KMX_DWORD_unaligned id;
-    KMX_DWORD_unaligned value;
-    KMX_DWORD_unaligned elem;
+    KMXPLUS_STR id;
+    KMXPLUS_STR value;
+    KMXPLUS_ELEM elem;
 };
 
 struct COMP_KMXPLUS_VARS {
   static const KMXPLUS_IDENT IDENT = LDML_SECTIONID_VARS;
   COMP_KMXPLUS_HEADER header;
-  KMX_DWORD_unaligned markers;
+  KMXPLUS_LIST markers;
   KMX_DWORD_unaligned varCount;
   COMP_KMXPLUS_VARS_ITEM varEntries[];
   /**
    * @brief True if section is valid.
    */
   bool valid(KMX_DWORD length) const;
+
+  const COMP_KMXPLUS_VARS_ITEM *findByStringId(KMX_DWORD strId) const;
 };
 
 static_assert(sizeof(struct COMP_KMXPLUS_VARS) % 0x4 == 0, "Structs prior to variable part should align to 32-bit boundary");

core/src/ldml/ldml_transforms.cpp

@@ -392,18 +392,74 @@ reorder_group::apply(std::u32string &str) const {
   return applied;
 }
 
-transform_entry::transform_entry(const transform_entry &other) :
-  fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr) {
+transform_entry::transform_entry(const transform_entry &other)
+    : fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr), fMapFromStrId(other.fMapFromStrId),
+      fMapToStrId(other.fMapToStrId), fMapFromList(other.fMapFromList), fMapToList(other.fMapToList) {
   if (other.fFromPattern) {
     // clone pattern
     fFromPattern.reset(other.fFromPattern->clone());
   }
 }
 
-transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) {
+transform_entry::transform_entry(const std::u32string &from, const std::u32string &to)
+    : fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(), fMapToStrId(), fMapFromList(), fMapToList() {
   assert(!fFrom.empty()); // TODO-LDML: should not happen?
 
+  init();
+}
+
+// TODO-LDML: How do we return errors from here?
+transform_entry::transform_entry(
+    const std::u32string &from,
+    const std::u32string &to,
+    KMX_DWORD mapFrom,
+    KMX_DWORD mapTo,
+    const kmx::kmx_plus &kplus)
+    : fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(mapFrom), fMapToStrId(mapTo) {
+  assert(!fFrom.empty()); // TODO-LDML: should not happen?
+  assert((fMapFromStrId == 0) == (fMapToStrId == 0));  // we have both or we have neither.
+  assert(kplus.strs != nullptr);
+  assert(kplus.vars != nullptr);
+  assert(kplus.elem != nullptr);
+  init();
+
+  // setup mapFrom
+  if (fMapFromStrId != 0) {
+    // Note: if we need the variable name it is available as follows,
+    // but isn't needed for normal processing. Could be useful for debug messages.
+    //  auto mapFrom = kplus.strs->get(fMapFromStrId);
+    //  auto mapTo   = kplus.strs->get(fMapToStrId);
+
+    // get the vars
+    auto *fromVar = kplus.vars->findByStringId(fMapFromStrId);
+    auto *toVar   = kplus.vars->findByStringId(fMapToStrId);
+    assert(fromVar != nullptr);
+    assert(toVar   != nullptr);
+
+
+    // get the element lists
+    assert(fromVar->type == LDML_VARS_ENTRY_TYPE_SET);
+    assert(toVar->type   == LDML_VARS_ENTRY_TYPE_SET);
+    KMX_DWORD fromLength, toLength;
+    auto *fromList = kplus.elem->getElementList(fromVar->elem, fromLength);
+    auto *toList   = kplus.elem->getElementList(toVar->elem, toLength);
+    assert(fromLength == toLength);
+    assert(fromList != nullptr);
+    assert(toList   != nullptr);
+
+    // populate the deques from the lists
+    fMapFromList = fromList->loadAsStringList(fromLength, *(kplus.strs));
+    fMapToList   = toList->loadAsStringList(toLength, *(kplus.strs));
+    // did we get the expected items?
+    assert(fMapFromList.size() == fromLength);
+    assert(fMapToList.size()   == toLength);
+  }
+}
+
+void
+transform_entry::init() {
   if (!fFrom.empty()) {
+    // TODO-LDML: if we have mapFrom, may need to do other processing.
     const std::u16string patstr = km::kbp::kmx::u32string_to_u16string(fFrom);
     UErrorCode status           = U_ZERO_ERROR;
     /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length());
@@ -421,7 +477,7 @@ transform_entry::apply(const std::u32string &input, std::u32string &output) cons
   // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher.
   UErrorCode status = U_ZERO_ERROR;
   const std::u16string matchstr = km::kbp::kmx::u32string_to_u16string(input);
-  icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
+  icu::UnicodeString matchustr  = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
   // TODO-LDML: create a new Matcher every time. These could be cached and reset.
   std::unique_ptr<icu::RegexMatcher> matcher(fFromPattern->matcher(matchustr, status));
   assert(U_SUCCESS(status));
@@ -438,25 +494,65 @@ transform_entry::apply(const std::u32string &input, std::u32string &output) cons
   // extract..
   const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd);
   // preflight to UTF-32 to get length
-  UErrorCode substrStatus = U_ZERO_ERROR;
+  UErrorCode substrStatus = U_ZERO_ERROR; // throwaway status
+  // we need the UTF-32 matchLen for our return.
   auto matchLen = substr.toUTF32(nullptr, 0, substrStatus);
+
+  // should have matched something.
   assert(matchLen > 0);
-  if (matchLen == 0) {
-    return 0;
-  }
-  // Now, we have a matchLen.
 
   // now, do the replace.
-  // Convert the fTo into u16 TODO-LDML (we could cache this?)
-  const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fTo);
-  icu::UnicodeString rustr  = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
-  // This replace will apply $1, $2 etc.  TODO-LDML it will NOT handle mapFrom or mapTo.
+
+  /** this is the 'to' or other replacement string.*/
+  icu::UnicodeString rustr;
+  if (fMapFromStrId == 0) {
+    // Normal case: not a map.
+    // This replace will apply $1, $2 etc.
+    // Convert the fTo into u16 TODO-LDML (we could cache this?)
+    const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fTo);
+    rustr  = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
+  } else {
+    // Set map case: mapping from/to
+
+    // we actually need the group(1) string here.
+    // this is only the content in parenthesis ()
+    icu::UnicodeString group1 = matcher->group(1, status);
+    assert(U_SUCCESS(status)); // TODO-LDML: could be a malformed from pattern
+    // now, how long is group1 in UTF-32, hmm?
+    UErrorCode preflightStatus = U_ZERO_ERROR; // throwaway status
+    auto group1Len             = group1.toUTF32(nullptr, 0, preflightStatus);
+    char32_t *s                = new char32_t[group1Len + 1];
+    assert(s != nullptr); // TODO-LDML: OOM
+    // convert
+    substr.toUTF32((UChar32 *)s, group1Len + 1, status);
+    assert(U_SUCCESS(status));
+    std::u32string match32(s, group1Len); // taken from just group1
+    // clean up buffer
+    delete [] s;
+
+    // Now we're ready to do the actual mapping.
+
+    // 1., we need to find the index in the source set.
+    auto matchIndex = findIndexFrom(match32);
+    assert(matchIndex != -1L); // TODO-LDML: not matching shouldn't happen, the regex wouldn't have matched.
+    // we already asserted on load that the from and to sets have the same cardinality.
+
+    // 2. get the target string, convert to utf-16
+    // we use the same matchIndex that was just found
+    const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fMapToList.at(matchIndex));
+
+    // 3. update the UnicodeString for replacement
+    rustr  = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
+    // and we return to the regular code flow.
+  }
+  // here we replace the match output.
   icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status);
-  assert(U_SUCCESS(status));
+  assert(U_SUCCESS(status)); // TODO-LDML: could fail here due to bad input (syntax err)
+
   // entireOutput includes all of 'input', but modified. Need to substring it.
   icu::UnicodeString outu = entireOutput.tempSubString(matchStart);
 
-  // Special case if there's no output
+  // Special case if there's no output, save some allocs
   if (outu.length() == 0) {
     output.clear();
   } else {
@@ -479,6 +575,20 @@ transform_entry::apply(const std::u32string &input, std::u32string &output) cons
   return matchLen;
 }
 
+int32_t transform_entry::findIndexFrom(const std::u32string &match) const {
+  return findIndex(match, fMapFromList);
+}
+
+int32_t transform_entry::findIndex(const std::u32string &match, const std::deque<std::u32string> list) {
+  int32_t index = 0;
+  for(auto e = list.begin(); e < list.end(); e++, index++) {
+    if (match == *e) {
+      return index;
+    }
+  }
+  return -1; // not found
+}
+
 any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() {
 }
 
@@ -686,16 +796,9 @@ transforms::load(
         const kmx::COMP_KMXPLUS_TRAN_TRANSFORM *element = tranHelper.getTransform(group->index + itemNumber);
         const std::u32string fromStr                    = kmx::u16string_to_u32string(kplus.strs->get(element->from));
         const std::u32string toStr                      = kmx::u16string_to_u32string(kplus.strs->get(element->to));
-        std::u16string mapFrom, mapTo;
-
-        if (element->mapFrom && element->mapTo) {
-          // strings: variable name of from/to
-          // TODO-LDML: not implemented
-          mapFrom = kplus.strs->get(element->mapFrom);
-          mapTo   = kplus.strs->get(element->mapTo);
-        }
-
-        newGroup.emplace_back(fromStr, toStr /* ,mapFrom, mapTo */);  // creating a transform_entry
+        KMX_DWORD mapFrom                               = element->mapFrom; // copy, because of alignment
+        KMX_DWORD mapTo                                 = element->mapTo;   // copy, because of alignment
+        newGroup.emplace_back(fromStr, toStr, mapFrom, mapTo, kplus);  // creating a transform_entry
       }
       transforms->addGroup(newGroup);
     } else if (group->type == LDML_TRAN_GROUP_TYPE_REORDER) {