Fix collation test gen. Also resolve all Java and CPP executor test failures. (#495)

* Fix problems from unescaping test strings

* UPdate collation generator to create correct rules

* Fixing collation generation to handle \x20 and \xA0 characters. ICU4J handle caseFirst and caseLevel

* FIxing all bug 6 ICU4C collation test failures

* Fixing all but 6 ICU4C collation test failures

* Fix all test failures in CPP collation

Author: Craig Cornelius

executors/cpp/coll.cpp

@@ -227,11 +227,24 @@ auto TestCollator(json_object *json_in) -> string {
     reorder_codes_v = BuildReorderList(reorder_string, debug_level);
   }
 
+  json_object *alternate_obj = json_object_object_get(json_in, "alternate");
+  UColAttributeValue alternate_value;
+  if (alternate_obj) {
+    string alternate = json_object_get_string(alternate_obj);
+    if (alternate == "shifted") {
+      alternate_value = UCOL_SHIFTED;
+    } else
+      if (alternate == "non-ignorable") {
+        alternate_value = UCOL_NON_IGNORABLE;
+      }
+  }
+
   // Check for rule-based collation
   json_object *rules_obj = json_object_object_get(json_in, "rules");
   int rules_len = json_object_get_string_len(rules_obj);
   string_view rules_string(json_object_get_string(rules_obj), rules_len);
   UnicodeString uni_rules = UnicodeString::fromUTF8(rules_string).unescape();
+  string actual_rules = "";
 
   // Handle some options
   json_object *ignore_obj =
@@ -262,16 +275,57 @@ auto TestCollator(json_object *json_in) -> string {
       return json_object_to_json_string(return_json);
     }
 
+    // Get the rules as seen by the collator.
+    UnicodeString gotten_rules = rb_coll->getRules();
+    gotten_rules.toUTF8String(actual_rules);
+
+    // Make sure that attributes and optionsare set for rule based collator, too.
     if (reorder_obj) {
-      if (debug_level > 0) {
-        cout << "# RB_COLL: reorder codes: " << reorder_string << "(" << reorder_codes_v.size() << ")" << endl;
-      }
       rb_coll->setReorderCodes(reorder_codes_v.data(), reorder_codes_v.size(), status);
       if (check_icu_error(status, return_json, "rb_coll with reorder")) {
         return json_object_to_json_string(return_json);
       }
     }
 
+    if (alternate_obj) {
+      rb_coll->setAttribute(UCOL_ALTERNATE_HANDLING, alternate_value, status);
+      if (check_icu_error(status, return_json, "alternate")) {
+        json_object_object_add(
+            return_json, "actual_options",
+            json_object_new_string(json_object_get_string(json_in)));
+
+        return json_object_to_json_string(return_json);
+      }
+    }
+
+    json_object *case_first_obj = json_object_object_get(json_in, "caseFirst");
+    if (case_first_obj) {
+      // TODO: Check status
+      string case_first = json_object_get_string(case_first_obj);
+      if (case_first == "lower") {
+        rb_coll->setAttribute(UCOL_CASE_FIRST, UCOL_LOWER_FIRST, status);
+      } else
+        if (case_first == "upper") {
+          rb_coll->setAttribute(UCOL_CASE_FIRST, UCOL_UPPER_FIRST, status);
+        }
+    }
+
+    json_object *case_level_obj = json_object_object_get(json_in, "caseLevel");
+    if (case_level_obj) {
+      // TODO: Check status
+      string case_level = json_object_get_string(case_level_obj);
+      if (case_level == "off") {
+        rb_coll->setAttribute(UCOL_CASE_LEVEL, UCOL_OFF, status);
+      } else
+        if (case_level == "on") {
+          rb_coll->setAttribute(UCOL_CASE_LEVEL, UCOL_ON, status);
+        }
+    }
+
+    if (strength_obj != nullptr) {
+      rb_coll->setStrength(strength_type);
+    }
+
     uni_result = rb_coll->compare(us1, us2, status);
     if (check_icu_error(status, return_json, "rb_coll->compare")) {
       return json_object_to_json_string(return_json);
@@ -292,12 +346,28 @@ auto TestCollator(json_object *json_in) -> string {
         this_locale = Locale(locale_string);
       }
       uni_coll = Collator::createInstance(this_locale, status);
+      if (check_icu_error(status, return_json,
+                          "Collator:createInstance")) {
+        json_object_object_add(
+            return_json, "actual_options",
+            json_object_new_string(json_object_get_string(json_in)));
+
+        return json_object_to_json_string(return_json);
+      }
     }
 
-    if (reorder_obj) {
-      if (debug_level > 0) {
-        cout << "# UNI_COLL: reorder codes: " << reorder_string << "(" << reorder_codes_v.size() << ")" << endl;
+    if (alternate_obj) {
+      uni_coll->setAttribute(UCOL_ALTERNATE_HANDLING, alternate_value, status);
+      if (check_icu_error(status, return_json, "alternate")) {
+        json_object_object_add(
+            return_json, "actual_options",
+            json_object_new_string(json_object_get_string(json_in)));
+
+        return json_object_to_json_string(return_json);
       }
+    }
+
+    if (reorder_obj) {
       uni_coll->setReorderCodes(reorder_codes_v.data(), reorder_codes_v.size(), status);
       if (check_icu_error(status, return_json, "uni_coll->setReorderCodes")) {
         return json_object_to_json_string(return_json);
@@ -344,9 +414,6 @@ auto TestCollator(json_object *json_in) -> string {
         values_it = values_map.find(test_value);
         if (values_it != values_map.end()) {
           // This is the value that we can set
-          if (debug_level > 0) {
-            cout << "# SETTING attribute " << key << " to " << test_value << " == " << values_it->second << endl;
-          }
           uni_coll->setAttribute(ucol_attribute, values_it->second, status);
           if (check_icu_error(
                   status, return_json,
@@ -365,29 +432,14 @@ auto TestCollator(json_object *json_in) -> string {
       values_it = val_attribute_map.find(test_value);
       if (values_it != val_attribute_map.end()) {
         uni_coll->setMaxVariable(values_it->second, status);
-        if (debug_level > 0) {
-          cout << "# SETTING maxVariable to " << values_it->second << " = " << test_value << endl;
-        }
       }
     }
 
-    // Just to check the result.
-    uni_coll->getAttribute(UCOL_ALTERNATE_HANDLING, status);  // ignore return
-    if (check_icu_error(
-            status, return_json,
-            "getet UCOL_ALTERNATE_HANDLING")) {
-      return json_object_to_json_string(return_json);
-    }
-
     // Perform the string comparison
     uni_result = uni_coll->compare(us1, us2, status);
-    if (check_icu_error( status, return_json, "uni_coll_compare")) {
+    if (check_icu_error( status, return_json, "uni_coll->compare")) {
       return json_object_to_json_string(return_json);
     }
-
-    if (uni_coll != nullptr) {
-      uni_coll->getAttribute(UCOL_ALTERNATE_HANDLING, status);  // ignore result
-    }
     delete uni_coll;
     if (check_icu_error( status, return_json, "uni_coll->getATTRIBUTE")) {
       return json_object_to_json_string(return_json);
@@ -408,7 +460,7 @@ auto TestCollator(json_object *json_in) -> string {
 
   if (!coll_result) {
     // Test did not succeed!
-    // Include data compared in the failing test
+    // Include the data compared in the failing test
     json_object* actual_values = json_object_new_object();
 
     json_object_object_add(
@@ -424,14 +476,12 @@ auto TestCollator(json_object *json_in) -> string {
         return_json, "actual_options",
         actual_values);
 
-
     if (rules_len > 0) {
+      // Show the rules that were actually found.
       json_object_object_add(
           actual_values,
           "rules_actual",
-          json_object_new_string_len(
-              rules_string.data(),
-              rules_string.size())
+          json_object_new_string(actual_rules.c_str())
                              );
     }
 
@@ -440,6 +490,7 @@ auto TestCollator(json_object *json_in) -> string {
         return_json, "compare", json_object_new_int64(uni_result));
   }
 
+  // The output
   json_object_object_add(
       return_json, "result", json_object_new_boolean(static_cast<json_bool>(coll_result)));
 

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorInputJson.java

@@ -33,9 +33,9 @@ public class CollatorInputJson implements ITestTypeInputJson {
   public int[] reorder_codes;
   public String unrecognized_script_codes;
 
-  public String reorder;
+  public String caseFirst;
 
-  public String case_first;
+  public String caseLevel;
 
   public String backwards;
 

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorTester.java

@@ -44,16 +44,19 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
     script_tags_map.put("Hani", UScript.HAN);
     script_tags_map.put("Hira", UScript.HIRAGANA);
     script_tags_map.put("Zzzz", UScript.UNKNOWN);
-
-    result.s1 = Utility.unescape((String) inputMapData.get("s1", null));
-    result.s2 = Utility.unescape((String) inputMapData.get("s2", null));
+    
+    result.s1 = (String) inputMapData.get("s1", null);
+    result.s2 = (String) inputMapData.get("s2", null);
 
     result.locale = (String) inputMapData.get("locale", null);
     result.strength = (String) inputMapData.get("strength", null);
 
     result.ignorePunctuation = (boolean) inputMapData.get("ignorePunctuation", false);
     result.line = (int) ((double) inputMapData.get("line", 0.0));
 
+    result.caseFirst = (String) inputMapData.get("caseFirst", null);
+    result.caseLevel = (String) inputMapData.get("caseLevel", null);
+
     // Resolve "&lt;"
     result.compare_type = (String) inputMapData.get("compare_type", null);
     if (result.compare_type != null && ! result.compare_type.equals("") && result.compare_type.length() > 4) {
@@ -238,6 +241,17 @@ public Collator getCollatorForInput(CollatorInputJson input) {
       } catch (Exception e) {
         return null;
       }
+      if (input.caseFirst != null) {
+        if (input.caseFirst.equals("lower")) {
+          collator.setLowerCaseFirst(true);
+        } else if (input.caseFirst.equals("upper")) {
+          collator.setUpperCaseFirst(true);
+
+        }
+      }
+      if (input.caseLevel != null && input.caseLevel.equals("on")) {
+        collator.setCaseLevel(true);
+      }
     }
 
     // ensure that ICU performs decomposition before collation in order to get proper collators,

executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/collator/icu74/CollatorTest.java

@@ -45,7 +45,7 @@ public void testAttributesAsArrayList() {
   public void testRule004() {
     // in ICU 76.1 data
     String testInput =
-        "{\"test_type\": \"collation\", \"compare_type\":\"&lt;3\",\"s1\":\"\u0002\",\"s2\":\"\u0300\",\"source_file\":\"collationtest.txt\",\"line\":43,\"label\":\"00002\",\"test_description\":\"simple CEs &amp; expansions\",\"rules\":\"&\\u0001<<<\\u0300&9<\\u0000&\\uA00A\\uA00B=\\uA002&\\uA00A\\uA00B\\u00050005=\\uA003\",\"hexhash\":\"7d3d23fab7f34c1cd44e90b40f7ed33c5bb317ba\"}";
+        "{\"test_type\": \"collation\", \"compare_type\":\"&lt;3\",\"s1\":\"\u0002\",\"s2\":\"\u0300\",\"source_file\":\"collationtest.txt\",\"line\":43,\"label\":\"00002\",\"test_description\":\"simple CEs & expansions\",\"rules\":\"&\\u0001<<<\\u0300&9<\\u0000&\\uA00A\\uA00B=\\uA002&\\uA00A\\uA00B\\u00050005=\\uA003\",\"hexhash\":\"7d3d23fab7f34c1cd44e90b40f7ed33c5bb317ba\"}";
 
     CollatorOutputJson output =
         (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
@@ -175,5 +175,59 @@ public void testSHIFTED_SHORT() {
 
     assertTrue(output.result);
   }
+
+  @Test
+  public void testNON_IGNORABLE_0235457() {
+    // Unexpected test failure
+    String testInput =
+"{\"test_type\": \"collation\",   \"label\": \"0235457\",   \"s1\": \"\\ufe68A\",   \"s2\": \"\\\\b\",   \"line\": 8263,   \"source_file\": \"CollationTest_NON_IGNORABLE_SHORT.txt\",   \"hexhash\": \"c5dbf2d2a1b6eba940b76b840f2d991fde8f29c7\"  }";
+    CollatorOutputJson output = (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Test
+  public void test_01195_rules() {
+    // Unexpected test failure
+    String testInput =
+        "{\"test_type\": \"collation\",\"compare_type\": \"<1\",   \"s1\": \"opd\",   \"s2\": \"op\\u0109\",   \"source_file\": \"collationtest.txt\",   \"line\": 2104,   \"label\": \"01195\",   \"test_description\": \"fall back to mappings with shorter prefixes, not immediately to ones with no prefixes\",   \"rules\": \"&x=op|\\u0109&y=p|\\u00e7\",   \"hexhash\": \"bc322b1e989cd75f5956b758dbf770b94f4011ff\"  }";
+
+    CollatorOutputJson output = (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Test
+  public void test_00312_rules() {
+    // Unexpected test failure
+    String testInput =
+        "{\"test_type\": \"collation\", \"compare_type\":\"<3\",\"s1\":\"aAt\",\"s2\":\"aa\",\"source_file\":\"collationtest.txt\",\"line\":658,\"label\":\"00312\",\"test_description\":\"tertiary CEs, tertiary, caseLevel=off, caseFirst=upper\",\"rules\":\"&\\\\u0001<<<t<<<T\",\"caseFirst\":\"upper\",\"hexhash\":\"453fdaeaaaf99825d778f1573de2612aafda84b2\"}";
+
+    CollatorOutputJson output = (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Test
+  public void test_00536_rules() {
+    // Unexpected test failure
+    String testInput =
+        "{\"test_type\": \"collation\", \"compare_type\":\"<1\",\"s1\":\"z\",\"s2\":\" \",\"source_file\":\"collationtest.txt\",\"line\":1047,\"label\":\"00536\",\"test_description\":\"adjust special reset positions according to previous rules, CLDR ticket 6070\",\"rules\":\"&[last primary ignorable]<<x<<<y&[last primary ignorable]<<z\",\"hexhash\":\"49f1099a0032106e6a3861bb5d228568c6a74d47\"}";
+
+    CollatorOutputJson output = (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Test
+  public void test_00543_rules() {
+    // Unexpected test failure
+    String testInput =
+        "{\"test_type\": \"collation\", \"compare_type\":\"<1\",\"s1\":\"\uFDD1\u00A0\",\"s2\":\"t\",\"source_file\":\"collationtest.txt\",\"line\":1070,\"label\":\"00543\",\"test_description\":\"adjust special reset positions according to previous rules, CLDR ticket 6070\",\"rules\":\"&[before 2][first variable]<<z&[before 2][first variable]<<y&[before 3][first variable]<<<x&[before 3][first variable]<<<w&[before 1][first variable]<v&[before 2][first variable]<<u&[before 3][first variable]<<<t&[before 2]\\\\uFDD1\\\\xA0<<s\",\"hexhash\":\"295bcd43ae62d58b89137aa45401f386881fc189\"}";
+
+    CollatorOutputJson output = (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
 }
 

testgen/generators/collation.py

@@ -45,7 +45,8 @@ def set_patterns(self):
             self.root_locale,
             self.test_line,
             self.attribute_test,
-            self.reorder_test
+            self.reorder_test,
+            self.rule_header_pattern
         ]
 
         self.rule_breakout_patterns = [
@@ -172,6 +173,7 @@ def check_parse_compare(self, line_index, lines, filename):
             if is_comparison_match := self.comparison_line.match(line_in):
                 compare_type = is_comparison_match.group(1)
                 raw_string2 = is_comparison_match.group(3)
+                # TODO: Handle raw_string.replace('\\x', '\u00')
                 string2 = ''
                 try:
                     string2 = raw_string2.rstrip()  # Don't do any unescaping
@@ -182,9 +184,13 @@ def check_parse_compare(self, line_index, lines, filename):
                 # Special cases for comparing only with \u0020 or \u000a
                 if string2 == '' and raw_line.find('\\u0020') > 0:
                     string2 = '\u0020'
+                if string2 == '' and raw_line.find('\\x20') > 0:
+                    string2 = '\u0020'
                 if string2 == '' and raw_line.find('\\u000A') > 0:
                     string2 = '\u000a'
-
+                # A very special case. Generalize?\ to hand \xYZ?
+                if raw_line.find('\\uFDD1\\xA0') > 0:
+                    string2 = '\ufdd1\u00a0'
                 new_test = {
                     'compare_type': compare_type,
                     's1': string1,